mcp-scraper 0.3.1 → 0.3.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +4 -1
- package/dist/bin/api-server.cjs +2050 -832
- package/dist/bin/api-server.cjs.map +1 -1
- package/dist/bin/api-server.js +2 -2
- package/dist/bin/browser-agent-stdio-server.cjs +1 -1
- package/dist/bin/browser-agent-stdio-server.cjs.map +1 -1
- package/dist/bin/browser-agent-stdio-server.js +2 -2
- package/dist/bin/mcp-scraper-cli.cjs +1 -1
- package/dist/bin/mcp-scraper-cli.cjs.map +1 -1
- package/dist/bin/mcp-scraper-cli.js +1 -1
- package/dist/bin/mcp-scraper-combined-stdio-server.cjs +351 -4
- package/dist/bin/mcp-scraper-combined-stdio-server.cjs.map +1 -1
- package/dist/bin/mcp-scraper-combined-stdio-server.js +4 -4
- package/dist/bin/mcp-scraper-install.cjs +3 -3
- package/dist/bin/mcp-scraper-install.cjs.map +1 -1
- package/dist/bin/mcp-scraper-install.js +2 -2
- package/dist/bin/mcp-stdio-server.cjs +349 -2
- package/dist/bin/mcp-stdio-server.cjs.map +1 -1
- package/dist/bin/mcp-stdio-server.js +2 -2
- package/dist/bin/paa-harvest.cjs +3 -1
- package/dist/bin/paa-harvest.cjs.map +1 -1
- package/dist/bin/paa-harvest.js +1 -1
- package/dist/{chunk-SXTXMFEQ.js → chunk-AZ5PKU4F.js} +34 -1
- package/dist/chunk-AZ5PKU4F.js.map +1 -0
- package/dist/{chunk-Q4STSM63.js → chunk-C3FGVJWH.js} +3 -3
- package/dist/chunk-C3FGVJWH.js.map +1 -0
- package/dist/{chunk-55T4SRLJ.js → chunk-MGWGZBL5.js} +350 -3
- package/dist/chunk-MGWGZBL5.js.map +1 -0
- package/dist/chunk-PVXDEREW.js +7 -0
- package/dist/chunk-PVXDEREW.js.map +1 -0
- package/dist/{chunk-LICHCMV6.js → chunk-RLBJ3QNC.js} +2 -2
- package/dist/{chunk-IPW4LFOT.js → chunk-UWSG3C5J.js} +4 -2
- package/dist/chunk-UWSG3C5J.js.map +1 -0
- package/dist/index.cjs +3 -1
- package/dist/index.cjs.map +1 -1
- package/dist/index.js +1 -1
- package/dist/{server-AXPNL2RV.js → server-D3XHIEQN.js} +997 -253
- package/dist/server-D3XHIEQN.js.map +1 -0
- package/dist/{worker-SLQ375UG.js → worker-56IXWOQU.js} +3 -3
- package/docs/mcp-tool-craft-lint.generated.md +5 -2
- package/docs/mcp-tool-manifest.generated.json +80 -5
- package/package.json +1 -1
- package/dist/chunk-55T4SRLJ.js.map +0 -1
- package/dist/chunk-D4JDGKOV.js +0 -7
- package/dist/chunk-D4JDGKOV.js.map +0 -1
- package/dist/chunk-IPW4LFOT.js.map +0 -1
- package/dist/chunk-Q4STSM63.js.map +0 -1
- package/dist/chunk-SXTXMFEQ.js.map +0 -1
- package/dist/server-AXPNL2RV.js.map +0 -1
- /package/dist/{chunk-LICHCMV6.js.map → chunk-RLBJ3QNC.js.map} +0 -0
- /package/dist/{worker-SLQ375UG.js.map → worker-56IXWOQU.js.map} +0 -0
package/dist/bin/api-server.cjs
CHANGED
|
@@ -4098,9 +4098,9 @@ async function extractBrandingFromPage(page) {
|
|
|
4098
4098
|
colors: {
|
|
4099
4099
|
primary: primary && !isTransparentOrWhite(primary) ? primary : null,
|
|
4100
4100
|
accent: accent && !isTransparentOrWhite(accent) ? accent : null,
|
|
4101
|
-
background: background
|
|
4102
|
-
text: text
|
|
4103
|
-
heading: heading
|
|
4101
|
+
background: background ?? null,
|
|
4102
|
+
text: text ?? null,
|
|
4103
|
+
heading: heading ?? null
|
|
4104
4104
|
},
|
|
4105
4105
|
fonts: {
|
|
4106
4106
|
body: firstFont(raw.bodyFont ?? ""),
|
|
@@ -4372,8 +4372,8 @@ async function downloadAsset(url, destDir, filename) {
|
|
|
4372
4372
|
}
|
|
4373
4373
|
const writer = (0, import_node_fs.createWriteStream)(dest);
|
|
4374
4374
|
await (0, import_promises2.pipeline)(import_node_stream.Readable.fromWeb(res.body), writer);
|
|
4375
|
-
const { statSync:
|
|
4376
|
-
const sizeBytes =
|
|
4375
|
+
const { statSync: statSync3 } = await import("fs");
|
|
4376
|
+
const sizeBytes = statSync3(dest).size;
|
|
4377
4377
|
return { savedPath: dest, sizeBytes, mimeType };
|
|
4378
4378
|
}
|
|
4379
4379
|
async function harvestPageMedia(html, pageUrl, options = {}) {
|
|
@@ -8590,8 +8590,8 @@ var init_site_audit_middleware = __esm({
|
|
|
8590
8590
|
|
|
8591
8591
|
// src/api/site-audit-routes.ts
|
|
8592
8592
|
function isPathInside(root, target) {
|
|
8593
|
-
const
|
|
8594
|
-
return
|
|
8593
|
+
const relative2 = import_node_path2.default.relative(root, target);
|
|
8594
|
+
return relative2 === "" || !relative2.startsWith("..") && !import_node_path2.default.isAbsolute(relative2);
|
|
8595
8595
|
}
|
|
8596
8596
|
var import_node_path2, import_node_os2, import_hono, import_zod11, siteAuditApp;
|
|
8597
8597
|
var init_site_audit_routes = __esm({
|
|
@@ -8827,6 +8827,9 @@ var init_rates = __esm({
|
|
|
8827
8827
|
maps_review: 50,
|
|
8828
8828
|
fb_search: 50,
|
|
8829
8829
|
fb_transcribe: 50,
|
|
8830
|
+
instagram_profile: 100,
|
|
8831
|
+
instagram_media: 100,
|
|
8832
|
+
instagram_transcribe: 50,
|
|
8830
8833
|
browser_minute: 4e3
|
|
8831
8834
|
};
|
|
8832
8835
|
MC_PER_BROWSER_MS = MC_COSTS.browser_minute / 6e4;
|
|
@@ -8927,6 +8930,30 @@ var init_rates = __esm({
|
|
|
8927
8930
|
unit: "per call",
|
|
8928
8931
|
notes: "Whisper transcription of Facebook ad video via fal.ai."
|
|
8929
8932
|
},
|
|
8933
|
+
{
|
|
8934
|
+
key: "instagram_profile",
|
|
8935
|
+
label: "Instagram profile content discovery",
|
|
8936
|
+
aliases: ["instagram_profile_content", "instagram profile", "instagram content list", "ig profile"],
|
|
8937
|
+
credits: mcToCredits(MC_COSTS.instagram_profile),
|
|
8938
|
+
unit: "per profile scan",
|
|
8939
|
+
notes: "Browser extraction of public Instagram profile grid links. Complete history may require a logged-in profile."
|
|
8940
|
+
},
|
|
8941
|
+
{
|
|
8942
|
+
key: "instagram_media",
|
|
8943
|
+
label: "Instagram media download",
|
|
8944
|
+
aliases: ["instagram_media_download", "instagram reel download", "instagram post download", "ig media"],
|
|
8945
|
+
credits: mcToCredits(MC_COSTS.instagram_media),
|
|
8946
|
+
unit: "per post or reel",
|
|
8947
|
+
notes: "Extracts post text, image URLs, and reel audio/video tracks, with local downloads when the server can write files."
|
|
8948
|
+
},
|
|
8949
|
+
{
|
|
8950
|
+
key: "instagram_transcribe",
|
|
8951
|
+
label: "Instagram media transcription",
|
|
8952
|
+
aliases: ["instagram transcript", "instagram reel transcript", "ig transcribe"],
|
|
8953
|
+
credits: mcToCredits(MC_COSTS.instagram_transcribe),
|
|
8954
|
+
unit: "per call",
|
|
8955
|
+
notes: "Whisper transcription of selected Instagram audio/video media via fal.ai."
|
|
8956
|
+
},
|
|
8930
8957
|
{
|
|
8931
8958
|
key: "browser_minute",
|
|
8932
8959
|
label: "Interactive browser session",
|
|
@@ -8985,6 +9012,12 @@ var init_rates = __esm({
|
|
|
8985
9012
|
FB_TRANSCRIBE: "fb_transcribe",
|
|
8986
9013
|
FB_SEARCH_REFUND: "fb_search_refund",
|
|
8987
9014
|
FB_TRANSCRIBE_REFUND: "fb_transcribe_refund",
|
|
9015
|
+
INSTAGRAM_PROFILE: "instagram_profile",
|
|
9016
|
+
INSTAGRAM_PROFILE_REFUND: "instagram_profile_refund",
|
|
9017
|
+
INSTAGRAM_MEDIA: "instagram_media",
|
|
9018
|
+
INSTAGRAM_MEDIA_REFUND: "instagram_media_refund",
|
|
9019
|
+
INSTAGRAM_TRANSCRIBE: "instagram_transcribe",
|
|
9020
|
+
INSTAGRAM_TRANSCRIBE_REFUND: "instagram_transcribe_refund",
|
|
8988
9021
|
BROWSER_SESSION: "browser_session"
|
|
8989
9022
|
};
|
|
8990
9023
|
}
|
|
@@ -9297,7 +9330,9 @@ var init_BrowserDriver = __esm({
|
|
|
9297
9330
|
}
|
|
9298
9331
|
const launchOpts = {
|
|
9299
9332
|
headless: config.headless,
|
|
9300
|
-
proxy: config.proxy ? { server: config.proxy } : void 0
|
|
9333
|
+
proxy: config.proxy ? { server: config.proxy } : void 0,
|
|
9334
|
+
...config.browserExecutablePath ? { executablePath: config.browserExecutablePath } : {},
|
|
9335
|
+
...config.ignoreDefaultArgs ? { ignoreDefaultArgs: config.ignoreDefaultArgs } : {}
|
|
9301
9336
|
};
|
|
9302
9337
|
const ctxOpts = rankCheckContextOptions(config);
|
|
9303
9338
|
if (config.profileDir) {
|
|
@@ -10283,8 +10318,8 @@ async function fetchViaKernelInnertube(videoId) {
|
|
|
10283
10318
|
const tracks = data?.captions?.playerCaptionsTracklistRenderer?.captionTracks ?? [];
|
|
10284
10319
|
if (tracks.length === 0) return null;
|
|
10285
10320
|
const track = tracks.find((t) => t.languageCode === "en") ?? tracks[0];
|
|
10286
|
-
const
|
|
10287
|
-
const xmlResp = await fetch(`${track.baseUrl}${
|
|
10321
|
+
const sep2 = track.baseUrl.includes("?") ? "&" : "?";
|
|
10322
|
+
const xmlResp = await fetch(`${track.baseUrl}${sep2}fmt=json3`);
|
|
10288
10323
|
return xmlResp.ok ? xmlResp.text() : null;
|
|
10289
10324
|
}, videoId);
|
|
10290
10325
|
if (!xml) return null;
|
|
@@ -12492,15 +12527,8 @@ var init_schemas3 = __esm({
|
|
|
12492
12527
|
}
|
|
12493
12528
|
});
|
|
12494
12529
|
|
|
12495
|
-
// src/
|
|
12496
|
-
function
|
|
12497
|
-
return { error_code: "invalid_request", message };
|
|
12498
|
-
}
|
|
12499
|
-
function isAllowedFacebookOrganicUrl(url) {
|
|
12500
|
-
const hostname = url.hostname.toLowerCase().replace(/^www\./, "").replace(/^m\./, "");
|
|
12501
|
-
return hostname === "facebook.com" || hostname === "fb.watch";
|
|
12502
|
-
}
|
|
12503
|
-
function facebookTranscriptMarkdown(title, text, chunks, durationMs) {
|
|
12530
|
+
// src/services/media-transcription.ts
|
|
12531
|
+
function transcriptMarkdown(title, text, chunks, durationMs) {
|
|
12504
12532
|
const fmtTs2 = (s) => `${Math.floor(s / 60)}:${String(Math.floor(s % 60)).padStart(2, "0")}`;
|
|
12505
12533
|
const lines = [title, "", `*Transcribed in ${(durationMs / 1e3).toFixed(1)}s*`, "", "## Full Text", "", text, ""];
|
|
12506
12534
|
if (chunks.length) {
|
|
@@ -12511,10 +12539,10 @@ function facebookTranscriptMarkdown(title, text, chunks, durationMs) {
|
|
|
12511
12539
|
}
|
|
12512
12540
|
return lines.join("\n");
|
|
12513
12541
|
}
|
|
12514
|
-
async function
|
|
12542
|
+
async function transcribeMediaUrl(mediaUrl, markdownTitle = "# Media Transcript") {
|
|
12515
12543
|
const startMs = Date.now();
|
|
12516
12544
|
const result = await import_client3.fal.subscribe("fal-ai/wizper", {
|
|
12517
|
-
input: { audio_url:
|
|
12545
|
+
input: { audio_url: mediaUrl, task: "transcribe", language: "en" },
|
|
12518
12546
|
logs: false,
|
|
12519
12547
|
pollInterval: 3e3
|
|
12520
12548
|
});
|
|
@@ -12526,9 +12554,28 @@ async function transcribeFacebookVideoUrl(videoUrl, markdownTitle = "# Facebook
|
|
|
12526
12554
|
text,
|
|
12527
12555
|
chunks,
|
|
12528
12556
|
durationMs,
|
|
12529
|
-
markdown:
|
|
12557
|
+
markdown: transcriptMarkdown(markdownTitle, text, chunks, durationMs)
|
|
12530
12558
|
};
|
|
12531
12559
|
}
|
|
12560
|
+
var import_client3;
|
|
12561
|
+
var init_media_transcription = __esm({
|
|
12562
|
+
"src/services/media-transcription.ts"() {
|
|
12563
|
+
"use strict";
|
|
12564
|
+
import_client3 = require("@fal-ai/client");
|
|
12565
|
+
}
|
|
12566
|
+
});
|
|
12567
|
+
|
|
12568
|
+
// src/api/facebook-ad-routes.ts
|
|
12569
|
+
function invalidRequest(message) {
|
|
12570
|
+
return { error_code: "invalid_request", message };
|
|
12571
|
+
}
|
|
12572
|
+
function isAllowedFacebookOrganicUrl(url) {
|
|
12573
|
+
const hostname = url.hostname.toLowerCase().replace(/^www\./, "").replace(/^m\./, "");
|
|
12574
|
+
return hostname === "facebook.com" || hostname === "fb.watch";
|
|
12575
|
+
}
|
|
12576
|
+
async function transcribeFacebookVideoUrl(videoUrl, markdownTitle = "# Facebook Video Transcript") {
|
|
12577
|
+
return transcribeMediaUrl(videoUrl, markdownTitle);
|
|
12578
|
+
}
|
|
12532
12579
|
async function detectSoftBlock(driver) {
|
|
12533
12580
|
const page = driver.getPage();
|
|
12534
12581
|
const bodyText = await page.evaluate(() => document.body?.innerText ?? "").catch(() => "");
|
|
@@ -12557,7 +12604,7 @@ async function kernelLaunchOptsResidential() {
|
|
|
12557
12604
|
}
|
|
12558
12605
|
return { headless: true, kernelApiKey: browserServiceApiKey(), kernelProxyId: proxyId, viewport: { width: 1280, height: 900 }, locale: "en-US" };
|
|
12559
12606
|
}
|
|
12560
|
-
var import_hono4, import_zod16,
|
|
12607
|
+
var import_hono4, import_zod16, import_client4, FacebookAdBodySchema, FacebookPageIntelBodySchema, FacebookTranscribeBodySchema, FacebookVideoTranscribeBodySchema, FacebookSearchBodySchema, FacebookMediaBodySchema, facebookAdApp, ALLOWED_MEDIA_HOSTS;
|
|
12561
12608
|
var init_facebook_ad_routes = __esm({
|
|
12562
12609
|
"src/api/facebook-ad-routes.ts"() {
|
|
12563
12610
|
"use strict";
|
|
@@ -12572,10 +12619,11 @@ var init_facebook_ad_routes = __esm({
|
|
|
12572
12619
|
init_FacebookOrganicVideoExtractor();
|
|
12573
12620
|
init_kernel_proxy_resolver();
|
|
12574
12621
|
init_schemas3();
|
|
12575
|
-
|
|
12622
|
+
import_client4 = require("@fal-ai/client");
|
|
12576
12623
|
init_api_auth();
|
|
12577
12624
|
init_url_utils();
|
|
12578
12625
|
init_concurrency_gates();
|
|
12626
|
+
init_media_transcription();
|
|
12579
12627
|
FacebookAdBodySchema = import_zod16.z.object({
|
|
12580
12628
|
url: import_zod16.z.string().trim().optional(),
|
|
12581
12629
|
libraryId: import_zod16.z.string().trim().optional(),
|
|
@@ -12714,7 +12762,7 @@ var init_facebook_ad_routes = __esm({
|
|
|
12714
12762
|
metadata: { videoUrl }
|
|
12715
12763
|
});
|
|
12716
12764
|
if (!gate.ok) return c.json(concurrencyLimitExceededResponse(gate), 429, { "Retry-After": String(gate.retryAfterSeconds) });
|
|
12717
|
-
|
|
12765
|
+
import_client4.fal.config({ credentials: process.env.FAL_KEY });
|
|
12718
12766
|
let debited = false;
|
|
12719
12767
|
try {
|
|
12720
12768
|
const { ok, balance_mc } = await debitMc(fbUser.id, MC_COSTS.fb_transcribe, LedgerOperation.FB_TRANSCRIBE, videoUrl);
|
|
@@ -12753,7 +12801,7 @@ var init_facebook_ad_routes = __esm({
|
|
|
12753
12801
|
metadata: { url: sourceUrl.href }
|
|
12754
12802
|
});
|
|
12755
12803
|
if (!gate.ok) return c.json(concurrencyLimitExceededResponse(gate), 429, { "Retry-After": String(gate.retryAfterSeconds) });
|
|
12756
|
-
|
|
12804
|
+
import_client4.fal.config({ credentials: process.env.FAL_KEY });
|
|
12757
12805
|
const driver = new BrowserDriver();
|
|
12758
12806
|
let debited = false;
|
|
12759
12807
|
try {
|
|
@@ -12930,6 +12978,824 @@ var init_facebook_ad_routes = __esm({
|
|
|
12930
12978
|
}
|
|
12931
12979
|
});
|
|
12932
12980
|
|
|
12981
|
+
// src/lib/chrome-profiles.ts
|
|
12982
|
+
function recommendedKernelProfileName(value) {
|
|
12983
|
+
return value.trim().toLowerCase().replace(/@/g, "-").replace(/[^a-z0-9]+/g, "-").replace(/^-+|-+$/g, "").slice(0, 80) || "browser-profile";
|
|
12984
|
+
}
|
|
12985
|
+
var import_promises4, import_node_os3, import_node_path5;
|
|
12986
|
+
var init_chrome_profiles = __esm({
|
|
12987
|
+
"src/lib/chrome-profiles.ts"() {
|
|
12988
|
+
"use strict";
|
|
12989
|
+
import_promises4 = require("fs/promises");
|
|
12990
|
+
import_node_os3 = require("os");
|
|
12991
|
+
import_node_path5 = require("path");
|
|
12992
|
+
}
|
|
12993
|
+
});
|
|
12994
|
+
|
|
12995
|
+
// src/lib/local-browser-profiles.ts
|
|
12996
|
+
function defaultImportedBrowserProfilesDir() {
|
|
12997
|
+
return process.env.MCP_SCRAPER_BROWSER_PROFILE_BASE_DIR?.trim() || (0, import_node_path6.join)((0, import_node_os4.homedir)(), ".mcp-scraper", "browser-profiles");
|
|
12998
|
+
}
|
|
12999
|
+
function defaultChromeExecutablePath() {
|
|
13000
|
+
const envPath = process.env.MCP_SCRAPER_BROWSER_EXECUTABLE?.trim() || process.env.CHROME_PATH?.trim();
|
|
13001
|
+
if (envPath) return envPath;
|
|
13002
|
+
const candidates = process.platform === "darwin" ? ["/Applications/Google Chrome.app/Contents/MacOS/Google Chrome"] : process.platform === "win32" ? [
|
|
13003
|
+
(0, import_node_path6.join)(process.env.PROGRAMFILES ?? "C:\\Program Files", "Google", "Chrome", "Application", "chrome.exe"),
|
|
13004
|
+
(0, import_node_path6.join)(process.env["PROGRAMFILES(X86)"] ?? "C:\\Program Files (x86)", "Google", "Chrome", "Application", "chrome.exe")
|
|
13005
|
+
] : ["/usr/bin/google-chrome", "/usr/bin/google-chrome-stable", "/usr/bin/chromium-browser", "/usr/bin/chromium"];
|
|
13006
|
+
return candidates.find((candidate) => (0, import_node_fs3.existsSync)(candidate));
|
|
13007
|
+
}
|
|
13008
|
+
function localBrowserModeEnabled() {
|
|
13009
|
+
return process.env.MCP_SCRAPER_BROWSER_MODE?.trim().toLowerCase() === "local";
|
|
13010
|
+
}
|
|
13011
|
+
function importedProfileName(value) {
|
|
13012
|
+
return recommendedKernelProfileName(value);
|
|
13013
|
+
}
|
|
13014
|
+
function importedProfileRoot(name, outputDir = defaultImportedBrowserProfilesDir()) {
|
|
13015
|
+
return (0, import_node_path6.join)(outputDir, importedProfileName(name));
|
|
13016
|
+
}
|
|
13017
|
+
function importedProfileManifestPath(name, outputDir = defaultImportedBrowserProfilesDir()) {
|
|
13018
|
+
return (0, import_node_path6.join)(importedProfileRoot(name, outputDir), "manifest.json");
|
|
13019
|
+
}
|
|
13020
|
+
async function readExistingManifest(path6) {
|
|
13021
|
+
try {
|
|
13022
|
+
return JSON.parse(await (0, import_promises5.readFile)(path6, "utf8"));
|
|
13023
|
+
} catch {
|
|
13024
|
+
return null;
|
|
13025
|
+
}
|
|
13026
|
+
}
|
|
13027
|
+
async function loadImportedBrowserProfile(name = process.env.MCP_SCRAPER_BROWSER_PROFILE?.trim() || "default", outputDir = defaultImportedBrowserProfilesDir()) {
|
|
13028
|
+
const profileName = importedProfileName(name);
|
|
13029
|
+
const manifestPath = importedProfileManifestPath(profileName, outputDir);
|
|
13030
|
+
const manifest = await readExistingManifest(manifestPath);
|
|
13031
|
+
if (!manifest) {
|
|
13032
|
+
throw new Error(
|
|
13033
|
+
`Imported browser profile "${profileName}" was not found. Run: mcp-scraper-cli browser import-chrome --name ${profileName}`
|
|
13034
|
+
);
|
|
13035
|
+
}
|
|
13036
|
+
return manifest;
|
|
13037
|
+
}
|
|
13038
|
+
var import_node_fs3, import_promises5, import_node_os4, import_node_path6;
|
|
13039
|
+
var init_local_browser_profiles = __esm({
|
|
13040
|
+
"src/lib/local-browser-profiles.ts"() {
|
|
13041
|
+
"use strict";
|
|
13042
|
+
import_node_fs3 = require("fs");
|
|
13043
|
+
import_promises5 = require("fs/promises");
|
|
13044
|
+
import_node_os4 = require("os");
|
|
13045
|
+
import_node_path6 = require("path");
|
|
13046
|
+
init_chrome_profiles();
|
|
13047
|
+
}
|
|
13048
|
+
});
|
|
13049
|
+
|
|
13050
|
+
// src/extractor/InstagramContentExtractor.ts
|
|
13051
|
+
function htmlDecode2(value) {
|
|
13052
|
+
return value.replace(/&/g, "&").replace(/"/g, '"').replace(/'/g, "'").replace(/</g, "<").replace(/>/g, ">");
|
|
13053
|
+
}
|
|
13054
|
+
function decodeEscapedValue2(raw) {
|
|
13055
|
+
let out = raw;
|
|
13056
|
+
try {
|
|
13057
|
+
out = JSON.parse(`"${raw.replace(/"/g, '\\"')}"`);
|
|
13058
|
+
} catch {
|
|
13059
|
+
out = raw.replace(/\\u0025/g, "%").replace(/\\u0026/g, "&").replace(/\\u003D/g, "=").replace(/\\u003d/g, "=").replace(/\\"/g, '"').replace(/\\\//g, "/");
|
|
13060
|
+
}
|
|
13061
|
+
return htmlDecode2(out);
|
|
13062
|
+
}
|
|
13063
|
+
function metadataContent2(html, key) {
|
|
13064
|
+
const escaped = key.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
|
13065
|
+
const re = new RegExp(`<meta\\s+[^>]*(?:property|name)=["']${escaped}["'][^>]*content=["']([^"']+)["'][^>]*>`, "i");
|
|
13066
|
+
const m = html.match(re);
|
|
13067
|
+
return m ? htmlDecode2(m[1]).trim() : null;
|
|
13068
|
+
}
|
|
13069
|
+
function instagramContentTypeFromPath(rawPath) {
|
|
13070
|
+
const match = rawPath.match(CONTENT_PATH_RE);
|
|
13071
|
+
if (!match) return null;
|
|
13072
|
+
const type = match[1].toLowerCase() === "p" ? "post" : match[1].toLowerCase();
|
|
13073
|
+
return { type, shortcode: match[2] };
|
|
13074
|
+
}
|
|
13075
|
+
function normalizeInstagramProfileInput(input) {
|
|
13076
|
+
const raw = input.trim();
|
|
13077
|
+
if (!raw) throw new Error("Instagram handle or profile URL is required");
|
|
13078
|
+
if (/^https?:\/\//i.test(raw) || /^www\.instagram\.com/i.test(raw) || /^instagram\.com/i.test(raw)) {
|
|
13079
|
+
const parsed = new URL(/^https?:\/\//i.test(raw) ? raw : `https://${raw}`);
|
|
13080
|
+
const host = parsed.hostname.toLowerCase().replace(/^www\./, "").replace(/^m\./, "");
|
|
13081
|
+
if (host !== "instagram.com") throw new Error("Profile URL must be on instagram.com");
|
|
13082
|
+
const first = parsed.pathname.split("/").filter(Boolean)[0];
|
|
13083
|
+
if (!first || ["p", "reel", "tv", "stories", "explore"].includes(first.toLowerCase())) {
|
|
13084
|
+
throw new Error("Profile URL must point to an Instagram profile, not a post, reel, story, or explore page");
|
|
13085
|
+
}
|
|
13086
|
+
return { handle: first, profileUrl: `https://www.instagram.com/${first}/` };
|
|
13087
|
+
}
|
|
13088
|
+
const handle = raw.replace(/^@/, "").replace(/^\/+|\/+$/g, "");
|
|
13089
|
+
if (!/^[A-Za-z0-9._]{1,30}$/.test(handle)) throw new Error("Invalid Instagram handle");
|
|
13090
|
+
return { handle, profileUrl: `https://www.instagram.com/${handle}/` };
|
|
13091
|
+
}
|
|
13092
|
+
function isAllowedInstagramPageUrl(url) {
|
|
13093
|
+
const host = url.hostname.toLowerCase().replace(/^www\./, "").replace(/^m\./, "");
|
|
13094
|
+
return host === "instagram.com";
|
|
13095
|
+
}
|
|
13096
|
+
function normalizeInstagramMediaUrl(raw) {
|
|
13097
|
+
try {
|
|
13098
|
+
const decoded = decodeEscapedValue2(raw);
|
|
13099
|
+
const parsed = new URL(decoded);
|
|
13100
|
+
if (!CDN_HOST_RE.test(parsed.hostname.toLowerCase())) return null;
|
|
13101
|
+
parsed.searchParams.delete("bytestart");
|
|
13102
|
+
parsed.searchParams.delete("byteend");
|
|
13103
|
+
return parsed.href;
|
|
13104
|
+
} catch {
|
|
13105
|
+
return null;
|
|
13106
|
+
}
|
|
13107
|
+
}
|
|
13108
|
+
function parseInstagramEfg(url) {
|
|
13109
|
+
try {
|
|
13110
|
+
const efg = new URL(url).searchParams.get("efg");
|
|
13111
|
+
if (!efg) return null;
|
|
13112
|
+
return JSON.parse(Buffer.from(efg, "base64").toString("utf8"));
|
|
13113
|
+
} catch {
|
|
13114
|
+
return null;
|
|
13115
|
+
}
|
|
13116
|
+
}
|
|
13117
|
+
function trackFromUrl(url) {
|
|
13118
|
+
const normalized = normalizeInstagramMediaUrl(url);
|
|
13119
|
+
if (!normalized) return null;
|
|
13120
|
+
const parsed = parseInstagramEfg(normalized);
|
|
13121
|
+
const tag = typeof parsed?.vencode_tag === "string" ? parsed.vencode_tag : null;
|
|
13122
|
+
const streamType = tag?.toLowerCase().includes("audio") ? "audio" : normalized.includes(".mp4") ? "video" : "unknown";
|
|
13123
|
+
return {
|
|
13124
|
+
url: normalized,
|
|
13125
|
+
streamType,
|
|
13126
|
+
bitrate: typeof parsed?.bitrate === "number" ? parsed.bitrate : null,
|
|
13127
|
+
durationSec: typeof parsed?.duration_s === "number" ? parsed.duration_s : null,
|
|
13128
|
+
vencodeTag: tag,
|
|
13129
|
+
width: typeof parsed?.width === "number" ? parsed.width : null,
|
|
13130
|
+
height: typeof parsed?.height === "number" ? parsed.height : null
|
|
13131
|
+
};
|
|
13132
|
+
}
|
|
13133
|
+
function extractInstagramMp4UrlsFromText(text) {
|
|
13134
|
+
const urls = [];
|
|
13135
|
+
const seen = /* @__PURE__ */ new Set();
|
|
13136
|
+
const patterns = [
|
|
13137
|
+
/https?:\\?\/\\?\/[^"'<>\s]+?\.mp4(?:\?[^"'<>\s]+)?/g,
|
|
13138
|
+
/https?:%5C?\/%5C?\/[^"'<>\s]+?\.mp4(?:\?[^"'<>\s]+)?/g
|
|
13139
|
+
];
|
|
13140
|
+
for (const pattern of patterns) {
|
|
13141
|
+
let match;
|
|
13142
|
+
while ((match = pattern.exec(text)) !== null) {
|
|
13143
|
+
const normalized = normalizeInstagramMediaUrl(match[0].replace(/%5C/gi, "\\"));
|
|
13144
|
+
if (!normalized || seen.has(normalized)) continue;
|
|
13145
|
+
seen.add(normalized);
|
|
13146
|
+
urls.push(normalized);
|
|
13147
|
+
}
|
|
13148
|
+
}
|
|
13149
|
+
return urls;
|
|
13150
|
+
}
|
|
13151
|
+
function instagramTracksFromUrls(urls) {
|
|
13152
|
+
const seen = /* @__PURE__ */ new Set();
|
|
13153
|
+
const tracks = [];
|
|
13154
|
+
for (const raw of urls) {
|
|
13155
|
+
const track = trackFromUrl(raw);
|
|
13156
|
+
if (!track || seen.has(track.url)) continue;
|
|
13157
|
+
seen.add(track.url);
|
|
13158
|
+
tracks.push(track);
|
|
13159
|
+
}
|
|
13160
|
+
return tracks;
|
|
13161
|
+
}
|
|
13162
|
+
function selectInstagramTracks(tracks) {
|
|
13163
|
+
const selectedVideoTrack = tracks.filter((track) => track.streamType === "video").sort((a, b) => {
|
|
13164
|
+
const aPixels = (a.width ?? 0) * (a.height ?? 0);
|
|
13165
|
+
const bPixels = (b.width ?? 0) * (b.height ?? 0);
|
|
13166
|
+
if (aPixels !== bPixels) return bPixels - aPixels;
|
|
13167
|
+
return (b.bitrate ?? 0) - (a.bitrate ?? 0);
|
|
13168
|
+
})[0] ?? null;
|
|
13169
|
+
const selectedAudioTrack = tracks.filter((track) => track.streamType === "audio").sort((a, b) => (b.bitrate ?? 0) - (a.bitrate ?? 0))[0] ?? null;
|
|
13170
|
+
return { selectedVideoTrack, selectedAudioTrack };
|
|
13171
|
+
}
|
|
13172
|
+
function parseCount(text) {
|
|
13173
|
+
if (!text) return null;
|
|
13174
|
+
const match = text.match(/([\d,.]+)\s*posts?/i);
|
|
13175
|
+
if (!match) return null;
|
|
13176
|
+
const value = Number(match[1].replace(/[,.]/g, ""));
|
|
13177
|
+
return Number.isFinite(value) ? value : null;
|
|
13178
|
+
}
|
|
13179
|
+
function captionFromMetadata(metadata) {
|
|
13180
|
+
const raw = metadata.ogDescription ?? metadata.description;
|
|
13181
|
+
if (!raw) return null;
|
|
13182
|
+
const quoted = raw.match(/:\s*"([\s\S]+)"\.?\s*$/);
|
|
13183
|
+
if (quoted?.[1]) return quoted[1].trim();
|
|
13184
|
+
return raw.trim();
|
|
13185
|
+
}
|
|
13186
|
+
function ownerFromMetadata(metadata) {
|
|
13187
|
+
const title = metadata.ogTitle ?? metadata.twitterTitle ?? metadata.title;
|
|
13188
|
+
if (!title) return null;
|
|
13189
|
+
const match = title.match(/^(.+?)\s+on Instagram:/i) ?? title.match(/^(.+?)\s+\(@[^)]+\)\s+•\s+Instagram/i);
|
|
13190
|
+
return match?.[1]?.trim() || null;
|
|
13191
|
+
}
|
|
13192
|
+
async function collectInstagramProfileContentFromPage(page, opts) {
|
|
13193
|
+
const maxItems = Math.min(500, Math.max(1, opts.maxItems ?? 50));
|
|
13194
|
+
const maxScrolls = Math.min(100, Math.max(0, opts.maxScrolls ?? 10));
|
|
13195
|
+
const scrollDelayMs = Math.min(5e3, Math.max(250, opts.scrollDelayMs ?? 1200));
|
|
13196
|
+
const stableScrollLimit = Math.min(10, Math.max(1, opts.stableScrollLimit ?? 4));
|
|
13197
|
+
const seen = /* @__PURE__ */ new Set();
|
|
13198
|
+
const items = [];
|
|
13199
|
+
const stages = [];
|
|
13200
|
+
let attemptedScrolls = 0;
|
|
13201
|
+
let stableScrolls = 0;
|
|
13202
|
+
let stoppedReason = null;
|
|
13203
|
+
const collect = async (stage) => {
|
|
13204
|
+
const before = items.length;
|
|
13205
|
+
const links = await page.evaluate(() => Array.from(document.querySelectorAll("a[href]")).map((a) => ({
|
|
13206
|
+
href: a.href,
|
|
13207
|
+
text: (a.textContent ?? "").replace(/\s+/g, " ").trim()
|
|
13208
|
+
})));
|
|
13209
|
+
for (const link of links) {
|
|
13210
|
+
let parsed;
|
|
13211
|
+
try {
|
|
13212
|
+
parsed = new URL(link.href);
|
|
13213
|
+
} catch {
|
|
13214
|
+
continue;
|
|
13215
|
+
}
|
|
13216
|
+
const found = instagramContentTypeFromPath(parsed.pathname);
|
|
13217
|
+
if (!found) continue;
|
|
13218
|
+
const normalized = `https://www.instagram.com/${found.type === "post" ? "p" : found.type}/${found.shortcode}/`;
|
|
13219
|
+
if (seen.has(normalized)) continue;
|
|
13220
|
+
seen.add(normalized);
|
|
13221
|
+
items.push({
|
|
13222
|
+
url: normalized,
|
|
13223
|
+
type: found.type,
|
|
13224
|
+
shortcode: found.shortcode,
|
|
13225
|
+
anchorText: link.text || null,
|
|
13226
|
+
firstSeenStage: stage
|
|
13227
|
+
});
|
|
13228
|
+
if (items.length >= maxItems) return items.length - before;
|
|
13229
|
+
}
|
|
13230
|
+
return items.length - before;
|
|
13231
|
+
};
|
|
13232
|
+
const readBodyText = async () => page.evaluate(() => document.body?.innerText ?? "").catch(() => "");
|
|
13233
|
+
const profileLines = (text) => text.split("\n").map((line) => line.trim()).filter(Boolean);
|
|
13234
|
+
const profilePostCountText = (text) => profileLines(text).find((line) => /\bposts?\b/i.test(line) && /[\d,.]/.test(line)) ?? null;
|
|
13235
|
+
const scrollMetrics = async () => page.evaluate(() => ({
|
|
13236
|
+
scrollY: typeof window.scrollY === "number" ? window.scrollY : null,
|
|
13237
|
+
scrollHeight: Math.max(
|
|
13238
|
+
document.body?.scrollHeight ?? 0,
|
|
13239
|
+
document.documentElement?.scrollHeight ?? 0
|
|
13240
|
+
) || null
|
|
13241
|
+
})).catch(() => ({ scrollY: null, scrollHeight: null }));
|
|
13242
|
+
const recordStage = async (stage, addedCount) => {
|
|
13243
|
+
const metrics = await scrollMetrics();
|
|
13244
|
+
stages.push({
|
|
13245
|
+
stage,
|
|
13246
|
+
itemCount: items.length,
|
|
13247
|
+
addedCount,
|
|
13248
|
+
scrollY: metrics.scrollY,
|
|
13249
|
+
scrollHeight: metrics.scrollHeight
|
|
13250
|
+
});
|
|
13251
|
+
};
|
|
13252
|
+
const advancePagination = async () => page.evaluate(() => {
|
|
13253
|
+
const contentLinks = Array.from(document.querySelectorAll('a[href*="/p/"], a[href*="/reel/"], a[href*="/tv/"]'));
|
|
13254
|
+
const last = contentLinks[contentLinks.length - 1];
|
|
13255
|
+
if (last) last.scrollIntoView({ block: "end", inline: "nearest" });
|
|
13256
|
+
const distance = Math.max(window.innerHeight * 0.9, 700);
|
|
13257
|
+
window.scrollBy(0, distance);
|
|
13258
|
+
return {
|
|
13259
|
+
scrollY: typeof window.scrollY === "number" ? window.scrollY : null,
|
|
13260
|
+
scrollHeight: Math.max(
|
|
13261
|
+
document.body?.scrollHeight ?? 0,
|
|
13262
|
+
document.documentElement?.scrollHeight ?? 0
|
|
13263
|
+
) || null
|
|
13264
|
+
};
|
|
13265
|
+
}).catch(() => ({ scrollY: null, scrollHeight: null }));
|
|
13266
|
+
await page.waitForLoadState("domcontentloaded", { timeout: 15e3 }).catch(() => {
|
|
13267
|
+
});
|
|
13268
|
+
await page.waitForTimeout(1500);
|
|
13269
|
+
await recordStage("initial", await collect("initial"));
|
|
13270
|
+
const initialReportedPostCount = parseCount(profilePostCountText(await readBodyText()));
|
|
13271
|
+
for (let i = 0; i < maxScrolls && items.length < maxItems && stableScrolls < stableScrollLimit; i++) {
|
|
13272
|
+
const before = items.length;
|
|
13273
|
+
const beforeMetrics = await scrollMetrics();
|
|
13274
|
+
attemptedScrolls = i + 1;
|
|
13275
|
+
await advancePagination();
|
|
13276
|
+
await page.keyboard.press("PageDown").catch(() => {
|
|
13277
|
+
});
|
|
13278
|
+
await page.waitForLoadState("networkidle", { timeout: 2500 }).catch(() => {
|
|
13279
|
+
});
|
|
13280
|
+
await page.waitForTimeout(scrollDelayMs);
|
|
13281
|
+
const added = await collect(`scroll_${i + 1}`);
|
|
13282
|
+
await recordStage(`scroll_${i + 1}`, added);
|
|
13283
|
+
const afterMetrics = await scrollMetrics();
|
|
13284
|
+
const advanced = (afterMetrics.scrollHeight ?? 0) > (beforeMetrics.scrollHeight ?? 0) || (afterMetrics.scrollY ?? 0) > (beforeMetrics.scrollY ?? 0);
|
|
13285
|
+
stableScrolls = items.length === before && !advanced ? stableScrolls + 1 : 0;
|
|
13286
|
+
if (initialReportedPostCount != null && items.length >= initialReportedPostCount) {
|
|
13287
|
+
stoppedReason = "reported_post_count";
|
|
13288
|
+
break;
|
|
13289
|
+
}
|
|
13290
|
+
if (items.length >= maxItems) {
|
|
13291
|
+
stoppedReason = "max_items";
|
|
13292
|
+
break;
|
|
13293
|
+
}
|
|
13294
|
+
if (stableScrolls >= stableScrollLimit) {
|
|
13295
|
+
stoppedReason = "stable_scrolls";
|
|
13296
|
+
break;
|
|
13297
|
+
}
|
|
13298
|
+
}
|
|
13299
|
+
const bodyText = await readBodyText();
|
|
13300
|
+
const lines = profileLines(bodyText);
|
|
13301
|
+
const reportedPostCountText = lines.find((line) => /\bposts?\b/i.test(line) && /[\d,.]/.test(line)) ?? null;
|
|
13302
|
+
const followerCountText = lines.find((line) => /\bfollowers?\b/i.test(line)) ?? null;
|
|
13303
|
+
const followingCountText = lines.find((line) => /\bfollowing\b/i.test(line)) ?? null;
|
|
13304
|
+
const profileName = lines.find((line) => line && line.toLowerCase() !== opts.handle.toLowerCase()) ?? null;
|
|
13305
|
+
const typeCounts = {
|
|
13306
|
+
post: items.filter((item) => item.type === "post").length,
|
|
13307
|
+
reel: items.filter((item) => item.type === "reel").length,
|
|
13308
|
+
tv: items.filter((item) => item.type === "tv").length
|
|
13309
|
+
};
|
|
13310
|
+
const reportedPostCount = parseCount(reportedPostCountText);
|
|
13311
|
+
const reachedMaxItems = items.length >= maxItems;
|
|
13312
|
+
const reachedReportedPostCount = reportedPostCount != null && items.length >= reportedPostCount;
|
|
13313
|
+
if (!stoppedReason) {
|
|
13314
|
+
stoppedReason = reachedMaxItems ? "max_items" : reachedReportedPostCount ? "reported_post_count" : maxScrolls === 0 ? "no_scrolls" : attemptedScrolls >= maxScrolls ? "max_scrolls" : "stable_scrolls";
|
|
13315
|
+
}
|
|
13316
|
+
const finalMetrics = await scrollMetrics();
|
|
13317
|
+
const pagination = {
|
|
13318
|
+
maxItems,
|
|
13319
|
+
maxScrolls,
|
|
13320
|
+
attemptedScrolls,
|
|
13321
|
+
stableScrolls,
|
|
13322
|
+
stableScrollLimit,
|
|
13323
|
+
scrollDelayMs,
|
|
13324
|
+
reachedMaxItems,
|
|
13325
|
+
reachedReportedPostCount,
|
|
13326
|
+
finalScrollHeight: finalMetrics.scrollHeight,
|
|
13327
|
+
stoppedReason,
|
|
13328
|
+
stages
|
|
13329
|
+
};
|
|
13330
|
+
const limitations = [];
|
|
13331
|
+
if (/log in|sign up/i.test(bodyText)) limitations.push("Login or sign-up prompts were visible during extraction.");
|
|
13332
|
+
if (reachedMaxItems && !reachedReportedPostCount) limitations.push("Collection stopped at maxItems before the full profile history was proven.");
|
|
13333
|
+
if (reportedPostCount != null && items.length < reportedPostCount && !reachedMaxItems) limitations.push("Collected item count is lower than the profile-reported post count.");
|
|
13334
|
+
if (stoppedReason === "stable_scrolls" && !reachedReportedPostCount) limitations.push("Pagination stopped after repeated stable scrolls without new content.");
|
|
13335
|
+
if (stoppedReason === "max_scrolls" && !reachedReportedPostCount) limitations.push("Pagination reached maxScrolls before the full profile history was proven.");
|
|
13336
|
+
return {
|
|
13337
|
+
handle: opts.handle,
|
|
13338
|
+
profileUrl: opts.profileUrl,
|
|
13339
|
+
pageUrl: page.url(),
|
|
13340
|
+
profileName,
|
|
13341
|
+
bio: null,
|
|
13342
|
+
reportedPostCount,
|
|
13343
|
+
reportedPostCountText,
|
|
13344
|
+
followerCountText,
|
|
13345
|
+
followingCountText,
|
|
13346
|
+
collectedContentCount: items.length,
|
|
13347
|
+
typeCounts,
|
|
13348
|
+
pagination,
|
|
13349
|
+
items,
|
|
13350
|
+
limited: limitations.length > 0,
|
|
13351
|
+
limitations,
|
|
13352
|
+
extractedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
13353
|
+
};
|
|
13354
|
+
}
|
|
13355
|
+
async function extractInstagramMediaFromPage(page, sourceUrl, capturedMediaUrls = []) {
|
|
13356
|
+
await page.waitForLoadState("domcontentloaded", { timeout: 15e3 }).catch(() => {
|
|
13357
|
+
});
|
|
13358
|
+
await page.waitForTimeout(2500);
|
|
13359
|
+
const html = await page.content();
|
|
13360
|
+
const bodyText = await page.evaluate(() => document.body?.innerText ?? "").catch(() => "");
|
|
13361
|
+
const metadata = {
|
|
13362
|
+
title: await page.title().catch(() => null),
|
|
13363
|
+
description: metadataContent2(html, "description"),
|
|
13364
|
+
ogTitle: metadataContent2(html, "og:title"),
|
|
13365
|
+
ogDescription: metadataContent2(html, "og:description"),
|
|
13366
|
+
ogImage: metadataContent2(html, "og:image"),
|
|
13367
|
+
ogUrl: metadataContent2(html, "og:url"),
|
|
13368
|
+
twitterImage: metadataContent2(html, "twitter:image"),
|
|
13369
|
+
twitterTitle: metadataContent2(html, "twitter:title")
|
|
13370
|
+
};
|
|
13371
|
+
const parsedPath = (() => {
|
|
13372
|
+
try {
|
|
13373
|
+
return instagramContentTypeFromPath(new URL(page.url()).pathname) ?? instagramContentTypeFromPath(new URL(sourceUrl).pathname);
|
|
13374
|
+
} catch {
|
|
13375
|
+
return null;
|
|
13376
|
+
}
|
|
13377
|
+
})();
|
|
13378
|
+
const htmlUrls = extractInstagramMp4UrlsFromText(html);
|
|
13379
|
+
const tracks = instagramTracksFromUrls([...capturedMediaUrls, ...htmlUrls]);
|
|
13380
|
+
const selected = selectInstagramTracks(tracks);
|
|
13381
|
+
const imageUrl = metadata.ogImage ?? metadata.twitterImage;
|
|
13382
|
+
const limitations = [];
|
|
13383
|
+
if (/log in|sign up/i.test(bodyText)) limitations.push("Login or sign-up prompts were visible during extraction.");
|
|
13384
|
+
if (parsedPath?.type === "reel" && tracks.length === 0) limitations.push("No Instagram MP4 tracks were captured; retry with a logged-in browser profile or longer wait.");
|
|
13385
|
+
return {
|
|
13386
|
+
sourceUrl,
|
|
13387
|
+
pageUrl: page.url(),
|
|
13388
|
+
type: parsedPath?.type ?? null,
|
|
13389
|
+
shortcode: parsedPath?.shortcode ?? null,
|
|
13390
|
+
ownerName: ownerFromMetadata(metadata),
|
|
13391
|
+
caption: captionFromMetadata(metadata),
|
|
13392
|
+
bodyText,
|
|
13393
|
+
metadata,
|
|
13394
|
+
imageUrl,
|
|
13395
|
+
tracks,
|
|
13396
|
+
selectedVideoTrack: selected.selectedVideoTrack,
|
|
13397
|
+
selectedAudioTrack: selected.selectedAudioTrack,
|
|
13398
|
+
limitations,
|
|
13399
|
+
extractedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
13400
|
+
};
|
|
13401
|
+
}
|
|
13402
|
+
var CONTENT_PATH_RE, CDN_HOST_RE;
|
|
13403
|
+
var init_InstagramContentExtractor = __esm({
|
|
13404
|
+
"src/extractor/InstagramContentExtractor.ts"() {
|
|
13405
|
+
"use strict";
|
|
13406
|
+
CONTENT_PATH_RE = /^\/(?:[^/]+\/)?(p|reel|tv)\/([A-Za-z0-9_-]+)\/?/i;
|
|
13407
|
+
CDN_HOST_RE = /(^|\.)((cdninstagram\.com)|(scontent[^/]*\.cdninstagram\.com))$/i;
|
|
13408
|
+
}
|
|
13409
|
+
});
|
|
13410
|
+
|
|
13411
|
+
// src/api/instagram-routes.ts
|
|
13412
|
+
function invalidRequest2(message) {
|
|
13413
|
+
return { error_code: "invalid_request", message };
|
|
13414
|
+
}
|
|
13415
|
+
function directProfileDir() {
|
|
13416
|
+
return process.env.MCP_SCRAPER_BROWSER_PROFILE_DIR?.trim() || void 0;
|
|
13417
|
+
}
|
|
13418
|
+
function configuredProfileName(profile) {
|
|
13419
|
+
const directDir = directProfileDir();
|
|
13420
|
+
return profile?.trim() || process.env.MCP_SCRAPER_BROWSER_PROFILE?.trim() || (directDir ? (0, import_node_path7.basename)(directDir) : "default");
|
|
13421
|
+
}
|
|
13422
|
+
function launchExecutablePath(manifestExecutablePath) {
|
|
13423
|
+
const configured = process.env.MCP_SCRAPER_BROWSER_EXECUTABLE?.trim() || manifestExecutablePath || defaultChromeExecutablePath();
|
|
13424
|
+
return configured && (0, import_node_fs4.existsSync)(configured) ? configured : void 0;
|
|
13425
|
+
}
|
|
13426
|
+
function ignoreDefaultArgsFor(executablePath) {
|
|
13427
|
+
if (!executablePath) return void 0;
|
|
13428
|
+
if (process.platform !== "darwin") return void 0;
|
|
13429
|
+
if (!/Google Chrome\.app\/Contents\/MacOS\/Google Chrome$/.test(executablePath)) return void 0;
|
|
13430
|
+
return ["--password-store=basic", "--use-mock-keychain"];
|
|
13431
|
+
}
|
|
13432
|
+
async function kernelLaunchOptsResidential2() {
|
|
13433
|
+
let proxyId = browserServiceProxyId();
|
|
13434
|
+
try {
|
|
13435
|
+
const resolution2 = await resolveKernelProxyId({
|
|
13436
|
+
kernelApiKey: browserServiceApiKey(),
|
|
13437
|
+
proxyMode: DEFAULT_PROXY_MODE,
|
|
13438
|
+
configuredKernelProxyId: browserServiceProxyId(),
|
|
13439
|
+
gl: "us"
|
|
13440
|
+
});
|
|
13441
|
+
if (resolution2.kernelProxyId) proxyId = resolution2.kernelProxyId;
|
|
13442
|
+
} catch {
|
|
13443
|
+
proxyId = browserServiceProxyId();
|
|
13444
|
+
}
|
|
13445
|
+
return {
|
|
13446
|
+
headless: true,
|
|
13447
|
+
kernelApiKey: browserServiceApiKey(),
|
|
13448
|
+
kernelProxyId: proxyId,
|
|
13449
|
+
viewport: { width: 1280, height: 900 },
|
|
13450
|
+
locale: "en-US"
|
|
13451
|
+
};
|
|
13452
|
+
}
|
|
13453
|
+
async function resolveInstagramLaunch(input) {
|
|
13454
|
+
const requestedMode = input.browserMode ?? "auto";
|
|
13455
|
+
const directDir = directProfileDir();
|
|
13456
|
+
const useLocal = requestedMode === "local" || requestedMode === "auto" && (Boolean(input.browserProfile) || Boolean(directDir) || localBrowserModeEnabled());
|
|
13457
|
+
if (!useLocal) {
|
|
13458
|
+
return {
|
|
13459
|
+
config: await kernelLaunchOptsResidential2(),
|
|
13460
|
+
browser: {
|
|
13461
|
+
mode: "hosted",
|
|
13462
|
+
requestedMode,
|
|
13463
|
+
profileName: null,
|
|
13464
|
+
profileSource: "hosted",
|
|
13465
|
+
profileDirConfigured: false,
|
|
13466
|
+
executablePathConfigured: false
|
|
13467
|
+
}
|
|
13468
|
+
};
|
|
13469
|
+
}
|
|
13470
|
+
const requestedName = configuredProfileName(input.browserProfile);
|
|
13471
|
+
if (directDir && (!input.browserProfile || requestedName === process.env.MCP_SCRAPER_BROWSER_PROFILE?.trim())) {
|
|
13472
|
+
const executablePath2 = launchExecutablePath(null);
|
|
13473
|
+
return {
|
|
13474
|
+
config: {
|
|
13475
|
+
headless: false,
|
|
13476
|
+
profileDir: directDir,
|
|
13477
|
+
browserExecutablePath: executablePath2,
|
|
13478
|
+
ignoreDefaultArgs: ignoreDefaultArgsFor(executablePath2),
|
|
13479
|
+
viewport: { width: 1280, height: 900 },
|
|
13480
|
+
locale: "en-US"
|
|
13481
|
+
},
|
|
13482
|
+
browser: {
|
|
13483
|
+
mode: "local",
|
|
13484
|
+
requestedMode,
|
|
13485
|
+
profileName: requestedName,
|
|
13486
|
+
profileSource: "direct_profile_dir",
|
|
13487
|
+
profileDirConfigured: true,
|
|
13488
|
+
executablePathConfigured: Boolean(executablePath2)
|
|
13489
|
+
}
|
|
13490
|
+
};
|
|
13491
|
+
}
|
|
13492
|
+
const manifest = await loadImportedBrowserProfile(requestedName);
|
|
13493
|
+
const executablePath = launchExecutablePath(manifest.browserExecutablePath);
|
|
13494
|
+
return {
|
|
13495
|
+
config: {
|
|
13496
|
+
headless: false,
|
|
13497
|
+
profileDir: manifest.userDataDir,
|
|
13498
|
+
browserExecutablePath: executablePath,
|
|
13499
|
+
ignoreDefaultArgs: ignoreDefaultArgsFor(executablePath),
|
|
13500
|
+
viewport: { width: 1280, height: 900 },
|
|
13501
|
+
locale: "en-US"
|
|
13502
|
+
},
|
|
13503
|
+
browser: {
|
|
13504
|
+
mode: "local",
|
|
13505
|
+
requestedMode,
|
|
13506
|
+
profileName: manifest.name,
|
|
13507
|
+
profileSource: "managed_profile",
|
|
13508
|
+
profileDirConfigured: true,
|
|
13509
|
+
executablePathConfigured: Boolean(executablePath)
|
|
13510
|
+
}
|
|
13511
|
+
};
|
|
13512
|
+
}
|
|
13513
|
+
function outputBaseDir() {
|
|
13514
|
+
return process.env.MCP_SCRAPER_OUTPUT_DIR?.trim() || (0, import_node_path7.join)((0, import_node_os5.homedir)(), "Downloads", "mcp-scraper");
|
|
13515
|
+
}
|
|
13516
|
+
function safeFilePart(input) {
|
|
13517
|
+
return input.replace(/^https?:\/\//, "").replace(/[^a-z0-9._-]+/gi, "-").replace(/^-+|-+$/g, "").slice(0, 80) || "instagram";
|
|
13518
|
+
}
|
|
13519
|
+
function mediaOutputDir(shortcode, sourceUrl) {
|
|
13520
|
+
const stamp = (/* @__PURE__ */ new Date()).toISOString().replace(/[:.]/g, "-").slice(0, 19);
|
|
13521
|
+
const slug = shortcode ? `instagram-${shortcode}` : safeFilePart(sourceUrl);
|
|
13522
|
+
const dir = (0, import_node_path7.join)(outputBaseDir(), "instagram", `${stamp}-${slug}`);
|
|
13523
|
+
(0, import_node_fs4.mkdirSync)(dir, { recursive: true });
|
|
13524
|
+
return dir;
|
|
13525
|
+
}
|
|
13526
|
+
async function downloadToFile(url, destPath, referer) {
|
|
13527
|
+
const res = await fetch(url, {
|
|
13528
|
+
headers: {
|
|
13529
|
+
"User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36",
|
|
13530
|
+
"Referer": referer
|
|
13531
|
+
},
|
|
13532
|
+
signal: AbortSignal.timeout(9e4)
|
|
13533
|
+
});
|
|
13534
|
+
if (!res.ok) throw new Error(`HTTP ${res.status}`);
|
|
13535
|
+
if (!res.body) throw new Error("Empty response body");
|
|
13536
|
+
await (0, import_promises6.pipeline)(import_node_stream2.Readable.fromWeb(res.body), (0, import_node_fs4.createWriteStream)(destPath));
|
|
13537
|
+
return {
|
|
13538
|
+
savedPath: destPath,
|
|
13539
|
+
sizeBytes: (0, import_node_fs4.statSync)(destPath).size,
|
|
13540
|
+
mimeType: res.headers.get("content-type")?.split(";")[0]?.trim() ?? null
|
|
13541
|
+
};
|
|
13542
|
+
}
|
|
13543
|
+
function extFromMime(mimeType, fallback) {
|
|
13544
|
+
const map = {
|
|
13545
|
+
"image/jpeg": "jpg",
|
|
13546
|
+
"image/png": "png",
|
|
13547
|
+
"image/webp": "webp",
|
|
13548
|
+
"image/gif": "gif",
|
|
13549
|
+
"video/mp4": "mp4",
|
|
13550
|
+
"audio/mp4": "mp4",
|
|
13551
|
+
"audio/aac": "aac"
|
|
13552
|
+
};
|
|
13553
|
+
return mimeType ? map[mimeType] ?? fallback : fallback;
|
|
13554
|
+
}
|
|
13555
|
+
function runFfmpegMux(videoPath, audioPath, outPath) {
|
|
13556
|
+
return new Promise((resolve) => {
|
|
13557
|
+
const child = (0, import_node_child_process2.spawn)("ffmpeg", ["-y", "-i", videoPath, "-i", audioPath, "-c", "copy", "-shortest", outPath], {
|
|
13558
|
+
stdio: ["ignore", "ignore", "pipe"]
|
|
13559
|
+
});
|
|
13560
|
+
let stderr = "";
|
|
13561
|
+
child.stderr.on("data", (chunk) => {
|
|
13562
|
+
stderr += String(chunk);
|
|
13563
|
+
});
|
|
13564
|
+
child.on("error", (err) => resolve({ ok: false, error: err.message }));
|
|
13565
|
+
child.on("close", (code) => resolve({ ok: code === 0, error: code === 0 ? null : stderr.trim() || `ffmpeg exited ${code}` }));
|
|
13566
|
+
});
|
|
13567
|
+
}
|
|
13568
|
+
function trackFilename(track, index, shortcode) {
|
|
13569
|
+
const label = shortcode ?? "instagram";
|
|
13570
|
+
const type = track.streamType === "audio" ? "audio" : track.streamType === "video" ? "video" : "track";
|
|
13571
|
+
const bitrate = track.bitrate ? `-${track.bitrate}` : "";
|
|
13572
|
+
return `${label}-${type}-${index}${bitrate}.mp4`;
|
|
13573
|
+
}
|
|
13574
|
+
var import_hono5, import_zod17, import_node_fs4, import_node_os5, import_node_path7, import_node_stream2, import_promises6, import_node_child_process2, InstagramBrowserModeSchema, InstagramBrowserOptionsSchema, InstagramProfileContentBodySchema, InstagramMediaTypeSchema, InstagramMediaDownloadBodySchema, instagramApp;
|
|
13575
|
+
var init_instagram_routes = __esm({
|
|
13576
|
+
"src/api/instagram-routes.ts"() {
|
|
13577
|
+
"use strict";
|
|
13578
|
+
import_hono5 = require("hono");
|
|
13579
|
+
import_zod17 = require("zod");
|
|
13580
|
+
import_node_fs4 = require("fs");
|
|
13581
|
+
import_node_os5 = require("os");
|
|
13582
|
+
import_node_path7 = require("path");
|
|
13583
|
+
import_node_stream2 = require("stream");
|
|
13584
|
+
import_promises6 = require("stream/promises");
|
|
13585
|
+
import_node_child_process2 = require("child_process");
|
|
13586
|
+
init_browser_service_env();
|
|
13587
|
+
init_BrowserDriver();
|
|
13588
|
+
init_kernel_proxy_resolver();
|
|
13589
|
+
init_schemas3();
|
|
13590
|
+
init_api_auth();
|
|
13591
|
+
init_url_utils();
|
|
13592
|
+
init_db();
|
|
13593
|
+
init_rates();
|
|
13594
|
+
init_concurrency_gates();
|
|
13595
|
+
init_local_browser_profiles();
|
|
13596
|
+
init_InstagramContentExtractor();
|
|
13597
|
+
init_media_transcription();
|
|
13598
|
+
InstagramBrowserModeSchema = import_zod17.z.enum(["auto", "hosted", "local"]);
|
|
13599
|
+
InstagramBrowserOptionsSchema = {
|
|
13600
|
+
browserMode: InstagramBrowserModeSchema.default("auto"),
|
|
13601
|
+
browserProfile: import_zod17.z.string().trim().min(1).optional()
|
|
13602
|
+
};
|
|
13603
|
+
InstagramProfileContentBodySchema = import_zod17.z.object({
|
|
13604
|
+
handle: import_zod17.z.string().trim().optional(),
|
|
13605
|
+
url: import_zod17.z.string().trim().optional(),
|
|
13606
|
+
maxItems: import_zod17.z.number().int().min(1).max(500).default(50),
|
|
13607
|
+
maxScrolls: import_zod17.z.number().int().min(0).max(100).default(10),
|
|
13608
|
+
scrollDelayMs: import_zod17.z.number().int().min(250).max(5e3).default(1200),
|
|
13609
|
+
stableScrollLimit: import_zod17.z.number().int().min(1).max(10).default(4),
|
|
13610
|
+
...InstagramBrowserOptionsSchema
|
|
13611
|
+
}).refine((d) => !!d.handle || !!d.url, {
|
|
13612
|
+
message: "handle or url is required"
|
|
13613
|
+
});
|
|
13614
|
+
InstagramMediaTypeSchema = import_zod17.z.enum(["image", "video", "audio"]);
|
|
13615
|
+
InstagramMediaDownloadBodySchema = import_zod17.z.object({
|
|
13616
|
+
url: import_zod17.z.string().trim().min(1, "url is required"),
|
|
13617
|
+
mediaTypes: import_zod17.z.array(InstagramMediaTypeSchema).default(["image", "video", "audio"]),
|
|
13618
|
+
downloadMedia: import_zod17.z.boolean().default(true),
|
|
13619
|
+
downloadAllTracks: import_zod17.z.boolean().default(false),
|
|
13620
|
+
includeTranscript: import_zod17.z.boolean().default(false),
|
|
13621
|
+
mux: import_zod17.z.boolean().default(true),
|
|
13622
|
+
...InstagramBrowserOptionsSchema
|
|
13623
|
+
});
|
|
13624
|
+
instagramApp = new import_hono5.Hono();
|
|
13625
|
+
instagramApp.post("/profile-content", createApiKeyAuth(), async (c) => {
|
|
13626
|
+
const raw = await c.req.json().catch(() => ({}));
|
|
13627
|
+
const parsed = InstagramProfileContentBodySchema.safeParse(raw);
|
|
13628
|
+
if (!parsed.success) return c.json(invalidRequest2(parsed.error.issues[0]?.message ?? "Invalid request"), 400);
|
|
13629
|
+
const body = parsed.data;
|
|
13630
|
+
let target;
|
|
13631
|
+
try {
|
|
13632
|
+
target = normalizeInstagramProfileInput(body.url ?? body.handle ?? "");
|
|
13633
|
+
} catch (err) {
|
|
13634
|
+
return c.json(invalidRequest2(err instanceof Error ? err.message : String(err)), 400);
|
|
13635
|
+
}
|
|
13636
|
+
const user = c.get("user");
|
|
13637
|
+
const gate = await acquireConcurrencyGate(user, "instagram_profile_content", {
|
|
13638
|
+
reuseLockId: c.req.header("x-mcp-scraper-concurrency-lock"),
|
|
13639
|
+
metadata: { handle: target.handle }
|
|
13640
|
+
});
|
|
13641
|
+
if (!gate.ok) return c.json(concurrencyLimitExceededResponse(gate), 429, { "Retry-After": String(gate.retryAfterSeconds) });
|
|
13642
|
+
const driver = new BrowserDriver();
|
|
13643
|
+
let debited = false;
|
|
13644
|
+
try {
|
|
13645
|
+
const { ok, balance_mc } = await debitMc(user.id, MC_COSTS.instagram_profile, LedgerOperation.INSTAGRAM_PROFILE, target.profileUrl);
|
|
13646
|
+
if (!ok) return c.json(insufficientBalanceResponse(balance_mc, MC_COSTS.instagram_profile), 402);
|
|
13647
|
+
debited = true;
|
|
13648
|
+
const launch = await resolveInstagramLaunch(body);
|
|
13649
|
+
await driver.launch(launch.config);
|
|
13650
|
+
await driver.navigateTo(target.profileUrl);
|
|
13651
|
+
const page = driver.getPage();
|
|
13652
|
+
const result = await collectInstagramProfileContentFromPage(page, {
|
|
13653
|
+
...target,
|
|
13654
|
+
maxItems: body.maxItems,
|
|
13655
|
+
maxScrolls: body.maxScrolls,
|
|
13656
|
+
scrollDelayMs: body.scrollDelayMs,
|
|
13657
|
+
stableScrollLimit: body.stableScrollLimit
|
|
13658
|
+
});
|
|
13659
|
+
const response = { ...result, browser: launch.browser };
|
|
13660
|
+
await logRequestEvent({ userId: user.id, source: "instagram_profile_content", status: "done", query: target.profileUrl, resultCount: result.items.length, result: response });
|
|
13661
|
+
return c.json(response);
|
|
13662
|
+
} catch (err) {
|
|
13663
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
13664
|
+
if (debited) await creditMc(user.id, MC_COSTS.instagram_profile, LedgerOperation.INSTAGRAM_PROFILE_REFUND, "failed call");
|
|
13665
|
+
await logRequestEvent({ userId: user.id, source: "instagram_profile_content", status: "failed", query: target.profileUrl, error: msg });
|
|
13666
|
+
if (msg.toLowerCase().includes("blocked") || msg.toLowerCase().includes("captcha")) return c.json({ error: msg }, 503);
|
|
13667
|
+
return c.json({ error: msg }, 500);
|
|
13668
|
+
} finally {
|
|
13669
|
+
await driver.close();
|
|
13670
|
+
await releaseConcurrencyGate(gate.lockId);
|
|
13671
|
+
}
|
|
13672
|
+
});
|
|
13673
|
+
instagramApp.post("/media-download", createApiKeyAuth(), async (c) => {
|
|
13674
|
+
const raw = await c.req.json().catch(() => ({}));
|
|
13675
|
+
const parsed = InstagramMediaDownloadBodySchema.safeParse(raw);
|
|
13676
|
+
if (!parsed.success) return c.json(invalidRequest2(parsed.error.issues[0]?.message ?? "Invalid request"), 400);
|
|
13677
|
+
const body = parsed.data;
|
|
13678
|
+
const urlCheck = await validatePublicHttpUrl(body.url, { field: "url", requireHttps: false });
|
|
13679
|
+
if (urlCheck.error) return c.json(invalidRequest2(urlCheck.error), 400);
|
|
13680
|
+
const sourceUrl = urlCheck.parsed;
|
|
13681
|
+
if (!isAllowedInstagramPageUrl(sourceUrl)) return c.json(invalidRequest2("url must be an Instagram post, reel, or tv URL"), 400);
|
|
13682
|
+
const user = c.get("user");
|
|
13683
|
+
const gate = await acquireConcurrencyGate(user, "instagram_media_download", {
|
|
13684
|
+
reuseLockId: c.req.header("x-mcp-scraper-concurrency-lock"),
|
|
13685
|
+
metadata: { url: sourceUrl.href }
|
|
13686
|
+
});
|
|
13687
|
+
if (!gate.ok) return c.json(concurrencyLimitExceededResponse(gate), 429, { "Retry-After": String(gate.retryAfterSeconds) });
|
|
13688
|
+
const driver = new BrowserDriver();
|
|
13689
|
+
let mediaDebited = false;
|
|
13690
|
+
let transcriptDebited = false;
|
|
13691
|
+
try {
|
|
13692
|
+
const { ok, balance_mc } = await debitMc(user.id, MC_COSTS.instagram_media, LedgerOperation.INSTAGRAM_MEDIA, sourceUrl.href);
|
|
13693
|
+
if (!ok) return c.json(insufficientBalanceResponse(balance_mc, MC_COSTS.instagram_media), 402);
|
|
13694
|
+
mediaDebited = true;
|
|
13695
|
+
const launch = await resolveInstagramLaunch(body);
|
|
13696
|
+
await driver.launch(launch.config);
|
|
13697
|
+
const page = driver.getPage();
|
|
13698
|
+
const capturedMediaUrls = [];
|
|
13699
|
+
page.on("response", (response) => {
|
|
13700
|
+
const url = normalizeInstagramMediaUrl(response.url());
|
|
13701
|
+
if (!url) return;
|
|
13702
|
+
const headers = response.headers();
|
|
13703
|
+
const contentType = headers["content-type"] ?? "";
|
|
13704
|
+
if (contentType.includes("video/mp4") || url.includes(".mp4")) capturedMediaUrls.push(url);
|
|
13705
|
+
});
|
|
13706
|
+
await driver.navigateTo(sourceUrl.href);
|
|
13707
|
+
const extraction = await extractInstagramMediaFromPage(page, sourceUrl.href, capturedMediaUrls);
|
|
13708
|
+
const mediaTypes = new Set(body.mediaTypes);
|
|
13709
|
+
const downloads = [];
|
|
13710
|
+
const warnings = [];
|
|
13711
|
+
let outputDir = null;
|
|
13712
|
+
if (body.downloadMedia) {
|
|
13713
|
+
outputDir = mediaOutputDir(extraction.shortcode, sourceUrl.href);
|
|
13714
|
+
const textPath = (0, import_node_path7.join)(outputDir, `${extraction.shortcode ?? "instagram"}-text.txt`);
|
|
13715
|
+
(0, import_node_fs4.writeFileSync)(textPath, [
|
|
13716
|
+
`URL: ${extraction.pageUrl}`,
|
|
13717
|
+
extraction.caption ? `Caption: ${extraction.caption}` : "",
|
|
13718
|
+
"",
|
|
13719
|
+
extraction.bodyText
|
|
13720
|
+
].filter(Boolean).join("\n"), "utf8");
|
|
13721
|
+
downloads.push({ kind: "text", url: null, savedPath: textPath, sizeBytes: (0, import_node_fs4.statSync)(textPath).size, mimeType: "text/plain", error: null });
|
|
13722
|
+
if (mediaTypes.has("image") && extraction.imageUrl) {
|
|
13723
|
+
const imagePathBase = (0, import_node_path7.join)(outputDir, `${extraction.shortcode ?? "instagram"}-image`);
|
|
13724
|
+
try {
|
|
13725
|
+
const downloaded = await downloadToFile(extraction.imageUrl, imagePathBase, extraction.pageUrl);
|
|
13726
|
+
const ext = extFromMime(downloaded.mimeType, "jpg");
|
|
13727
|
+
const finalPath = downloaded.savedPath.endsWith(`.${ext}`) ? downloaded.savedPath : `${downloaded.savedPath}.${ext}`;
|
|
13728
|
+
if (finalPath !== downloaded.savedPath) {
|
|
13729
|
+
const { renameSync } = await import("fs");
|
|
13730
|
+
renameSync(downloaded.savedPath, finalPath);
|
|
13731
|
+
}
|
|
13732
|
+
downloads.push({ kind: "image", url: extraction.imageUrl, savedPath: finalPath, sizeBytes: (0, import_node_fs4.statSync)(finalPath).size, mimeType: downloaded.mimeType, error: null });
|
|
13733
|
+
} catch (err) {
|
|
13734
|
+
downloads.push({ kind: "image", url: extraction.imageUrl, savedPath: null, sizeBytes: null, mimeType: null, error: err instanceof Error ? err.message : String(err) });
|
|
13735
|
+
}
|
|
13736
|
+
}
|
|
13737
|
+
const tracksToDownload = body.downloadAllTracks ? extraction.tracks : [extraction.selectedVideoTrack, extraction.selectedAudioTrack].filter((track) => Boolean(track));
|
|
13738
|
+
const savedByUrl = /* @__PURE__ */ new Map();
|
|
13739
|
+
for (const [index, track] of tracksToDownload.entries()) {
|
|
13740
|
+
if (track.streamType === "video" && !mediaTypes.has("video")) continue;
|
|
13741
|
+
if (track.streamType === "audio" && !mediaTypes.has("audio")) continue;
|
|
13742
|
+
const kind = track.streamType === "audio" ? "audio" : "video";
|
|
13743
|
+
const target = (0, import_node_path7.join)(outputDir, trackFilename(track, index, extraction.shortcode));
|
|
13744
|
+
try {
|
|
13745
|
+
const downloaded = await downloadToFile(track.url, target, extraction.pageUrl);
|
|
13746
|
+
downloads.push({ kind, url: track.url, savedPath: downloaded.savedPath, sizeBytes: downloaded.sizeBytes, mimeType: downloaded.mimeType, error: null });
|
|
13747
|
+
savedByUrl.set(track.url, downloaded.savedPath);
|
|
13748
|
+
} catch (err) {
|
|
13749
|
+
downloads.push({ kind, url: track.url, savedPath: null, sizeBytes: null, mimeType: null, error: err instanceof Error ? err.message : String(err) });
|
|
13750
|
+
}
|
|
13751
|
+
}
|
|
13752
|
+
if (body.mux && mediaTypes.has("video") && mediaTypes.has("audio") && extraction.selectedVideoTrack && extraction.selectedAudioTrack && savedByUrl.has(extraction.selectedVideoTrack.url) && savedByUrl.has(extraction.selectedAudioTrack.url)) {
|
|
13753
|
+
const outPath = (0, import_node_path7.join)(outputDir, `${extraction.shortcode ?? "instagram"}-muxed.mp4`);
|
|
13754
|
+
const muxed = await runFfmpegMux(savedByUrl.get(extraction.selectedVideoTrack.url), savedByUrl.get(extraction.selectedAudioTrack.url), outPath);
|
|
13755
|
+
if (muxed.ok) {
|
|
13756
|
+
downloads.push({ kind: "muxed_video", url: null, savedPath: outPath, sizeBytes: (0, import_node_fs4.statSync)(outPath).size, mimeType: "video/mp4", error: null });
|
|
13757
|
+
} else {
|
|
13758
|
+
warnings.push(`Mux skipped/failed: ${muxed.error}`);
|
|
13759
|
+
}
|
|
13760
|
+
}
|
|
13761
|
+
}
|
|
13762
|
+
let transcript = null;
|
|
13763
|
+
if (body.includeTranscript) {
|
|
13764
|
+
const audioUrl = extraction.selectedAudioTrack?.url ?? extraction.selectedVideoTrack?.url;
|
|
13765
|
+
if (!audioUrl) {
|
|
13766
|
+
warnings.push("Transcript requested, but no audio/video track was available.");
|
|
13767
|
+
} else {
|
|
13768
|
+
const tx = await debitMc(user.id, MC_COSTS.instagram_transcribe, LedgerOperation.INSTAGRAM_TRANSCRIBE, sourceUrl.href);
|
|
13769
|
+
if (!tx.ok) return c.json(insufficientBalanceResponse(tx.balance_mc, MC_COSTS.instagram_transcribe), 402);
|
|
13770
|
+
transcriptDebited = true;
|
|
13771
|
+
transcript = await transcribeMediaUrl(audioUrl, "# Instagram Media Transcript");
|
|
13772
|
+
}
|
|
13773
|
+
}
|
|
13774
|
+
const result = {
|
|
13775
|
+
...extraction,
|
|
13776
|
+
browser: launch.browser,
|
|
13777
|
+
outputDir,
|
|
13778
|
+
downloads,
|
|
13779
|
+
warnings,
|
|
13780
|
+
transcript
|
|
13781
|
+
};
|
|
13782
|
+
await logRequestEvent({ userId: user.id, source: "instagram_media_download", status: "done", query: sourceUrl.href, resultCount: downloads.length, result });
|
|
13783
|
+
return c.json(result);
|
|
13784
|
+
} catch (err) {
|
|
13785
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
13786
|
+
if (transcriptDebited) await creditMc(user.id, MC_COSTS.instagram_transcribe, LedgerOperation.INSTAGRAM_TRANSCRIBE_REFUND, "failed call");
|
|
13787
|
+
if (mediaDebited) await creditMc(user.id, MC_COSTS.instagram_media, LedgerOperation.INSTAGRAM_MEDIA_REFUND, "failed call");
|
|
13788
|
+
await logRequestEvent({ userId: user.id, source: "instagram_media_download", status: "failed", query: sourceUrl.href, error: msg });
|
|
13789
|
+
if (msg.toLowerCase().includes("blocked") || msg.toLowerCase().includes("captcha")) return c.json({ error: msg }, 503);
|
|
13790
|
+
return c.json({ error: msg }, 500);
|
|
13791
|
+
} finally {
|
|
13792
|
+
await driver.close();
|
|
13793
|
+
await releaseConcurrencyGate(gate.lockId);
|
|
13794
|
+
}
|
|
13795
|
+
});
|
|
13796
|
+
}
|
|
13797
|
+
});
|
|
13798
|
+
|
|
12933
13799
|
// src/extractor/MapsNavigator.ts
|
|
12934
13800
|
var MapsNavigator;
|
|
12935
13801
|
var init_MapsNavigator = __esm({
|
|
@@ -13117,7 +13983,7 @@ var init_MapsReviewCollector = __esm({
|
|
|
13117
13983
|
});
|
|
13118
13984
|
|
|
13119
13985
|
// src/extractor/MapsExtractor.ts
|
|
13120
|
-
var
|
|
13986
|
+
var import_zod18, MapsExtractor;
|
|
13121
13987
|
var init_MapsExtractor = __esm({
|
|
13122
13988
|
"src/extractor/MapsExtractor.ts"() {
|
|
13123
13989
|
"use strict";
|
|
@@ -13125,7 +13991,7 @@ var init_MapsExtractor = __esm({
|
|
|
13125
13991
|
init_selectors();
|
|
13126
13992
|
init_MapsNavigator();
|
|
13127
13993
|
init_MapsReviewCollector();
|
|
13128
|
-
|
|
13994
|
+
import_zod18 = require("zod");
|
|
13129
13995
|
MapsExtractor = class {
|
|
13130
13996
|
constructor(driver) {
|
|
13131
13997
|
this.driver = driver;
|
|
@@ -13311,7 +14177,7 @@ var init_MapsExtractor = __esm({
|
|
|
13311
14177
|
});
|
|
13312
14178
|
return rows;
|
|
13313
14179
|
}, { hoursTable: MapsSelectors.hoursTable, hoursTableAlt: MapsSelectors.hoursTableAlt });
|
|
13314
|
-
const result =
|
|
14180
|
+
const result = import_zod18.z.array(RawMapsHoursRowSchema).safeParse(raw);
|
|
13315
14181
|
if (!result.success) {
|
|
13316
14182
|
console.warn("[MapsExtractor] hours parse failed", result.error.flatten());
|
|
13317
14183
|
return [];
|
|
@@ -13375,7 +14241,7 @@ var init_MapsExtractor = __esm({
|
|
|
13375
14241
|
});
|
|
13376
14242
|
return results;
|
|
13377
14243
|
});
|
|
13378
|
-
const result =
|
|
14244
|
+
const result = import_zod18.z.array(RawMapsAboutAttributeSchema).safeParse(raw);
|
|
13379
14245
|
if (!result.success) {
|
|
13380
14246
|
console.warn("[MapsExtractor] about parse failed", result.error.flatten());
|
|
13381
14247
|
return [];
|
|
@@ -13767,11 +14633,11 @@ function mapsErrorResponse(c, err, errorCode) {
|
|
|
13767
14633
|
attempts: rotationError?.attempts ?? void 0
|
|
13768
14634
|
}, retryable ? 503 : 500);
|
|
13769
14635
|
}
|
|
13770
|
-
var
|
|
14636
|
+
var import_hono6, mapsApp;
|
|
13771
14637
|
var init_maps_routes = __esm({
|
|
13772
14638
|
"src/api/maps-routes.ts"() {
|
|
13773
14639
|
"use strict";
|
|
13774
|
-
|
|
14640
|
+
import_hono6 = require("hono");
|
|
13775
14641
|
init_db();
|
|
13776
14642
|
init_rates();
|
|
13777
14643
|
init_MapsExtractor();
|
|
@@ -13782,7 +14648,7 @@ var init_maps_routes = __esm({
|
|
|
13782
14648
|
init_browser_service_env();
|
|
13783
14649
|
init_maps_search_rotation();
|
|
13784
14650
|
init_concurrency_gates();
|
|
13785
|
-
mapsApp = new
|
|
14651
|
+
mapsApp = new import_hono6.Hono();
|
|
13786
14652
|
mapsApp.post("/search", createApiKeyAuth(), async (c) => {
|
|
13787
14653
|
const user = c.get("user");
|
|
13788
14654
|
const body = await c.req.json().catch(() => ({}));
|
|
@@ -14058,17 +14924,17 @@ function reportTitle(full) {
|
|
|
14058
14924
|
const title = full.split("\n").find((line) => line.startsWith("# "));
|
|
14059
14925
|
return title?.replace(/^#\s+/, "").trim() || "MCP Scraper Report";
|
|
14060
14926
|
}
|
|
14061
|
-
function
|
|
14062
|
-
return process.env.MCP_SCRAPER_OUTPUT_DIR?.trim() || (0,
|
|
14927
|
+
function outputBaseDir2() {
|
|
14928
|
+
return process.env.MCP_SCRAPER_OUTPUT_DIR?.trim() || (0, import_node_path8.join)((0, import_node_os6.homedir)(), "Downloads", "mcp-scraper");
|
|
14063
14929
|
}
|
|
14064
14930
|
function saveFullReport(full) {
|
|
14065
14931
|
if (!reportSavingEnabled || process.env.MCP_SCRAPER_SAVE_REPORTS === "false") return null;
|
|
14066
|
-
const outDir =
|
|
14932
|
+
const outDir = outputBaseDir2();
|
|
14067
14933
|
try {
|
|
14068
|
-
(0,
|
|
14934
|
+
(0, import_node_fs5.mkdirSync)(outDir, { recursive: true });
|
|
14069
14935
|
const stamp = (/* @__PURE__ */ new Date()).toISOString().replace(/[:.]/g, "-");
|
|
14070
|
-
const file = (0,
|
|
14071
|
-
(0,
|
|
14936
|
+
const file = (0, import_node_path8.join)(outDir, `${stamp}-${slugifyReportName(reportTitle(full))}.md`);
|
|
14937
|
+
(0, import_node_fs5.writeFileSync)(file, full, "utf8");
|
|
14072
14938
|
return file;
|
|
14073
14939
|
} catch {
|
|
14074
14940
|
return null;
|
|
@@ -14077,12 +14943,12 @@ function saveFullReport(full) {
|
|
|
14077
14943
|
function persistScreenshotLocally(base64, url) {
|
|
14078
14944
|
if (!reportSavingEnabled || process.env.MCP_SCRAPER_SAVE_REPORTS === "false") return null;
|
|
14079
14945
|
try {
|
|
14080
|
-
const dir = (0,
|
|
14081
|
-
(0,
|
|
14946
|
+
const dir = (0, import_node_path8.join)(outputBaseDir2(), "screenshots");
|
|
14947
|
+
(0, import_node_fs5.mkdirSync)(dir, { recursive: true });
|
|
14082
14948
|
const stamp = (/* @__PURE__ */ new Date()).toISOString().replace(/[:.]/g, "-");
|
|
14083
14949
|
const slug = url.replace(/^https?:\/\//, "").replace(/[^a-z0-9]+/gi, "-").replace(/^-+|-+$/g, "").slice(0, 60);
|
|
14084
|
-
const filePath = (0,
|
|
14085
|
-
(0,
|
|
14950
|
+
const filePath = (0, import_node_path8.join)(dir, `${stamp}-${slug}.png`);
|
|
14951
|
+
(0, import_node_fs5.writeFileSync)(filePath, Buffer.from(base64, "base64"));
|
|
14086
14952
|
return filePath;
|
|
14087
14953
|
} catch {
|
|
14088
14954
|
return null;
|
|
@@ -14399,7 +15265,9 @@ ${headingSection}${kpoSection}${brandingSection}${bodySection}${screenshotSectio
|
|
|
14399
15265
|
entityTypes: kpo?.type ?? [],
|
|
14400
15266
|
napScore: kpo?.napScore ?? null,
|
|
14401
15267
|
missingSchemaFields: kpo?.missingFields ?? [],
|
|
14402
|
-
screenshotSaved: screenshotPath ?? null
|
|
15268
|
+
screenshotSaved: screenshotPath ?? null,
|
|
15269
|
+
branding: branding ?? null,
|
|
15270
|
+
mediaAssets: media?.assets ?? null
|
|
14403
15271
|
};
|
|
14404
15272
|
if (screenshotMeta?.base64) {
|
|
14405
15273
|
return {
|
|
@@ -15371,6 +16239,216 @@ ${chunkRows}` : "",
|
|
|
15371
16239
|
}
|
|
15372
16240
|
};
|
|
15373
16241
|
}
|
|
16242
|
+
function structuredInstagramBrowser(raw) {
|
|
16243
|
+
const browser = raw && typeof raw === "object" ? raw : {};
|
|
16244
|
+
const mode = browser.mode === "local" ? "local" : "hosted";
|
|
16245
|
+
const requestedMode = browser.requestedMode === "local" || browser.requestedMode === "hosted" || browser.requestedMode === "auto" ? browser.requestedMode : "auto";
|
|
16246
|
+
const profileSource = browser.profileSource === "managed_profile" || browser.profileSource === "direct_profile_dir" ? browser.profileSource : "hosted";
|
|
16247
|
+
return {
|
|
16248
|
+
mode,
|
|
16249
|
+
requestedMode,
|
|
16250
|
+
profileName: typeof browser.profileName === "string" ? browser.profileName : null,
|
|
16251
|
+
profileSource,
|
|
16252
|
+
profileDirConfigured: browser.profileDirConfigured === true,
|
|
16253
|
+
executablePathConfigured: browser.executablePathConfigured === true
|
|
16254
|
+
};
|
|
16255
|
+
}
|
|
16256
|
+
function structuredInstagramPagination(raw, input) {
|
|
16257
|
+
const pagination = raw && typeof raw === "object" ? raw : {};
|
|
16258
|
+
const stages = Array.isArray(pagination.stages) ? pagination.stages : [];
|
|
16259
|
+
const stoppedReason = typeof pagination.stoppedReason === "string" && ["max_items", "reported_post_count", "stable_scrolls", "max_scrolls", "no_scrolls"].includes(pagination.stoppedReason) ? pagination.stoppedReason : "no_scrolls";
|
|
16260
|
+
return {
|
|
16261
|
+
maxItems: typeof pagination.maxItems === "number" ? pagination.maxItems : Number(input.maxItems ?? 50),
|
|
16262
|
+
maxScrolls: typeof pagination.maxScrolls === "number" ? pagination.maxScrolls : Number(input.maxScrolls ?? 10),
|
|
16263
|
+
attemptedScrolls: typeof pagination.attemptedScrolls === "number" ? pagination.attemptedScrolls : 0,
|
|
16264
|
+
stableScrolls: typeof pagination.stableScrolls === "number" ? pagination.stableScrolls : 0,
|
|
16265
|
+
stableScrollLimit: typeof pagination.stableScrollLimit === "number" ? pagination.stableScrollLimit : Number(input.stableScrollLimit ?? 4),
|
|
16266
|
+
scrollDelayMs: typeof pagination.scrollDelayMs === "number" ? pagination.scrollDelayMs : Number(input.scrollDelayMs ?? 1200),
|
|
16267
|
+
reachedMaxItems: pagination.reachedMaxItems === true,
|
|
16268
|
+
reachedReportedPostCount: pagination.reachedReportedPostCount === true,
|
|
16269
|
+
finalScrollHeight: typeof pagination.finalScrollHeight === "number" ? pagination.finalScrollHeight : null,
|
|
16270
|
+
stoppedReason,
|
|
16271
|
+
stages: stages.map((stage) => {
|
|
16272
|
+
const row = stage && typeof stage === "object" ? stage : {};
|
|
16273
|
+
return {
|
|
16274
|
+
stage: String(row.stage ?? ""),
|
|
16275
|
+
itemCount: typeof row.itemCount === "number" ? row.itemCount : 0,
|
|
16276
|
+
addedCount: typeof row.addedCount === "number" ? row.addedCount : 0,
|
|
16277
|
+
scrollY: typeof row.scrollY === "number" ? row.scrollY : null,
|
|
16278
|
+
scrollHeight: typeof row.scrollHeight === "number" ? row.scrollHeight : null
|
|
16279
|
+
};
|
|
16280
|
+
})
|
|
16281
|
+
};
|
|
16282
|
+
}
|
|
16283
|
+
function formatInstagramProfileContent(raw, input) {
|
|
16284
|
+
const parsed = parseData(raw);
|
|
16285
|
+
if ("error" in parsed) return { content: [{ type: "text", text: parsed.error }], isError: true };
|
|
16286
|
+
const d = parsed.data;
|
|
16287
|
+
const items = Array.isArray(d.items) ? d.items : [];
|
|
16288
|
+
const typeCounts = d.typeCounts;
|
|
16289
|
+
const limitations = Array.isArray(d.limitations) ? d.limitations.map(String) : [];
|
|
16290
|
+
const browser = structuredInstagramBrowser(d.browser);
|
|
16291
|
+
const pagination = structuredInstagramPagination(d.pagination, input);
|
|
16292
|
+
const itemRows = items.slice(0, 100).map(
|
|
16293
|
+
(item, i) => `| ${i + 1} | ${item.type} | \`${item.shortcode}\` | ${item.url} | ${cell(item.firstSeenStage ?? "")} |`
|
|
16294
|
+
).join("\n");
|
|
16295
|
+
const browserLabel = browser.mode === "local" ? `local profile ${browser.profileName ? `\`${browser.profileName}\`` : "(unnamed)"}` : "hosted browser";
|
|
16296
|
+
const full = [
|
|
16297
|
+
`# Instagram Profile Content: ${d.handle ?? input.handle ?? input.url ?? "profile"}`,
|
|
16298
|
+
`**Collected:** ${items.length} items \xB7 posts ${typeCounts?.post ?? 0} \xB7 reels ${typeCounts?.reel ?? 0} \xB7 tv ${typeCounts?.tv ?? 0}`,
|
|
16299
|
+
`**Browser:** ${browserLabel}`,
|
|
16300
|
+
`**Pagination:** ${pagination.attemptedScrolls} scrolls \xB7 stopped: ${pagination.stoppedReason}`,
|
|
16301
|
+
d.reportedPostCountText ? `**Profile count:** ${d.reportedPostCountText}` : "",
|
|
16302
|
+
d.followerCountText ? `**Followers:** ${d.followerCountText}` : "",
|
|
16303
|
+
`
|
|
16304
|
+
## Content Links
|
|
16305
|
+
| # | Type | Shortcode | URL | First Seen |
|
|
16306
|
+
|---|------|-----------|-----|------------|
|
|
16307
|
+
${itemRows || "| \u2014 | \u2014 | \u2014 | \u2014 | \u2014 |"}`,
|
|
16308
|
+
limitations.length ? `
|
|
16309
|
+
## Limits
|
|
16310
|
+
${limitations.map((l) => `- ${l}`).join("\n")}` : "",
|
|
16311
|
+
`
|
|
16312
|
+
---
|
|
16313
|
+
\u{1F4A1} Use \`instagram_media_download\` with one of these URLs to download text, image, reel tracks, and optional transcript.`
|
|
16314
|
+
].filter(Boolean).join("\n");
|
|
16315
|
+
return {
|
|
16316
|
+
...oneBlock(full),
|
|
16317
|
+
structuredContent: {
|
|
16318
|
+
handle: String(d.handle ?? input.handle ?? ""),
|
|
16319
|
+
profileUrl: String(d.profileUrl ?? input.url ?? ""),
|
|
16320
|
+
pageUrl: String(d.pageUrl ?? d.profileUrl ?? input.url ?? ""),
|
|
16321
|
+
browser,
|
|
16322
|
+
profileName: typeof d.profileName === "string" ? d.profileName : null,
|
|
16323
|
+
reportedPostCount: typeof d.reportedPostCount === "number" ? d.reportedPostCount : null,
|
|
16324
|
+
reportedPostCountText: typeof d.reportedPostCountText === "string" ? d.reportedPostCountText : null,
|
|
16325
|
+
followerCountText: typeof d.followerCountText === "string" ? d.followerCountText : null,
|
|
16326
|
+
followingCountText: typeof d.followingCountText === "string" ? d.followingCountText : null,
|
|
16327
|
+
collectedContentCount: items.length,
|
|
16328
|
+
typeCounts: {
|
|
16329
|
+
post: Number(typeCounts?.post ?? 0),
|
|
16330
|
+
reel: Number(typeCounts?.reel ?? 0),
|
|
16331
|
+
tv: Number(typeCounts?.tv ?? 0)
|
|
16332
|
+
},
|
|
16333
|
+
pagination,
|
|
16334
|
+
limited: d.limited === true,
|
|
16335
|
+
limitations,
|
|
16336
|
+
items: items.map((item) => ({
|
|
16337
|
+
url: String(item.url ?? ""),
|
|
16338
|
+
type: item.type,
|
|
16339
|
+
shortcode: String(item.shortcode ?? ""),
|
|
16340
|
+
anchorText: item.anchorText ?? null,
|
|
16341
|
+
firstSeenStage: String(item.firstSeenStage ?? "")
|
|
16342
|
+
}))
|
|
16343
|
+
}
|
|
16344
|
+
};
|
|
16345
|
+
}
|
|
16346
|
+
function structuredInstagramTrack(track) {
|
|
16347
|
+
if (!track) return null;
|
|
16348
|
+
return {
|
|
16349
|
+
url: String(track.url ?? ""),
|
|
16350
|
+
streamType: track.streamType ?? "unknown",
|
|
16351
|
+
bitrate: typeof track.bitrate === "number" ? track.bitrate : null,
|
|
16352
|
+
durationSec: typeof track.durationSec === "number" ? track.durationSec : null,
|
|
16353
|
+
vencodeTag: track.vencodeTag ?? null,
|
|
16354
|
+
width: typeof track.width === "number" ? track.width : null,
|
|
16355
|
+
height: typeof track.height === "number" ? track.height : null
|
|
16356
|
+
};
|
|
16357
|
+
}
|
|
16358
|
+
function formatInstagramMediaDownload(raw, input) {
|
|
16359
|
+
const parsed = parseData(raw);
|
|
16360
|
+
if ("error" in parsed) return { content: [{ type: "text", text: parsed.error }], isError: true };
|
|
16361
|
+
const d = parsed.data;
|
|
16362
|
+
const tracks = Array.isArray(d.tracks) ? d.tracks : [];
|
|
16363
|
+
const downloads = Array.isArray(d.downloads) ? d.downloads : [];
|
|
16364
|
+
const warnings = Array.isArray(d.warnings) ? d.warnings.map(String) : [];
|
|
16365
|
+
const limitations = Array.isArray(d.limitations) ? d.limitations.map(String) : [];
|
|
16366
|
+
const transcript = d.transcript;
|
|
16367
|
+
const transcriptText = transcript?.text ?? "";
|
|
16368
|
+
const chunks = transcript?.chunks ?? [];
|
|
16369
|
+
const browser = structuredInstagramBrowser(d.browser);
|
|
16370
|
+
const browserLabel = browser.mode === "local" ? `local profile ${browser.profileName ? `\`${browser.profileName}\`` : "(unnamed)"}` : "hosted browser";
|
|
16371
|
+
const downloadRows = downloads.map((download, i) => {
|
|
16372
|
+
const status = download.error ? `error: ${cell(download.error)}` : `${download.sizeBytes ?? 0} bytes`;
|
|
16373
|
+
return `| ${i + 1} | ${download.kind} | ${download.savedPath ? `\`${download.savedPath}\`` : "\u2014"} | ${status} |`;
|
|
16374
|
+
}).join("\n");
|
|
16375
|
+
const trackRows = tracks.slice(0, 20).map(
|
|
16376
|
+
(track, i) => `| ${i + 1} | ${track.streamType} | ${track.bitrate ?? "\u2014"} | ${track.durationSec ?? "\u2014"} | ${cell(track.vencodeTag ?? "")} |`
|
|
16377
|
+
).join("\n");
|
|
16378
|
+
const full = [
|
|
16379
|
+
`# Instagram Media Download`,
|
|
16380
|
+
`**URL:** ${d.pageUrl ?? input.url}`,
|
|
16381
|
+
`**Browser:** ${browserLabel}`,
|
|
16382
|
+
d.ownerName ? `**Owner:** ${d.ownerName}` : "",
|
|
16383
|
+
d.shortcode ? `**Shortcode:** \`${d.shortcode}\`` : "",
|
|
16384
|
+
d.caption ? `
|
|
16385
|
+
## Caption
|
|
16386
|
+
${truncate(String(d.caption), 1200)}` : "",
|
|
16387
|
+
d.imageUrl ? `
|
|
16388
|
+
## Image
|
|
16389
|
+
${d.imageUrl}` : "",
|
|
16390
|
+
tracks.length ? `
|
|
16391
|
+
## Media Tracks
|
|
16392
|
+
| # | Type | Bitrate | Duration | Tag |
|
|
16393
|
+
|---|------|---------|----------|-----|
|
|
16394
|
+
${trackRows}` : "\n## Media Tracks\n*None captured.*",
|
|
16395
|
+
downloads.length ? `
|
|
16396
|
+
## Downloads
|
|
16397
|
+
| # | Kind | File | Status |
|
|
16398
|
+
|---|------|------|--------|
|
|
16399
|
+
${downloadRows}` : "",
|
|
16400
|
+
d.outputDir ? `
|
|
16401
|
+
**Output directory:** \`${d.outputDir}\`` : "",
|
|
16402
|
+
transcript ? `
|
|
16403
|
+
## Transcript
|
|
16404
|
+
**${wordCount(transcriptText)} words** \xB7 ${chunks.length} chunks
|
|
16405
|
+
|
|
16406
|
+
${transcriptText}` : "",
|
|
16407
|
+
warnings.length ? `
|
|
16408
|
+
## Warnings
|
|
16409
|
+
${warnings.map((w) => `- ${w}`).join("\n")}` : "",
|
|
16410
|
+
limitations.length ? `
|
|
16411
|
+
## Limits
|
|
16412
|
+
${limitations.map((l) => `- ${l}`).join("\n")}` : "",
|
|
16413
|
+
`
|
|
16414
|
+
---
|
|
16415
|
+
\u{1F4A1} Reels may expose separate video-only and audio-only MP4 tracks. Use the muxed file when present; otherwise use the selected video/audio track URLs or saved files.`
|
|
16416
|
+
].filter(Boolean).join("\n");
|
|
16417
|
+
return {
|
|
16418
|
+
...oneBlock(full),
|
|
16419
|
+
structuredContent: {
|
|
16420
|
+
sourceUrl: String(d.sourceUrl ?? input.url),
|
|
16421
|
+
pageUrl: String(d.pageUrl ?? input.url),
|
|
16422
|
+
browser,
|
|
16423
|
+
type: d.type === "post" || d.type === "reel" || d.type === "tv" ? d.type : null,
|
|
16424
|
+
shortcode: typeof d.shortcode === "string" ? d.shortcode : null,
|
|
16425
|
+
ownerName: typeof d.ownerName === "string" ? d.ownerName : null,
|
|
16426
|
+
caption: typeof d.caption === "string" ? d.caption : null,
|
|
16427
|
+
imageUrl: typeof d.imageUrl === "string" ? d.imageUrl : null,
|
|
16428
|
+
trackCount: tracks.length,
|
|
16429
|
+
selectedVideoTrack: structuredInstagramTrack(d.selectedVideoTrack),
|
|
16430
|
+
selectedAudioTrack: structuredInstagramTrack(d.selectedAudioTrack),
|
|
16431
|
+
downloads: downloads.map((download) => ({
|
|
16432
|
+
kind: download.kind,
|
|
16433
|
+
url: download.url ?? null,
|
|
16434
|
+
savedPath: download.savedPath ?? null,
|
|
16435
|
+
sizeBytes: typeof download.sizeBytes === "number" ? download.sizeBytes : null,
|
|
16436
|
+
mimeType: download.mimeType ?? null,
|
|
16437
|
+
error: download.error ?? null
|
|
16438
|
+
})),
|
|
16439
|
+
outputDir: typeof d.outputDir === "string" ? d.outputDir : null,
|
|
16440
|
+
warnings,
|
|
16441
|
+
limitations,
|
|
16442
|
+
transcript: transcript ? {
|
|
16443
|
+
wordCount: wordCount(transcriptText),
|
|
16444
|
+
chunkCount: chunks.length,
|
|
16445
|
+
durationMs: typeof transcript.durationMs === "number" ? transcript.durationMs : null,
|
|
16446
|
+
transcriptText,
|
|
16447
|
+
chunks: structuredTranscriptChunks(chunks)
|
|
16448
|
+
} : null
|
|
16449
|
+
}
|
|
16450
|
+
};
|
|
16451
|
+
}
|
|
15374
16452
|
function formatCaptureSerpSnapshot(raw, input) {
|
|
15375
16453
|
const parsed = parseData(raw);
|
|
15376
16454
|
if ("error" in parsed) return { content: [{ type: "text", text: parsed.error }], isError: true };
|
|
@@ -15461,13 +16539,13 @@ ${rows}` : ""
|
|
|
15461
16539
|
}
|
|
15462
16540
|
};
|
|
15463
16541
|
}
|
|
15464
|
-
var
|
|
16542
|
+
var import_node_fs5, import_node_os6, import_node_path8, reportSavingEnabled;
|
|
15465
16543
|
var init_mcp_response_formatter = __esm({
|
|
15466
16544
|
"src/mcp/mcp-response-formatter.ts"() {
|
|
15467
16545
|
"use strict";
|
|
15468
|
-
|
|
15469
|
-
|
|
15470
|
-
|
|
16546
|
+
import_node_fs5 = require("fs");
|
|
16547
|
+
import_node_os6 = require("os");
|
|
16548
|
+
import_node_path8 = require("path");
|
|
15471
16549
|
init_errors();
|
|
15472
16550
|
init_workflow_catalog();
|
|
15473
16551
|
reportSavingEnabled = true;
|
|
@@ -15574,7 +16652,7 @@ function localLocationFileAllowed() {
|
|
|
15574
16652
|
async function existingPath(value) {
|
|
15575
16653
|
const trimmed = value?.trim();
|
|
15576
16654
|
if (!trimmed) return null;
|
|
15577
|
-
await (0,
|
|
16655
|
+
await (0, import_promises7.access)(trimmed);
|
|
15578
16656
|
return trimmed;
|
|
15579
16657
|
}
|
|
15580
16658
|
async function resolveUsZipsPath(requestedPath) {
|
|
@@ -15592,7 +16670,7 @@ async function loadZipGroups(stateAbbr, requestedPath, warnings) {
|
|
|
15592
16670
|
}
|
|
15593
16671
|
const path6 = await resolveUsZipsPath(requestedPath);
|
|
15594
16672
|
if (!path6) return { path: null, groups: /* @__PURE__ */ new Map() };
|
|
15595
|
-
const records = csvRecords(await (0,
|
|
16673
|
+
const records = csvRecords(await (0, import_promises7.readFile)(path6, "utf8"));
|
|
15596
16674
|
const groups = /* @__PURE__ */ new Map();
|
|
15597
16675
|
for (const record of records) {
|
|
15598
16676
|
const state = (record.state_abbr ?? record.state ?? "").trim().toUpperCase();
|
|
@@ -15643,11 +16721,11 @@ async function resolveDirectoryMarkets(options) {
|
|
|
15643
16721
|
}
|
|
15644
16722
|
return { markets, censusSourceUrl: sourceUrl, usZipsSourcePath: zipData.path, warnings };
|
|
15645
16723
|
}
|
|
15646
|
-
var
|
|
16724
|
+
var import_promises7, POPULATION_YEARS, STATE_META, STATE_BY_NAME;
|
|
15647
16725
|
var init_location_db = __esm({
|
|
15648
16726
|
"src/directory/location-db.ts"() {
|
|
15649
16727
|
"use strict";
|
|
15650
|
-
|
|
16728
|
+
import_promises7 = require("fs/promises");
|
|
15651
16729
|
init_csv();
|
|
15652
16730
|
POPULATION_YEARS = [2020, 2021, 2022, 2023, 2024, 2025];
|
|
15653
16731
|
STATE_META = {
|
|
@@ -15863,11 +16941,11 @@ function csvRowsFor(result) {
|
|
|
15863
16941
|
return rows;
|
|
15864
16942
|
}
|
|
15865
16943
|
async function saveDirectoryCsv(result) {
|
|
15866
|
-
const outDir = (0,
|
|
15867
|
-
await (0,
|
|
16944
|
+
const outDir = (0, import_node_path9.join)(outputBaseDir2(), "directory-workflows");
|
|
16945
|
+
await (0, import_promises8.mkdir)(outDir, { recursive: true });
|
|
15868
16946
|
const stamp = result.extractedAt.replace(/[:.]/g, "-");
|
|
15869
16947
|
const slug = `${result.state}-${result.query}`.toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-+|-+$/g, "").slice(0, 80);
|
|
15870
|
-
const path6 = (0,
|
|
16948
|
+
const path6 = (0, import_node_path9.join)(outDir, `${stamp}-${slug}-directory-workflow.csv`);
|
|
15871
16949
|
const headers = [
|
|
15872
16950
|
"source_query",
|
|
15873
16951
|
"source_location",
|
|
@@ -15899,7 +16977,7 @@ async function saveDirectoryCsv(result) {
|
|
|
15899
16977
|
"extracted_at",
|
|
15900
16978
|
"duration_ms"
|
|
15901
16979
|
];
|
|
15902
|
-
await (0,
|
|
16980
|
+
await (0, import_promises8.writeFile)(path6, rowsToCsv(headers, csvRowsFor(result)), "utf8");
|
|
15903
16981
|
return path6;
|
|
15904
16982
|
}
|
|
15905
16983
|
async function runDirectoryWorkflowFromPlan(options, plan) {
|
|
@@ -15925,47 +17003,47 @@ async function runDirectoryWorkflowFromPlan(options, plan) {
|
|
|
15925
17003
|
const csvPath = options.saveCsv ? await saveDirectoryCsv(base) : null;
|
|
15926
17004
|
return { ...base, csvPath };
|
|
15927
17005
|
}
|
|
15928
|
-
var
|
|
17006
|
+
var import_promises8, import_node_path9, import_zod19, DirectoryWorkflowOptionsSchema;
|
|
15929
17007
|
var init_directory_workflow = __esm({
|
|
15930
17008
|
"src/directory/directory-workflow.ts"() {
|
|
15931
17009
|
"use strict";
|
|
15932
|
-
|
|
15933
|
-
|
|
15934
|
-
|
|
17010
|
+
import_promises8 = require("fs/promises");
|
|
17011
|
+
import_node_path9 = require("path");
|
|
17012
|
+
import_zod19 = require("zod");
|
|
15935
17013
|
init_mcp_response_formatter();
|
|
15936
17014
|
init_browser_service_env();
|
|
15937
17015
|
init_maps_search_rotation();
|
|
15938
17016
|
init_schemas3();
|
|
15939
17017
|
init_csv();
|
|
15940
17018
|
init_location_db();
|
|
15941
|
-
DirectoryWorkflowOptionsSchema =
|
|
15942
|
-
query:
|
|
15943
|
-
state:
|
|
15944
|
-
minPopulation:
|
|
15945
|
-
populationYear:
|
|
15946
|
-
maxCities:
|
|
15947
|
-
maxResultsPerCity:
|
|
15948
|
-
concurrency:
|
|
15949
|
-
includeZipGroups:
|
|
15950
|
-
usZipsCsvPath:
|
|
15951
|
-
saveCsv:
|
|
15952
|
-
gl:
|
|
15953
|
-
hl:
|
|
15954
|
-
proxyMode:
|
|
15955
|
-
proxyZip:
|
|
15956
|
-
debug:
|
|
15957
|
-
headless:
|
|
15958
|
-
kernelApiKey:
|
|
17019
|
+
DirectoryWorkflowOptionsSchema = import_zod19.z.object({
|
|
17020
|
+
query: import_zod19.z.string().min(1),
|
|
17021
|
+
state: import_zod19.z.string().min(2).default("TN"),
|
|
17022
|
+
minPopulation: import_zod19.z.number().int().min(0).default(1e5),
|
|
17023
|
+
populationYear: import_zod19.z.union(POPULATION_YEARS.map((year) => import_zod19.z.literal(year))).default(2025),
|
|
17024
|
+
maxCities: import_zod19.z.number().int().min(1).max(100).default(25),
|
|
17025
|
+
maxResultsPerCity: import_zod19.z.number().int().min(1).max(50).default(50),
|
|
17026
|
+
concurrency: import_zod19.z.number().int().min(1).max(5).default(5),
|
|
17027
|
+
includeZipGroups: import_zod19.z.boolean().default(true),
|
|
17028
|
+
usZipsCsvPath: import_zod19.z.string().optional(),
|
|
17029
|
+
saveCsv: import_zod19.z.boolean().default(true),
|
|
17030
|
+
gl: import_zod19.z.string().length(2).default("us"),
|
|
17031
|
+
hl: import_zod19.z.string().length(2).default("en"),
|
|
17032
|
+
proxyMode: import_zod19.z.enum(["location", "configured", "none"]).default(DEFAULT_MAPS_PROXY_MODE),
|
|
17033
|
+
proxyZip: import_zod19.z.string().regex(/^\d{5}$/).optional(),
|
|
17034
|
+
debug: import_zod19.z.boolean().default(false),
|
|
17035
|
+
headless: import_zod19.z.boolean().default(true),
|
|
17036
|
+
kernelApiKey: import_zod19.z.string().optional()
|
|
15959
17037
|
});
|
|
15960
17038
|
}
|
|
15961
17039
|
});
|
|
15962
17040
|
|
|
15963
17041
|
// src/api/directory-routes.ts
|
|
15964
|
-
var
|
|
17042
|
+
var import_hono7, directoryApp;
|
|
15965
17043
|
var init_directory_routes = __esm({
|
|
15966
17044
|
"src/api/directory-routes.ts"() {
|
|
15967
17045
|
"use strict";
|
|
15968
|
-
|
|
17046
|
+
import_hono7 = require("hono");
|
|
15969
17047
|
init_api_auth();
|
|
15970
17048
|
init_db();
|
|
15971
17049
|
init_rates();
|
|
@@ -15973,7 +17051,7 @@ var init_directory_routes = __esm({
|
|
|
15973
17051
|
init_location_db();
|
|
15974
17052
|
init_browser_service_env();
|
|
15975
17053
|
init_concurrency_gates();
|
|
15976
|
-
directoryApp = new
|
|
17054
|
+
directoryApp = new import_hono7.Hono();
|
|
15977
17055
|
directoryApp.post("/run", createApiKeyAuth(), async (c) => {
|
|
15978
17056
|
const user = c.get("user");
|
|
15979
17057
|
const body = await c.req.json().catch(() => ({}));
|
|
@@ -16057,7 +17135,7 @@ var init_slugify = __esm({
|
|
|
16057
17135
|
|
|
16058
17136
|
// src/workflows/artifact-writer.ts
|
|
16059
17137
|
function workflowOutputBaseDir(outputDir) {
|
|
16060
|
-
return outputDir?.trim() || process.env.MCP_SCRAPER_OUTPUT_DIR?.trim() || (0,
|
|
17138
|
+
return outputDir?.trim() || process.env.MCP_SCRAPER_OUTPUT_DIR?.trim() || (0, import_node_path10.join)((0, import_node_os7.homedir)(), "Downloads", "mcp-scraper");
|
|
16061
17139
|
}
|
|
16062
17140
|
function timestamp() {
|
|
16063
17141
|
return (/* @__PURE__ */ new Date()).toISOString().replace(/[:.]/g, "-");
|
|
@@ -16066,13 +17144,13 @@ function safeSlug(value) {
|
|
|
16066
17144
|
return slugify(value).replace(/^-+|-+$/g, "").slice(0, 80) || "run";
|
|
16067
17145
|
}
|
|
16068
17146
|
function indexPath(baseDir = workflowOutputBaseDir()) {
|
|
16069
|
-
return (0,
|
|
17147
|
+
return (0, import_node_path10.join)(baseDir, "workflows", "index.json");
|
|
16070
17148
|
}
|
|
16071
17149
|
async function readIndex(baseDir) {
|
|
16072
17150
|
const path6 = indexPath(baseDir);
|
|
16073
|
-
if (!(0,
|
|
17151
|
+
if (!(0, import_node_fs6.existsSync)(path6)) return { runs: [] };
|
|
16074
17152
|
try {
|
|
16075
|
-
return JSON.parse(await (0,
|
|
17153
|
+
return JSON.parse(await (0, import_promises9.readFile)(path6, "utf8"));
|
|
16076
17154
|
} catch {
|
|
16077
17155
|
return { runs: [] };
|
|
16078
17156
|
}
|
|
@@ -16092,18 +17170,18 @@ async function updateWorkflowIndex(manifest, manifestPath, baseDir) {
|
|
|
16092
17170
|
summary: `${manifest.title} (${manifest.status})`
|
|
16093
17171
|
};
|
|
16094
17172
|
index.runs = [entry, ...index.runs.filter((r) => r.runId !== manifest.runId)].slice(0, 200);
|
|
16095
|
-
await (0,
|
|
16096
|
-
await (0,
|
|
17173
|
+
await (0, import_promises9.mkdir)((0, import_node_path10.dirname)(path6), { recursive: true });
|
|
17174
|
+
await (0, import_promises9.writeFile)(path6, JSON.stringify(index, null, 2), "utf8");
|
|
16097
17175
|
}
|
|
16098
|
-
var
|
|
17176
|
+
var import_promises9, import_node_fs6, import_node_os7, import_node_path10, import_node_child_process3, ArtifactWriter;
|
|
16099
17177
|
var init_artifact_writer = __esm({
|
|
16100
17178
|
"src/workflows/artifact-writer.ts"() {
|
|
16101
17179
|
"use strict";
|
|
16102
|
-
|
|
16103
|
-
|
|
16104
|
-
|
|
16105
|
-
|
|
16106
|
-
|
|
17180
|
+
import_promises9 = require("fs/promises");
|
|
17181
|
+
import_node_fs6 = require("fs");
|
|
17182
|
+
import_node_os7 = require("os");
|
|
17183
|
+
import_node_path10 = require("path");
|
|
17184
|
+
import_node_child_process3 = require("child_process");
|
|
16107
17185
|
init_csv();
|
|
16108
17186
|
init_slugify();
|
|
16109
17187
|
ArtifactWriter = class _ArtifactWriter {
|
|
@@ -16129,31 +17207,31 @@ var init_artifact_writer = __esm({
|
|
|
16129
17207
|
const runId = forcedRunId?.trim() || `${timestamp()}-${safeSlug(workflowId)}-${Math.random().toString(36).slice(2, 8)}`;
|
|
16130
17208
|
const nameSource = String(input.keyword ?? input.query ?? input.domain ?? input.state ?? workflowId);
|
|
16131
17209
|
const baseDir = workflowOutputBaseDir(outputDir);
|
|
16132
|
-
const runDir = (0,
|
|
16133
|
-
await (0,
|
|
17210
|
+
const runDir = (0, import_node_path10.join)(baseDir, "workflows", workflowId, `${timestamp()}-${safeSlug(nameSource)}-${safeSlug(runId).slice(0, 20)}`);
|
|
17211
|
+
await (0, import_promises9.mkdir)(runDir, { recursive: true });
|
|
16134
17212
|
return new _ArtifactWriter(workflowId, title, runId, baseDir, runDir, startedAt, input);
|
|
16135
17213
|
}
|
|
16136
17214
|
async remember(kind, label, path6, rows) {
|
|
16137
|
-
const size = await (0,
|
|
17215
|
+
const size = await (0, import_promises9.stat)(path6).then((s) => s.size).catch(() => void 0);
|
|
16138
17216
|
this.artifacts.push({ kind, label, path: path6, bytes: size, rows });
|
|
16139
17217
|
return path6;
|
|
16140
17218
|
}
|
|
16141
17219
|
async writeJson(label, relativePath, data) {
|
|
16142
|
-
const path6 = (0,
|
|
16143
|
-
await (0,
|
|
16144
|
-
await (0,
|
|
17220
|
+
const path6 = (0, import_node_path10.join)(this.runDir, relativePath);
|
|
17221
|
+
await (0, import_promises9.mkdir)((0, import_node_path10.dirname)(path6), { recursive: true });
|
|
17222
|
+
await (0, import_promises9.writeFile)(path6, JSON.stringify(data, null, 2), "utf8");
|
|
16145
17223
|
return this.remember("json", label, path6);
|
|
16146
17224
|
}
|
|
16147
17225
|
async writeText(label, relativePath, text, kind = "markdown") {
|
|
16148
|
-
const path6 = (0,
|
|
16149
|
-
await (0,
|
|
16150
|
-
await (0,
|
|
17226
|
+
const path6 = (0, import_node_path10.join)(this.runDir, relativePath);
|
|
17227
|
+
await (0, import_promises9.mkdir)((0, import_node_path10.dirname)(path6), { recursive: true });
|
|
17228
|
+
await (0, import_promises9.writeFile)(path6, text, "utf8");
|
|
16151
17229
|
return this.remember(kind, label, path6);
|
|
16152
17230
|
}
|
|
16153
17231
|
async writeCsv(label, relativePath, headers, rows) {
|
|
16154
|
-
const path6 = (0,
|
|
16155
|
-
await (0,
|
|
16156
|
-
await (0,
|
|
17232
|
+
const path6 = (0, import_node_path10.join)(this.runDir, relativePath);
|
|
17233
|
+
await (0, import_promises9.mkdir)((0, import_node_path10.dirname)(path6), { recursive: true });
|
|
17234
|
+
await (0, import_promises9.writeFile)(path6, rowsToCsv(headers, rows), "utf8");
|
|
16157
17235
|
return this.remember("csv", label, path6, rows.length);
|
|
16158
17236
|
}
|
|
16159
17237
|
async writeHtml(label, relativePath, html) {
|
|
@@ -16173,8 +17251,8 @@ var init_artifact_writer = __esm({
|
|
|
16173
17251
|
errors,
|
|
16174
17252
|
counts
|
|
16175
17253
|
};
|
|
16176
|
-
const path6 = (0,
|
|
16177
|
-
await (0,
|
|
17254
|
+
const path6 = (0, import_node_path10.join)(this.runDir, "manifest.json");
|
|
17255
|
+
await (0, import_promises9.writeFile)(path6, JSON.stringify(manifest, null, 2), "utf8");
|
|
16178
17256
|
await updateWorkflowIndex(manifest, path6, this.baseDir);
|
|
16179
17257
|
return path6;
|
|
16180
17258
|
}
|
|
@@ -16354,21 +17432,21 @@ function competitorRows(rows, targetDomain) {
|
|
|
16354
17432
|
source_url_count: entry.urls.size
|
|
16355
17433
|
})).sort((a, b) => Number(a.organic_best_position || 999) - Number(b.organic_best_position || 999));
|
|
16356
17434
|
}
|
|
16357
|
-
var
|
|
17435
|
+
var import_zod20, AgentPacketInputSchema, agentPacketWorkflowDefinition;
|
|
16358
17436
|
var init_agent_packet = __esm({
|
|
16359
17437
|
"src/workflows/workflows/agent-packet.ts"() {
|
|
16360
17438
|
"use strict";
|
|
16361
|
-
|
|
17439
|
+
import_zod20 = require("zod");
|
|
16362
17440
|
init_report_renderer();
|
|
16363
|
-
AgentPacketInputSchema =
|
|
16364
|
-
keyword:
|
|
16365
|
-
domain:
|
|
16366
|
-
location:
|
|
16367
|
-
maxQuestions:
|
|
16368
|
-
includeSerp:
|
|
16369
|
-
includePaa:
|
|
16370
|
-
includeAiOverview:
|
|
16371
|
-
returnPartial:
|
|
17441
|
+
AgentPacketInputSchema = import_zod20.z.object({
|
|
17442
|
+
keyword: import_zod20.z.string().min(1),
|
|
17443
|
+
domain: import_zod20.z.string().optional(),
|
|
17444
|
+
location: import_zod20.z.string().optional(),
|
|
17445
|
+
maxQuestions: import_zod20.z.number().int().min(1).max(200).default(40),
|
|
17446
|
+
includeSerp: import_zod20.z.boolean().default(true),
|
|
17447
|
+
includePaa: import_zod20.z.boolean().default(true),
|
|
17448
|
+
includeAiOverview: import_zod20.z.boolean().default(true),
|
|
17449
|
+
returnPartial: import_zod20.z.boolean().default(true)
|
|
16372
17450
|
});
|
|
16373
17451
|
agentPacketWorkflowDefinition = {
|
|
16374
17452
|
id: "agent-packet",
|
|
@@ -16566,22 +17644,22 @@ function directoryRows(result) {
|
|
|
16566
17644
|
}
|
|
16567
17645
|
return rows;
|
|
16568
17646
|
}
|
|
16569
|
-
var
|
|
17647
|
+
var import_zod21, DirectoryWorkflowCliInputSchema, DIRECTORY_CSV_HEADERS, directoryWorkflowDefinition;
|
|
16570
17648
|
var init_directory = __esm({
|
|
16571
17649
|
"src/workflows/workflows/directory.ts"() {
|
|
16572
17650
|
"use strict";
|
|
16573
|
-
|
|
17651
|
+
import_zod21 = require("zod");
|
|
16574
17652
|
init_report_renderer();
|
|
16575
17653
|
init_schemas3();
|
|
16576
|
-
DirectoryWorkflowCliInputSchema =
|
|
16577
|
-
query:
|
|
16578
|
-
state:
|
|
16579
|
-
minPopulation:
|
|
16580
|
-
maxCities:
|
|
16581
|
-
maxResultsPerCity:
|
|
16582
|
-
concurrency:
|
|
16583
|
-
proxyMode:
|
|
16584
|
-
saveCsv:
|
|
17654
|
+
DirectoryWorkflowCliInputSchema = import_zod21.z.object({
|
|
17655
|
+
query: import_zod21.z.string().min(1),
|
|
17656
|
+
state: import_zod21.z.string().min(2).default("TN"),
|
|
17657
|
+
minPopulation: import_zod21.z.number().int().min(0).default(1e5),
|
|
17658
|
+
maxCities: import_zod21.z.number().int().min(1).max(100).default(25),
|
|
17659
|
+
maxResultsPerCity: import_zod21.z.number().int().min(1).max(50).default(20),
|
|
17660
|
+
concurrency: import_zod21.z.number().int().min(1).max(5).default(5),
|
|
17661
|
+
proxyMode: import_zod21.z.enum(["location", "configured", "none"]).default(DEFAULT_MAPS_PROXY_MODE),
|
|
17662
|
+
saveCsv: import_zod21.z.boolean().default(true)
|
|
16585
17663
|
});
|
|
16586
17664
|
DIRECTORY_CSV_HEADERS = [
|
|
16587
17665
|
"source_query",
|
|
@@ -16690,25 +17768,25 @@ function termsFrom(texts) {
|
|
|
16690
17768
|
}
|
|
16691
17769
|
return [...counts.entries()].sort((a, b) => b[1] - a[1]).slice(0, 8).map(([term, count]) => `${term} (${count})`).join("; ");
|
|
16692
17770
|
}
|
|
16693
|
-
var
|
|
17771
|
+
var import_zod22, LocalCompetitiveAuditInputSchema, localCompetitiveAuditWorkflowDefinition;
|
|
16694
17772
|
var init_local_competitive_audit = __esm({
|
|
16695
17773
|
"src/workflows/workflows/local-competitive-audit.ts"() {
|
|
16696
17774
|
"use strict";
|
|
16697
|
-
|
|
17775
|
+
import_zod22 = require("zod");
|
|
16698
17776
|
init_report_renderer();
|
|
16699
17777
|
init_directory();
|
|
16700
17778
|
init_schemas3();
|
|
16701
|
-
LocalCompetitiveAuditInputSchema =
|
|
16702
|
-
query:
|
|
16703
|
-
state:
|
|
16704
|
-
minPopulation:
|
|
16705
|
-
maxCities:
|
|
16706
|
-
maxResultsPerCity:
|
|
16707
|
-
hydrateTop:
|
|
16708
|
-
maxReviews:
|
|
16709
|
-
concurrency:
|
|
16710
|
-
proxyMode:
|
|
16711
|
-
returnPartial:
|
|
17779
|
+
LocalCompetitiveAuditInputSchema = import_zod22.z.object({
|
|
17780
|
+
query: import_zod22.z.string().min(1),
|
|
17781
|
+
state: import_zod22.z.string().min(2).default("TN"),
|
|
17782
|
+
minPopulation: import_zod22.z.number().int().min(0).default(1e5),
|
|
17783
|
+
maxCities: import_zod22.z.number().int().min(1).max(100).default(25),
|
|
17784
|
+
maxResultsPerCity: import_zod22.z.number().int().min(1).max(50).default(20),
|
|
17785
|
+
hydrateTop: import_zod22.z.number().int().min(0).max(10).default(5),
|
|
17786
|
+
maxReviews: import_zod22.z.number().int().min(0).max(500).default(50),
|
|
17787
|
+
concurrency: import_zod22.z.number().int().min(1).max(5).default(5),
|
|
17788
|
+
proxyMode: import_zod22.z.enum(["location", "configured", "none"]).default(DEFAULT_MAPS_PROXY_MODE),
|
|
17789
|
+
returnPartial: import_zod22.z.boolean().default(true)
|
|
16712
17790
|
});
|
|
16713
17791
|
localCompetitiveAuditWorkflowDefinition = {
|
|
16714
17792
|
id: "local-competitive-audit",
|
|
@@ -17151,58 +18229,58 @@ async function extractPages(ctx, sources, warnings, labelPrefix) {
|
|
|
17151
18229
|
}
|
|
17152
18230
|
}).then((items) => items.filter((item) => item !== null));
|
|
17153
18231
|
}
|
|
17154
|
-
var
|
|
18232
|
+
var import_zod23, ProxyModeSchema, MapComparisonInputSchema, SerpComparisonInputSchema, PaaExpansionBriefInputSchema, AiOverviewLanguageInputSchema, mapComparisonWorkflowDefinition, serpComparisonWorkflowDefinition, paaExpansionBriefWorkflowDefinition, aiOverviewLanguageWorkflowDefinition;
|
|
17155
18233
|
var init_comparison_briefs = __esm({
|
|
17156
18234
|
"src/workflows/workflows/comparison-briefs.ts"() {
|
|
17157
18235
|
"use strict";
|
|
17158
|
-
|
|
18236
|
+
import_zod23 = require("zod");
|
|
17159
18237
|
init_report_renderer();
|
|
17160
18238
|
init_directory();
|
|
17161
18239
|
init_seo_workflow_utils();
|
|
17162
18240
|
init_schemas3();
|
|
17163
|
-
ProxyModeSchema =
|
|
17164
|
-
MapComparisonInputSchema =
|
|
17165
|
-
query:
|
|
17166
|
-
location:
|
|
17167
|
-
state:
|
|
17168
|
-
minPopulation:
|
|
17169
|
-
maxCities:
|
|
17170
|
-
maxResultsPerCity:
|
|
17171
|
-
hydrateTop:
|
|
17172
|
-
maxReviews:
|
|
17173
|
-
concurrency:
|
|
18241
|
+
ProxyModeSchema = import_zod23.z.enum(["location", "configured", "none"]);
|
|
18242
|
+
MapComparisonInputSchema = import_zod23.z.object({
|
|
18243
|
+
query: import_zod23.z.string().min(1),
|
|
18244
|
+
location: import_zod23.z.string().optional(),
|
|
18245
|
+
state: import_zod23.z.string().optional(),
|
|
18246
|
+
minPopulation: import_zod23.z.number().int().min(0).default(1e5),
|
|
18247
|
+
maxCities: import_zod23.z.number().int().min(1).max(100).default(5),
|
|
18248
|
+
maxResultsPerCity: import_zod23.z.number().int().min(1).max(50).default(20),
|
|
18249
|
+
hydrateTop: import_zod23.z.number().int().min(0).max(10).default(5),
|
|
18250
|
+
maxReviews: import_zod23.z.number().int().min(0).max(500).default(25),
|
|
18251
|
+
concurrency: import_zod23.z.number().int().min(1).max(5).default(5),
|
|
17174
18252
|
proxyMode: ProxyModeSchema.default(DEFAULT_MAPS_PROXY_MODE),
|
|
17175
|
-
returnPartial:
|
|
18253
|
+
returnPartial: import_zod23.z.boolean().default(true)
|
|
17176
18254
|
}).refine((input) => input.location || input.state, {
|
|
17177
18255
|
message: "Either location or state is required for map-comparison"
|
|
17178
18256
|
});
|
|
17179
|
-
SerpComparisonInputSchema =
|
|
17180
|
-
keyword:
|
|
17181
|
-
domain:
|
|
17182
|
-
url:
|
|
17183
|
-
location:
|
|
17184
|
-
maxResults:
|
|
17185
|
-
maxQuestions:
|
|
17186
|
-
extractTop:
|
|
17187
|
-
includePaa:
|
|
17188
|
-
includeAiOverview:
|
|
17189
|
-
returnPartial:
|
|
18257
|
+
SerpComparisonInputSchema = import_zod23.z.object({
|
|
18258
|
+
keyword: import_zod23.z.string().min(1),
|
|
18259
|
+
domain: import_zod23.z.string().optional(),
|
|
18260
|
+
url: import_zod23.z.string().url().optional(),
|
|
18261
|
+
location: import_zod23.z.string().optional(),
|
|
18262
|
+
maxResults: import_zod23.z.number().int().min(1).max(20).default(10),
|
|
18263
|
+
maxQuestions: import_zod23.z.number().int().min(1).max(200).default(40),
|
|
18264
|
+
extractTop: import_zod23.z.number().int().min(0).max(10).default(5),
|
|
18265
|
+
includePaa: import_zod23.z.boolean().default(true),
|
|
18266
|
+
includeAiOverview: import_zod23.z.boolean().default(true),
|
|
18267
|
+
returnPartial: import_zod23.z.boolean().default(true)
|
|
17190
18268
|
});
|
|
17191
|
-
PaaExpansionBriefInputSchema =
|
|
17192
|
-
keyword:
|
|
17193
|
-
location:
|
|
17194
|
-
maxQuestions:
|
|
17195
|
-
depth:
|
|
17196
|
-
returnPartial:
|
|
18269
|
+
PaaExpansionBriefInputSchema = import_zod23.z.object({
|
|
18270
|
+
keyword: import_zod23.z.string().min(1),
|
|
18271
|
+
location: import_zod23.z.string().optional(),
|
|
18272
|
+
maxQuestions: import_zod23.z.number().int().min(1).max(300).default(80),
|
|
18273
|
+
depth: import_zod23.z.number().int().min(1).max(6).default(3),
|
|
18274
|
+
returnPartial: import_zod23.z.boolean().default(true)
|
|
17197
18275
|
});
|
|
17198
|
-
AiOverviewLanguageInputSchema =
|
|
17199
|
-
keyword:
|
|
17200
|
-
domain:
|
|
17201
|
-
url:
|
|
17202
|
-
location:
|
|
17203
|
-
maxQuestions:
|
|
17204
|
-
extractTop:
|
|
17205
|
-
returnPartial:
|
|
18276
|
+
AiOverviewLanguageInputSchema = import_zod23.z.object({
|
|
18277
|
+
keyword: import_zod23.z.string().min(1),
|
|
18278
|
+
domain: import_zod23.z.string().optional(),
|
|
18279
|
+
url: import_zod23.z.string().url().optional(),
|
|
18280
|
+
location: import_zod23.z.string().optional(),
|
|
18281
|
+
maxQuestions: import_zod23.z.number().int().min(1).max(200).default(40),
|
|
18282
|
+
extractTop: import_zod23.z.number().int().min(0).max(8).default(3),
|
|
18283
|
+
returnPartial: import_zod23.z.boolean().default(true)
|
|
17206
18284
|
});
|
|
17207
18285
|
mapComparisonWorkflowDefinition = {
|
|
17208
18286
|
id: "map-comparison",
|
|
@@ -17653,7 +18731,7 @@ async function runWorkflowStep(id, rawInput, opts) {
|
|
|
17653
18731
|
const writtenArtifacts = await Promise.all(
|
|
17654
18732
|
artifacts.artifacts.map(async (artifact) => ({
|
|
17655
18733
|
...artifact,
|
|
17656
|
-
content: await (0,
|
|
18734
|
+
content: await (0, import_promises10.readFile)(artifact.path, "utf8").catch(() => "")
|
|
17657
18735
|
}))
|
|
17658
18736
|
);
|
|
17659
18737
|
return {
|
|
@@ -17672,12 +18750,12 @@ async function runWorkflowStep(id, rawInput, opts) {
|
|
|
17672
18750
|
artifacts: writtenArtifacts
|
|
17673
18751
|
};
|
|
17674
18752
|
}
|
|
17675
|
-
var
|
|
18753
|
+
var import_promises10, import_zod24, DEFINITIONS;
|
|
17676
18754
|
var init_registry2 = __esm({
|
|
17677
18755
|
"src/workflows/registry.ts"() {
|
|
17678
18756
|
"use strict";
|
|
17679
|
-
|
|
17680
|
-
|
|
18757
|
+
import_promises10 = require("fs/promises");
|
|
18758
|
+
import_zod24 = require("zod");
|
|
17681
18759
|
init_artifact_writer();
|
|
17682
18760
|
init_http_client2();
|
|
17683
18761
|
init_agent_packet();
|
|
@@ -17745,7 +18823,7 @@ async function readManifestFromSummary(summary) {
|
|
|
17745
18823
|
if (!summary.reportPath) return null;
|
|
17746
18824
|
const manifestPath = summary.reportPath.replace(/report\.html$/, "manifest.json");
|
|
17747
18825
|
try {
|
|
17748
|
-
return JSON.parse(await (0,
|
|
18826
|
+
return JSON.parse(await (0, import_promises11.readFile)(manifestPath, "utf8"));
|
|
17749
18827
|
} catch {
|
|
17750
18828
|
return null;
|
|
17751
18829
|
}
|
|
@@ -17960,47 +19038,47 @@ async function dispatchDueWorkflowSchedules(apiUrl, limit = 3) {
|
|
|
17960
19038
|
}
|
|
17961
19039
|
return { dispatched: results.length, results };
|
|
17962
19040
|
}
|
|
17963
|
-
var import_node_crypto3,
|
|
19041
|
+
var import_node_crypto3, import_promises11, import_hono8, import_zod25, workflowApp, WorkflowInputSchema, WorkflowIdSchema, CadenceSchema, ScheduleStatusSchema, RunBodySchema, ScheduleCreateSchema, SchedulePatchSchema, TERMINAL_RUN_STATUSES;
|
|
17964
19042
|
var init_workflow_routes = __esm({
|
|
17965
19043
|
"src/api/workflow-routes.ts"() {
|
|
17966
19044
|
"use strict";
|
|
17967
19045
|
import_node_crypto3 = require("crypto");
|
|
17968
|
-
|
|
17969
|
-
|
|
17970
|
-
|
|
19046
|
+
import_promises11 = require("fs/promises");
|
|
19047
|
+
import_hono8 = require("hono");
|
|
19048
|
+
import_zod25 = require("zod");
|
|
17971
19049
|
init_artifact_writer();
|
|
17972
19050
|
init_registry2();
|
|
17973
19051
|
init_api_auth();
|
|
17974
19052
|
init_db();
|
|
17975
19053
|
init_url_utils();
|
|
17976
19054
|
init_concurrency_gates();
|
|
17977
|
-
workflowApp = new
|
|
17978
|
-
WorkflowInputSchema =
|
|
17979
|
-
WorkflowIdSchema =
|
|
17980
|
-
CadenceSchema =
|
|
17981
|
-
ScheduleStatusSchema =
|
|
17982
|
-
RunBodySchema =
|
|
19055
|
+
workflowApp = new import_hono8.Hono();
|
|
19056
|
+
WorkflowInputSchema = import_zod25.z.record(import_zod25.z.unknown()).default({});
|
|
19057
|
+
WorkflowIdSchema = import_zod25.z.string().min(1);
|
|
19058
|
+
CadenceSchema = import_zod25.z.enum(["daily", "weekly", "monthly"]);
|
|
19059
|
+
ScheduleStatusSchema = import_zod25.z.enum(["active", "paused"]);
|
|
19060
|
+
RunBodySchema = import_zod25.z.object({
|
|
17983
19061
|
workflowId: WorkflowIdSchema,
|
|
17984
19062
|
input: WorkflowInputSchema,
|
|
17985
|
-
webhookUrl:
|
|
19063
|
+
webhookUrl: import_zod25.z.string().url().optional()
|
|
17986
19064
|
});
|
|
17987
|
-
ScheduleCreateSchema =
|
|
19065
|
+
ScheduleCreateSchema = import_zod25.z.object({
|
|
17988
19066
|
workflowId: WorkflowIdSchema,
|
|
17989
|
-
name:
|
|
19067
|
+
name: import_zod25.z.string().min(1).max(120).optional(),
|
|
17990
19068
|
input: WorkflowInputSchema,
|
|
17991
19069
|
cadence: CadenceSchema.default("weekly"),
|
|
17992
|
-
timezone:
|
|
17993
|
-
webhookUrl:
|
|
17994
|
-
nextRunAt:
|
|
19070
|
+
timezone: import_zod25.z.string().min(1).max(64).default("UTC"),
|
|
19071
|
+
webhookUrl: import_zod25.z.string().url().optional(),
|
|
19072
|
+
nextRunAt: import_zod25.z.string().datetime().optional()
|
|
17995
19073
|
});
|
|
17996
|
-
SchedulePatchSchema =
|
|
17997
|
-
name:
|
|
19074
|
+
SchedulePatchSchema = import_zod25.z.object({
|
|
19075
|
+
name: import_zod25.z.string().min(1).max(120).optional(),
|
|
17998
19076
|
status: ScheduleStatusSchema.optional(),
|
|
17999
19077
|
input: WorkflowInputSchema.optional(),
|
|
18000
19078
|
cadence: CadenceSchema.optional(),
|
|
18001
|
-
timezone:
|
|
18002
|
-
webhookUrl:
|
|
18003
|
-
nextRunAt:
|
|
19079
|
+
timezone: import_zod25.z.string().min(1).max(64).optional(),
|
|
19080
|
+
webhookUrl: import_zod25.z.string().url().nullable().optional(),
|
|
19081
|
+
nextRunAt: import_zod25.z.string().datetime().nullable().optional()
|
|
18004
19082
|
});
|
|
18005
19083
|
workflowApp.get("/definitions", createApiKeyAuth(), (c) => {
|
|
18006
19084
|
return c.json({ workflows: listWorkflowDefinitions() });
|
|
@@ -18135,7 +19213,7 @@ var init_workflow_routes = __esm({
|
|
|
18135
19213
|
return new Response(artifact.content, { headers });
|
|
18136
19214
|
}
|
|
18137
19215
|
try {
|
|
18138
|
-
const content = await (0,
|
|
19216
|
+
const content = await (0, import_promises11.readFile)(artifact.path);
|
|
18139
19217
|
return new Response(content, { headers });
|
|
18140
19218
|
} catch {
|
|
18141
19219
|
return c.json({ error: "Artifact file is no longer available" }, 410);
|
|
@@ -19794,52 +20872,52 @@ var init_PAAExtractor = __esm({
|
|
|
19794
20872
|
});
|
|
19795
20873
|
|
|
19796
20874
|
// src/output/OutputSerializer.ts
|
|
19797
|
-
var
|
|
20875
|
+
var import_node_fs7, import_node_path11, import_papaparse2, OutputSerializer;
|
|
19798
20876
|
var init_OutputSerializer = __esm({
|
|
19799
20877
|
"src/output/OutputSerializer.ts"() {
|
|
19800
20878
|
"use strict";
|
|
19801
|
-
|
|
19802
|
-
|
|
20879
|
+
import_node_fs7 = require("fs");
|
|
20880
|
+
import_node_path11 = __toESM(require("path"), 1);
|
|
19803
20881
|
import_papaparse2 = __toESM(require("papaparse"), 1);
|
|
19804
20882
|
OutputSerializer = class {
|
|
19805
20883
|
async writeJSON(result, outputDir) {
|
|
19806
|
-
await
|
|
20884
|
+
await import_node_fs7.promises.mkdir(outputDir, { recursive: true });
|
|
19807
20885
|
const slug = result.seed.toLowerCase().replace(/\W+/g, "-").slice(0, 40);
|
|
19808
20886
|
const filename = `${slug}-${Date.now()}.json`;
|
|
19809
|
-
const fullPath =
|
|
19810
|
-
await
|
|
20887
|
+
const fullPath = import_node_path11.default.join(outputDir, filename);
|
|
20888
|
+
await import_node_fs7.promises.writeFile(fullPath, JSON.stringify(result, null, 2), "utf8");
|
|
19811
20889
|
return fullPath;
|
|
19812
20890
|
}
|
|
19813
20891
|
async writeCSV(rows, outputDir) {
|
|
19814
|
-
await
|
|
20892
|
+
await import_node_fs7.promises.mkdir(outputDir, { recursive: true });
|
|
19815
20893
|
const seedRaw = rows[0]?.seed_query ?? "paa";
|
|
19816
20894
|
const slug = seedRaw.toLowerCase().replace(/\W+/g, "-").slice(0, 40);
|
|
19817
20895
|
const csv = import_papaparse2.default.unparse(rows, { header: true });
|
|
19818
20896
|
const filename = `${slug}-${Date.now()}.csv`;
|
|
19819
|
-
const fullPath =
|
|
19820
|
-
await
|
|
20897
|
+
const fullPath = import_node_path11.default.join(outputDir, filename);
|
|
20898
|
+
await import_node_fs7.promises.writeFile(fullPath, csv, "utf8");
|
|
19821
20899
|
return fullPath;
|
|
19822
20900
|
}
|
|
19823
20901
|
async writeVideoCSV(videos, seed, outputDir) {
|
|
19824
|
-
await
|
|
20902
|
+
await import_node_fs7.promises.mkdir(outputDir, { recursive: true });
|
|
19825
20903
|
const slug = seed.toLowerCase().replace(/\W+/g, "-").slice(0, 40);
|
|
19826
20904
|
const csv = import_papaparse2.default.unparse(videos, { header: true });
|
|
19827
20905
|
const filename = `${slug}-videos-${Date.now()}.csv`;
|
|
19828
|
-
const fullPath =
|
|
19829
|
-
await
|
|
20906
|
+
const fullPath = import_node_path11.default.join(outputDir, filename);
|
|
20907
|
+
await import_node_fs7.promises.writeFile(fullPath, csv, "utf8");
|
|
19830
20908
|
return fullPath;
|
|
19831
20909
|
}
|
|
19832
20910
|
async writeForumCSV(forums, seed, outputDir) {
|
|
19833
|
-
await
|
|
20911
|
+
await import_node_fs7.promises.mkdir(outputDir, { recursive: true });
|
|
19834
20912
|
const slug = seed.toLowerCase().replace(/\W+/g, "-").slice(0, 40);
|
|
19835
20913
|
const csv = import_papaparse2.default.unparse(forums, { header: true });
|
|
19836
20914
|
const filename = `${slug}-forums-${Date.now()}.csv`;
|
|
19837
|
-
const fullPath =
|
|
19838
|
-
await
|
|
20915
|
+
const fullPath = import_node_path11.default.join(outputDir, filename);
|
|
20916
|
+
await import_node_fs7.promises.writeFile(fullPath, csv, "utf8");
|
|
19839
20917
|
return fullPath;
|
|
19840
20918
|
}
|
|
19841
20919
|
async writeAIOverviewCSV(citations, text, seed, outputDir) {
|
|
19842
|
-
await
|
|
20920
|
+
await import_node_fs7.promises.mkdir(outputDir, { recursive: true });
|
|
19843
20921
|
const slug = seed.toLowerCase().replace(/\W+/g, "-").slice(0, 40);
|
|
19844
20922
|
const rows = citations.map((c, i) => ({
|
|
19845
20923
|
seed_query: seed,
|
|
@@ -19849,12 +20927,12 @@ var init_OutputSerializer = __esm({
|
|
|
19849
20927
|
}));
|
|
19850
20928
|
const csv = import_papaparse2.default.unparse(rows, { header: true });
|
|
19851
20929
|
const filename = `${slug}-ai-overview-${Date.now()}.csv`;
|
|
19852
|
-
const fullPath =
|
|
19853
|
-
await
|
|
20930
|
+
const fullPath = import_node_path11.default.join(outputDir, filename);
|
|
20931
|
+
await import_node_fs7.promises.writeFile(fullPath, csv, "utf8");
|
|
19854
20932
|
return fullPath;
|
|
19855
20933
|
}
|
|
19856
20934
|
async writeAIModeCSV(citations, text, seed, outputDir) {
|
|
19857
|
-
await
|
|
20935
|
+
await import_node_fs7.promises.mkdir(outputDir, { recursive: true });
|
|
19858
20936
|
const slug = seed.toLowerCase().replace(/\W+/g, "-").slice(0, 40);
|
|
19859
20937
|
const rows = citations.map((c, i) => ({
|
|
19860
20938
|
seed_query: seed,
|
|
@@ -19864,18 +20942,18 @@ var init_OutputSerializer = __esm({
|
|
|
19864
20942
|
}));
|
|
19865
20943
|
const csv = import_papaparse2.default.unparse(rows, { header: true });
|
|
19866
20944
|
const filename = `${slug}-ai-mode-${Date.now()}.csv`;
|
|
19867
|
-
const fullPath =
|
|
19868
|
-
await
|
|
20945
|
+
const fullPath = import_node_path11.default.join(outputDir, filename);
|
|
20946
|
+
await import_node_fs7.promises.writeFile(fullPath, csv, "utf8");
|
|
19869
20947
|
return fullPath;
|
|
19870
20948
|
}
|
|
19871
20949
|
async writeWhatPeopleSayingCSV(cards, seed, outputDir) {
|
|
19872
|
-
await
|
|
20950
|
+
await import_node_fs7.promises.mkdir(outputDir, { recursive: true });
|
|
19873
20951
|
const slug = seed.toLowerCase().replace(/\W+/g, "-").slice(0, 40);
|
|
19874
20952
|
const rows = cards.map((c) => ({ seed_query: seed, ...c }));
|
|
19875
20953
|
const csv = import_papaparse2.default.unparse(rows, { header: true });
|
|
19876
20954
|
const filename = `${slug}-what-people-saying-${Date.now()}.csv`;
|
|
19877
|
-
const fullPath =
|
|
19878
|
-
await
|
|
20955
|
+
const fullPath = import_node_path11.default.join(outputDir, filename);
|
|
20956
|
+
await import_node_fs7.promises.writeFile(fullPath, csv, "utf8");
|
|
19879
20957
|
return fullPath;
|
|
19880
20958
|
}
|
|
19881
20959
|
};
|
|
@@ -20426,11 +21504,11 @@ function isPublicHttpUrl(value) {
|
|
|
20426
21504
|
return false;
|
|
20427
21505
|
}
|
|
20428
21506
|
}
|
|
20429
|
-
var
|
|
21507
|
+
var import_zod26, SerpIntelligenceDeviceValues, SerpIntelligenceProxyModeValues, SerpIntelligenceAttemptOutcomeValues, SerpIntelligenceLocalizationStatusValues, SerpPageSnapshotSourceKindValues, SerpPageFetchStatusValues, SerpPageFetchedViaValues, HostnameSuffixPattern, Ipv4Pattern, SerpIntelligencePublicHttpUrlSchema, SerpIntelligenceCaptureBodySchema, SerpIntelligencePageSnapshotRequestSchema, SerpIntelligencePageSnapshotsBodySchema, SerpIntelligenceAICitationSchema, SerpIntelligenceOrganicResultSchema, SerpIntelligenceLocationEvidenceSchema, SerpIntelligenceHarvestResultSchema, SerpIntelligenceCaptureAttemptSchema, SerpPageSnapshotCaptureSchema, SerpIntelligenceCaptureResponseSchema, SerpIntelligencePageSnapshotsResponseSchema;
|
|
20430
21508
|
var init_schemas4 = __esm({
|
|
20431
21509
|
"src/serp-intelligence/schemas.ts"() {
|
|
20432
21510
|
"use strict";
|
|
20433
|
-
|
|
21511
|
+
import_zod26 = require("zod");
|
|
20434
21512
|
init_schemas3();
|
|
20435
21513
|
SerpIntelligenceDeviceValues = ["desktop", "mobile"];
|
|
20436
21514
|
SerpIntelligenceProxyModeValues = ["location", "configured", "none"];
|
|
@@ -20465,171 +21543,171 @@ var init_schemas4 = __esm({
|
|
|
20465
21543
|
SerpPageFetchedViaValues = ["fetch", "headless", "browser", "mcp"];
|
|
20466
21544
|
HostnameSuffixPattern = /(^|\.)localhost$/i;
|
|
20467
21545
|
Ipv4Pattern = /^\d{1,3}(?:\.\d{1,3}){3}$/;
|
|
20468
|
-
SerpIntelligencePublicHttpUrlSchema =
|
|
20469
|
-
SerpIntelligenceCaptureBodySchema =
|
|
20470
|
-
query:
|
|
20471
|
-
location:
|
|
20472
|
-
gl:
|
|
20473
|
-
hl:
|
|
20474
|
-
device:
|
|
20475
|
-
proxyMode:
|
|
20476
|
-
proxyZip:
|
|
20477
|
-
pages:
|
|
20478
|
-
debug:
|
|
20479
|
-
includePageSnapshots:
|
|
20480
|
-
pageSnapshotLimit:
|
|
21546
|
+
SerpIntelligencePublicHttpUrlSchema = import_zod26.z.string().url().refine(isPublicHttpUrl, "url must be a public HTTP or HTTPS URL");
|
|
21547
|
+
SerpIntelligenceCaptureBodySchema = import_zod26.z.object({
|
|
21548
|
+
query: import_zod26.z.string().trim().min(1, "query is required"),
|
|
21549
|
+
location: import_zod26.z.string().trim().min(1).optional(),
|
|
21550
|
+
gl: import_zod26.z.string().trim().length(2).default("us"),
|
|
21551
|
+
hl: import_zod26.z.string().trim().length(2).default("en"),
|
|
21552
|
+
device: import_zod26.z.enum(SerpIntelligenceDeviceValues).default("desktop"),
|
|
21553
|
+
proxyMode: import_zod26.z.enum(SerpIntelligenceProxyModeValues).default(DEFAULT_PROXY_MODE),
|
|
21554
|
+
proxyZip: import_zod26.z.string().regex(/^\d{5}$/).optional(),
|
|
21555
|
+
pages: import_zod26.z.number().int().min(1).max(2).default(1),
|
|
21556
|
+
debug: import_zod26.z.boolean().default(false),
|
|
21557
|
+
includePageSnapshots: import_zod26.z.boolean().default(false),
|
|
21558
|
+
pageSnapshotLimit: import_zod26.z.number().int().min(0).max(10).default(0)
|
|
20481
21559
|
}).strict();
|
|
20482
|
-
SerpIntelligencePageSnapshotRequestSchema =
|
|
21560
|
+
SerpIntelligencePageSnapshotRequestSchema = import_zod26.z.object({
|
|
20483
21561
|
url: SerpIntelligencePublicHttpUrlSchema,
|
|
20484
|
-
sourceKind:
|
|
20485
|
-
sourcePosition:
|
|
21562
|
+
sourceKind: import_zod26.z.enum(SerpPageSnapshotSourceKindValues).default("configured_target"),
|
|
21563
|
+
sourcePosition: import_zod26.z.number().int().min(1).optional()
|
|
20486
21564
|
}).strict();
|
|
20487
|
-
SerpIntelligencePageSnapshotsBodySchema =
|
|
20488
|
-
urls:
|
|
20489
|
-
targets:
|
|
20490
|
-
maxConcurrency:
|
|
20491
|
-
timeoutMs:
|
|
20492
|
-
debug:
|
|
21565
|
+
SerpIntelligencePageSnapshotsBodySchema = import_zod26.z.object({
|
|
21566
|
+
urls: import_zod26.z.array(SerpIntelligencePublicHttpUrlSchema).min(1).max(25),
|
|
21567
|
+
targets: import_zod26.z.array(SerpIntelligencePageSnapshotRequestSchema).min(1).max(25).optional(),
|
|
21568
|
+
maxConcurrency: import_zod26.z.number().int().min(1).max(5).default(2),
|
|
21569
|
+
timeoutMs: import_zod26.z.number().int().min(1e3).max(6e4).default(15e3),
|
|
21570
|
+
debug: import_zod26.z.boolean().default(false)
|
|
20493
21571
|
}).strict();
|
|
20494
|
-
SerpIntelligenceAICitationSchema =
|
|
20495
|
-
text:
|
|
20496
|
-
href:
|
|
21572
|
+
SerpIntelligenceAICitationSchema = import_zod26.z.object({
|
|
21573
|
+
text: import_zod26.z.string(),
|
|
21574
|
+
href: import_zod26.z.string()
|
|
20497
21575
|
}).strict();
|
|
20498
|
-
SerpIntelligenceOrganicResultSchema =
|
|
20499
|
-
position:
|
|
20500
|
-
title:
|
|
20501
|
-
url:
|
|
20502
|
-
domain:
|
|
20503
|
-
cite:
|
|
20504
|
-
snippet:
|
|
20505
|
-
isRedditStyle:
|
|
20506
|
-
inlineRating:
|
|
20507
|
-
value:
|
|
20508
|
-
count:
|
|
21576
|
+
SerpIntelligenceOrganicResultSchema = import_zod26.z.object({
|
|
21577
|
+
position: import_zod26.z.number().int().min(1),
|
|
21578
|
+
title: import_zod26.z.string(),
|
|
21579
|
+
url: import_zod26.z.string(),
|
|
21580
|
+
domain: import_zod26.z.string(),
|
|
21581
|
+
cite: import_zod26.z.string().nullable(),
|
|
21582
|
+
snippet: import_zod26.z.string().nullable(),
|
|
21583
|
+
isRedditStyle: import_zod26.z.boolean(),
|
|
21584
|
+
inlineRating: import_zod26.z.object({
|
|
21585
|
+
value: import_zod26.z.string(),
|
|
21586
|
+
count: import_zod26.z.string()
|
|
20509
21587
|
}).strict().nullable()
|
|
20510
21588
|
}).strict();
|
|
20511
|
-
SerpIntelligenceLocationEvidenceSchema =
|
|
20512
|
-
status:
|
|
20513
|
-
expected:
|
|
20514
|
-
city:
|
|
20515
|
-
regionCode:
|
|
20516
|
-
canonicalLocation:
|
|
21589
|
+
SerpIntelligenceLocationEvidenceSchema = import_zod26.z.object({
|
|
21590
|
+
status: import_zod26.z.enum(SerpIntelligenceLocalizationStatusValues),
|
|
21591
|
+
expected: import_zod26.z.object({
|
|
21592
|
+
city: import_zod26.z.string(),
|
|
21593
|
+
regionCode: import_zod26.z.string().nullable(),
|
|
21594
|
+
canonicalLocation: import_zod26.z.string()
|
|
20517
21595
|
}).strict().nullable(),
|
|
20518
|
-
candidates:
|
|
20519
|
-
city:
|
|
20520
|
-
regionCode:
|
|
20521
|
-
count:
|
|
20522
|
-
examples:
|
|
21596
|
+
candidates: import_zod26.z.array(import_zod26.z.object({
|
|
21597
|
+
city: import_zod26.z.string(),
|
|
21598
|
+
regionCode: import_zod26.z.string(),
|
|
21599
|
+
count: import_zod26.z.number().int().min(0),
|
|
21600
|
+
examples: import_zod26.z.array(import_zod26.z.string())
|
|
20523
21601
|
}).strict())
|
|
20524
21602
|
}).strict();
|
|
20525
|
-
SerpIntelligenceHarvestResultSchema =
|
|
20526
|
-
seed:
|
|
20527
|
-
location:
|
|
20528
|
-
extractedAt:
|
|
20529
|
-
totalQuestions:
|
|
20530
|
-
surface:
|
|
20531
|
-
aiOverview:
|
|
20532
|
-
detected:
|
|
20533
|
-
text:
|
|
20534
|
-
citations:
|
|
20535
|
-
expanded:
|
|
20536
|
-
fullyExpanded:
|
|
20537
|
-
sections:
|
|
21603
|
+
SerpIntelligenceHarvestResultSchema = import_zod26.z.object({
|
|
21604
|
+
seed: import_zod26.z.string(),
|
|
21605
|
+
location: import_zod26.z.string().nullable(),
|
|
21606
|
+
extractedAt: import_zod26.z.string(),
|
|
21607
|
+
totalQuestions: import_zod26.z.number().int().min(0),
|
|
21608
|
+
surface: import_zod26.z.enum(["web", "aim", "unknown"]),
|
|
21609
|
+
aiOverview: import_zod26.z.object({
|
|
21610
|
+
detected: import_zod26.z.boolean(),
|
|
21611
|
+
text: import_zod26.z.string().nullable(),
|
|
21612
|
+
citations: import_zod26.z.array(SerpIntelligenceAICitationSchema),
|
|
21613
|
+
expanded: import_zod26.z.boolean().optional(),
|
|
21614
|
+
fullyExpanded: import_zod26.z.boolean().optional(),
|
|
21615
|
+
sections: import_zod26.z.array(import_zod26.z.string()).optional()
|
|
20538
21616
|
}).strict(),
|
|
20539
|
-
aiMode:
|
|
20540
|
-
detected:
|
|
20541
|
-
text:
|
|
20542
|
-
citations:
|
|
21617
|
+
aiMode: import_zod26.z.object({
|
|
21618
|
+
detected: import_zod26.z.boolean(),
|
|
21619
|
+
text: import_zod26.z.string().nullable(),
|
|
21620
|
+
citations: import_zod26.z.array(SerpIntelligenceAICitationSchema)
|
|
20543
21621
|
}).strict(),
|
|
20544
|
-
tree:
|
|
20545
|
-
flat:
|
|
20546
|
-
videos:
|
|
20547
|
-
forums:
|
|
20548
|
-
organicResults:
|
|
20549
|
-
localPack:
|
|
20550
|
-
entityIds:
|
|
20551
|
-
entities:
|
|
20552
|
-
name:
|
|
20553
|
-
kgId:
|
|
20554
|
-
cid:
|
|
20555
|
-
gcid:
|
|
21622
|
+
tree: import_zod26.z.array(import_zod26.z.unknown()),
|
|
21623
|
+
flat: import_zod26.z.array(import_zod26.z.unknown()),
|
|
21624
|
+
videos: import_zod26.z.array(import_zod26.z.unknown()),
|
|
21625
|
+
forums: import_zod26.z.array(import_zod26.z.unknown()),
|
|
21626
|
+
organicResults: import_zod26.z.array(SerpIntelligenceOrganicResultSchema),
|
|
21627
|
+
localPack: import_zod26.z.array(import_zod26.z.unknown()),
|
|
21628
|
+
entityIds: import_zod26.z.object({
|
|
21629
|
+
entities: import_zod26.z.array(import_zod26.z.object({
|
|
21630
|
+
name: import_zod26.z.string(),
|
|
21631
|
+
kgId: import_zod26.z.string().nullable(),
|
|
21632
|
+
cid: import_zod26.z.string().nullable(),
|
|
21633
|
+
gcid: import_zod26.z.string().nullable()
|
|
20556
21634
|
}).strict()),
|
|
20557
|
-
kgIds:
|
|
20558
|
-
cids:
|
|
20559
|
-
gcids:
|
|
21635
|
+
kgIds: import_zod26.z.array(import_zod26.z.string()),
|
|
21636
|
+
cids: import_zod26.z.array(import_zod26.z.string()),
|
|
21637
|
+
gcids: import_zod26.z.array(import_zod26.z.string())
|
|
20560
21638
|
}).strict(),
|
|
20561
|
-
stats:
|
|
20562
|
-
seed:
|
|
20563
|
-
totalQuestions:
|
|
20564
|
-
maxDepthReached:
|
|
20565
|
-
durationMs:
|
|
20566
|
-
errorCount:
|
|
21639
|
+
stats: import_zod26.z.object({
|
|
21640
|
+
seed: import_zod26.z.string(),
|
|
21641
|
+
totalQuestions: import_zod26.z.number().int().min(0),
|
|
21642
|
+
maxDepthReached: import_zod26.z.number().int().min(0),
|
|
21643
|
+
durationMs: import_zod26.z.number().min(0),
|
|
21644
|
+
errorCount: import_zod26.z.number().int().min(0)
|
|
20567
21645
|
}).strict(),
|
|
20568
|
-
diagnostics:
|
|
20569
|
-
completionStatus:
|
|
20570
|
-
problem:
|
|
20571
|
-
warnings:
|
|
20572
|
-
debug:
|
|
21646
|
+
diagnostics: import_zod26.z.object({
|
|
21647
|
+
completionStatus: import_zod26.z.enum(["paa_found", "no_paa", "serp_only"]),
|
|
21648
|
+
problem: import_zod26.z.null(),
|
|
21649
|
+
warnings: import_zod26.z.array(import_zod26.z.unknown()).optional(),
|
|
21650
|
+
debug: import_zod26.z.object({
|
|
20573
21651
|
locationEvidence: SerpIntelligenceLocationEvidenceSchema.optional()
|
|
20574
21652
|
}).passthrough().optional()
|
|
20575
21653
|
}).passthrough(),
|
|
20576
|
-
whatPeopleSaying:
|
|
21654
|
+
whatPeopleSaying: import_zod26.z.array(import_zod26.z.unknown())
|
|
20577
21655
|
}).strict();
|
|
20578
|
-
SerpIntelligenceCaptureAttemptSchema =
|
|
20579
|
-
attemptNumber:
|
|
20580
|
-
outcome:
|
|
20581
|
-
startedAt:
|
|
20582
|
-
completedAt:
|
|
20583
|
-
durationMs:
|
|
20584
|
-
problemCode:
|
|
20585
|
-
message:
|
|
20586
|
-
kernelSessionId:
|
|
20587
|
-
cleanupSucceeded:
|
|
21656
|
+
SerpIntelligenceCaptureAttemptSchema = import_zod26.z.object({
|
|
21657
|
+
attemptNumber: import_zod26.z.number().int().min(1),
|
|
21658
|
+
outcome: import_zod26.z.enum(SerpIntelligenceAttemptOutcomeValues),
|
|
21659
|
+
startedAt: import_zod26.z.string().optional(),
|
|
21660
|
+
completedAt: import_zod26.z.string().optional(),
|
|
21661
|
+
durationMs: import_zod26.z.number().min(0).optional(),
|
|
21662
|
+
problemCode: import_zod26.z.string().optional(),
|
|
21663
|
+
message: import_zod26.z.string().optional(),
|
|
21664
|
+
kernelSessionId: import_zod26.z.string().nullable().optional(),
|
|
21665
|
+
cleanupSucceeded: import_zod26.z.boolean().nullable().optional()
|
|
20588
21666
|
}).strict();
|
|
20589
|
-
SerpPageSnapshotCaptureSchema =
|
|
21667
|
+
SerpPageSnapshotCaptureSchema = import_zod26.z.object({
|
|
20590
21668
|
url: SerpIntelligencePublicHttpUrlSchema,
|
|
20591
21669
|
requestedUrl: SerpIntelligencePublicHttpUrlSchema,
|
|
20592
21670
|
finalUrl: SerpIntelligencePublicHttpUrlSchema.nullable(),
|
|
20593
|
-
sourceKind:
|
|
20594
|
-
sourcePosition:
|
|
20595
|
-
status:
|
|
20596
|
-
fetchedVia:
|
|
20597
|
-
httpStatus:
|
|
20598
|
-
contentType:
|
|
20599
|
-
title:
|
|
21671
|
+
sourceKind: import_zod26.z.enum(SerpPageSnapshotSourceKindValues),
|
|
21672
|
+
sourcePosition: import_zod26.z.number().int().min(1).nullable(),
|
|
21673
|
+
status: import_zod26.z.enum(SerpPageFetchStatusValues),
|
|
21674
|
+
fetchedVia: import_zod26.z.enum(SerpPageFetchedViaValues).nullable(),
|
|
21675
|
+
httpStatus: import_zod26.z.number().int().min(100).max(599).nullable(),
|
|
21676
|
+
contentType: import_zod26.z.string().nullable(),
|
|
21677
|
+
title: import_zod26.z.string().nullable(),
|
|
20600
21678
|
canonicalUrl: SerpIntelligencePublicHttpUrlSchema.nullable(),
|
|
20601
|
-
metaDescription:
|
|
20602
|
-
headings:
|
|
20603
|
-
level:
|
|
20604
|
-
text:
|
|
21679
|
+
metaDescription: import_zod26.z.string().nullable(),
|
|
21680
|
+
headings: import_zod26.z.array(import_zod26.z.object({
|
|
21681
|
+
level: import_zod26.z.number().int().min(1).max(6),
|
|
21682
|
+
text: import_zod26.z.string()
|
|
20605
21683
|
}).strict()).default([]),
|
|
20606
|
-
artifact:
|
|
20607
|
-
htmlBlobUrl:
|
|
20608
|
-
textBlobUrl:
|
|
20609
|
-
markdownBlobUrl:
|
|
20610
|
-
screenshotBlobUrl:
|
|
20611
|
-
contentSha256:
|
|
20612
|
-
capturedAt:
|
|
21684
|
+
artifact: import_zod26.z.object({
|
|
21685
|
+
htmlBlobUrl: import_zod26.z.string().url().nullable(),
|
|
21686
|
+
textBlobUrl: import_zod26.z.string().url().nullable(),
|
|
21687
|
+
markdownBlobUrl: import_zod26.z.string().url().nullable(),
|
|
21688
|
+
screenshotBlobUrl: import_zod26.z.string().url().nullable(),
|
|
21689
|
+
contentSha256: import_zod26.z.string().nullable(),
|
|
21690
|
+
capturedAt: import_zod26.z.string().nullable()
|
|
20613
21691
|
}).strict(),
|
|
20614
|
-
error:
|
|
20615
|
-
code:
|
|
20616
|
-
message:
|
|
21692
|
+
error: import_zod26.z.object({
|
|
21693
|
+
code: import_zod26.z.string(),
|
|
21694
|
+
message: import_zod26.z.string()
|
|
20617
21695
|
}).strict().nullable()
|
|
20618
21696
|
}).strict();
|
|
20619
|
-
SerpIntelligenceCaptureResponseSchema =
|
|
21697
|
+
SerpIntelligenceCaptureResponseSchema = import_zod26.z.object({
|
|
20620
21698
|
harvestResult: SerpIntelligenceHarvestResultSchema,
|
|
20621
|
-
attempts:
|
|
21699
|
+
attempts: import_zod26.z.array(SerpIntelligenceCaptureAttemptSchema),
|
|
20622
21700
|
locationEvidence: SerpIntelligenceLocationEvidenceSchema.nullable(),
|
|
20623
|
-
pageSnapshotArtifacts:
|
|
20624
|
-
billing:
|
|
20625
|
-
creditsUsed:
|
|
20626
|
-
requestId:
|
|
20627
|
-
jobId:
|
|
21701
|
+
pageSnapshotArtifacts: import_zod26.z.array(SerpPageSnapshotCaptureSchema),
|
|
21702
|
+
billing: import_zod26.z.object({
|
|
21703
|
+
creditsUsed: import_zod26.z.number().min(0).optional(),
|
|
21704
|
+
requestId: import_zod26.z.string().optional(),
|
|
21705
|
+
jobId: import_zod26.z.string().optional()
|
|
20628
21706
|
}).strict().optional()
|
|
20629
21707
|
}).strict();
|
|
20630
|
-
SerpIntelligencePageSnapshotsResponseSchema =
|
|
20631
|
-
pageSnapshotArtifacts:
|
|
20632
|
-
attempts:
|
|
21708
|
+
SerpIntelligencePageSnapshotsResponseSchema = import_zod26.z.object({
|
|
21709
|
+
pageSnapshotArtifacts: import_zod26.z.array(SerpPageSnapshotCaptureSchema),
|
|
21710
|
+
attempts: import_zod26.z.array(SerpIntelligenceCaptureAttemptSchema).default([])
|
|
20633
21711
|
}).strict();
|
|
20634
21712
|
}
|
|
20635
21713
|
});
|
|
@@ -20853,11 +21931,11 @@ function pageSnapshotTargetsFromBody(body) {
|
|
|
20853
21931
|
sourcePosition: null
|
|
20854
21932
|
}));
|
|
20855
21933
|
}
|
|
20856
|
-
var
|
|
21934
|
+
var import_hono9, SERP_INTELLIGENCE_RATE_LIMIT, SERP_INTELLIGENCE_RATE_WINDOW_SECONDS, POST_CAPTURE_ROUTE_LABEL, POST_PAGE_SNAPSHOTS_ROUTE_LABEL, serpIntelligenceApp;
|
|
20857
21935
|
var init_serp_intelligence_routes = __esm({
|
|
20858
21936
|
"src/api/serp-intelligence-routes.ts"() {
|
|
20859
21937
|
"use strict";
|
|
20860
|
-
|
|
21938
|
+
import_hono9 = require("hono");
|
|
20861
21939
|
init_browser_service_env();
|
|
20862
21940
|
init_page_snapshot_extractor();
|
|
20863
21941
|
init_serp_capture_service();
|
|
@@ -20870,7 +21948,7 @@ var init_serp_intelligence_routes = __esm({
|
|
|
20870
21948
|
SERP_INTELLIGENCE_RATE_WINDOW_SECONDS = 60;
|
|
20871
21949
|
POST_CAPTURE_ROUTE_LABEL = "POST /capture";
|
|
20872
21950
|
POST_PAGE_SNAPSHOTS_ROUTE_LABEL = "POST /page-snapshots";
|
|
20873
|
-
serpIntelligenceApp = new
|
|
21951
|
+
serpIntelligenceApp = new import_hono9.Hono();
|
|
20874
21952
|
serpIntelligenceApp.use("*", createApiKeyAuth());
|
|
20875
21953
|
serpIntelligenceApp.post("/capture", async (c) => {
|
|
20876
21954
|
void POST_CAPTURE_ROUTE_LABEL;
|
|
@@ -21002,139 +22080,159 @@ var PACKAGE_VERSION;
|
|
|
21002
22080
|
var init_version = __esm({
|
|
21003
22081
|
"src/version.ts"() {
|
|
21004
22082
|
"use strict";
|
|
21005
|
-
PACKAGE_VERSION = "0.3.
|
|
22083
|
+
PACKAGE_VERSION = "0.3.3";
|
|
21006
22084
|
}
|
|
21007
22085
|
});
|
|
21008
22086
|
|
|
21009
22087
|
// src/mcp/mcp-tool-schemas.ts
|
|
21010
|
-
var
|
|
22088
|
+
var import_zod27, HarvestPaaInputSchema, ExtractUrlInputSchema, MapSiteUrlsInputSchema, ExtractSiteInputSchema, YoutubeHarvestInputSchema, YoutubeTranscribeInputSchema, FacebookPageIntelInputSchema, FacebookAdSearchInputSchema, FacebookAdTranscribeInputSchema, FacebookVideoTranscribeInputSchema, InstagramProfileContentInputSchema, InstagramMediaDownloadInputSchema, MapsPlaceIntelInputSchema, MapsSearchInputSchema, DirectoryWorkflowInputSchema, RankTrackerModeSchema, RankTrackerBlueprintInputSchema, NullableString, MapsSearchAttemptOutput, MapsSearchOutputSchema, DirectoryMapsBusinessOutput, DirectoryWorkflowOutputSchema, RankTrackerToolPlanOutput, RankTrackerTableOutput, RankTrackerCronJobOutput, RankTrackerBlueprintOutputSchema, OrganicResultOutput, AiOverviewOutput, EntityIdsOutput, HarvestPaaOutputSchema, SearchSerpOutputSchema, ExtractUrlOutputSchema, ExtractSiteOutputSchema, MapsPlaceIntelOutputSchema, CreditsInfoOutputSchema, MapSiteUrlsOutputSchema, YoutubeHarvestOutputSchema, FacebookAdSearchOutputSchema, FacebookPageIntelOutputSchema, FacebookVideoTranscribeOutputSchema, TranscriptChunkOutput, InstagramBrowserOutput, InstagramPaginationOutput, InstagramProfileContentOutputSchema, InstagramMediaTrackOutput, InstagramDownloadOutput, InstagramMediaDownloadOutputSchema, YoutubeTranscribeOutputSchema, FacebookAdTranscribeOutputSchema, CaptureSerpSnapshotOutputSchema, CaptureSerpPageSnapshotsOutputSchema, CreditsInfoInputSchema, WorkflowIdSchema2, WorkflowListInputSchema, WorkflowSuggestInputSchema, WorkflowRunInputSchema, WorkflowStepInputSchema, WorkflowStatusInputSchema, WorkflowArtifactReadInputSchema, WorkflowRecipeOutput, WorkflowDefinitionOutput, WorkflowArtifactOutput, WorkflowListOutputSchema, WorkflowSuggestOutputSchema, WorkflowRunOutputSchema, WorkflowStepOutputSchema, WorkflowStatusOutputSchema, WorkflowArtifactReadOutputSchema, SearchSerpInputSchema, CaptureSerpSnapshotInputSchema, ScreenshotInputSchema, CaptureSerpPageSnapshotsInputSchema;
|
|
21011
22089
|
var init_mcp_tool_schemas = __esm({
|
|
21012
22090
|
"src/mcp/mcp-tool-schemas.ts"() {
|
|
21013
22091
|
"use strict";
|
|
21014
|
-
|
|
22092
|
+
import_zod27 = require("zod");
|
|
21015
22093
|
init_schemas3();
|
|
21016
22094
|
HarvestPaaInputSchema = {
|
|
21017
|
-
query:
|
|
21018
|
-
location:
|
|
21019
|
-
maxQuestions:
|
|
21020
|
-
gl:
|
|
21021
|
-
hl:
|
|
21022
|
-
device:
|
|
21023
|
-
proxyMode:
|
|
21024
|
-
proxyZip:
|
|
21025
|
-
debug:
|
|
22095
|
+
query: import_zod27.z.string().min(1).describe('Core search topic only. If the user says "best hvac company in Denver CO", use query="best hvac company" and location="Denver, CO". Do not include the location in query when it can be separated.'),
|
|
22096
|
+
location: import_zod27.z.string().optional().describe('City, region, or country for geo-targeted results, inferred from the user request when present, e.g. "Denver, CO", "Tokyo, Japan", "London, UK".'),
|
|
22097
|
+
maxQuestions: import_zod27.z.number().int().min(1).max(200).default(30).describe("Number of PAA questions to extract. Default 30. Maximum 200. Use 10 for quick probes, 30 for normal research, 100-200 when the user asks for everything/full/deep research. Larger harvests get a longer server time budget (151-200 questions \u2192 up to 280s). Credits are charged by extracted question; unused request hold is refunded."),
|
|
22098
|
+
gl: import_zod27.z.string().length(2).default("us").describe("Google country code inferred from location or user language. Examples: United States us, United Kingdom gb, Japan jp, Canada ca, Australia au."),
|
|
22099
|
+
hl: import_zod27.z.string().default("en").describe("Google interface/content language inferred from the user request. Use en unless the user asks for another language or locale."),
|
|
22100
|
+
device: import_zod27.z.enum(["desktop", "mobile"]).default("desktop").describe("SERP device context. Use desktop by default; use mobile only when the user asks for mobile rankings."),
|
|
22101
|
+
proxyMode: import_zod27.z.enum(["location", "configured", "none"]).default(DEFAULT_PROXY_MODE).describe("Proxy targeting mode. Default configured uses the service proxy without city/ZIP targeting for the highest general success rate. Use location only when the user explicitly needs city/ZIP-targeted residential proxy evidence. Use none only for direct-network debugging."),
|
|
22102
|
+
proxyZip: import_zod27.z.string().regex(/^\d{5}$/).optional().describe("Optional US ZIP override for residential location proxy targeting. Use only with proxyMode location when the user gives a specific ZIP or city-center targeting needs to be forced."),
|
|
22103
|
+
debug: import_zod27.z.boolean().default(false).describe("Include sanitized browser/session/location diagnostics in the response. Use true when debugging localization, CAPTCHA, or proxy behavior.")
|
|
21026
22104
|
};
|
|
21027
22105
|
ExtractUrlInputSchema = {
|
|
21028
|
-
url:
|
|
21029
|
-
screenshot:
|
|
21030
|
-
screenshotDevice:
|
|
21031
|
-
extractBranding:
|
|
21032
|
-
downloadMedia:
|
|
21033
|
-
mediaTypes:
|
|
21034
|
-
allowLocal:
|
|
22106
|
+
url: import_zod27.z.string().url().describe("Public http/https URL to extract. Use this when the user provides one specific page URL."),
|
|
22107
|
+
screenshot: import_zod27.z.boolean().default(false).describe("Also capture a full-page screenshot of the URL. Saved to ~/Downloads/mcp-scraper/screenshots/ and returned inline. Use when the user asks to see or capture the page visually."),
|
|
22108
|
+
screenshotDevice: import_zod27.z.enum(["desktop", "mobile"]).default("desktop").describe("Viewport for screenshot. desktop = 1440\xD7900. mobile = 390\xD7844. Default desktop."),
|
|
22109
|
+
extractBranding: import_zod27.z.boolean().default(false).describe("Extract brand colors, fonts, logo, and favicon using a rendered browser session. Returns colorScheme (light/dark), colors (primary/accent/background/text/heading as hex), fonts (heading/body family names), and assets (logo URL, favicon URL). Use when the user asks about brand colors, site theme, or brand assets."),
|
|
22110
|
+
downloadMedia: import_zod27.z.boolean().default(false).describe("Extract and download all page media (images, video, audio) to ~/Downloads/mcp-scraper/media/. Ad networks, tracking pixels, and noise URLs are filtered automatically. Use when the user asks to download or harvest assets from a page."),
|
|
22111
|
+
mediaTypes: import_zod27.z.array(import_zod27.z.enum(["image", "video", "audio"])).default(["image", "video", "audio"]).describe("Which media types to download. Default all three."),
|
|
22112
|
+
allowLocal: import_zod27.z.boolean().default(false).describe("Allow localhost and private-network URLs. For local development only.")
|
|
21035
22113
|
};
|
|
21036
22114
|
MapSiteUrlsInputSchema = {
|
|
21037
|
-
url:
|
|
21038
|
-
maxUrls:
|
|
22115
|
+
url: import_zod27.z.string().url().describe("Public website URL or domain to crawl for internal URLs. Use before extract_site when the user asks to audit/map/crawl a site."),
|
|
22116
|
+
maxUrls: import_zod27.z.number().int().min(1).max(500).optional().describe("Maximum URLs to discover. Use 100 for normal maps, higher when the user asks for a full inventory.")
|
|
21039
22117
|
};
|
|
21040
22118
|
ExtractSiteInputSchema = {
|
|
21041
|
-
url:
|
|
21042
|
-
maxPages:
|
|
22119
|
+
url: import_zod27.z.string().url().describe("Public website URL or domain to extract across multiple pages. Use when the user asks for a site audit, website crawl, or full-site content/schema extraction."),
|
|
22120
|
+
maxPages: import_zod27.z.number().int().min(1).max(50).optional().describe("Maximum pages to extract. Use 50 when the user asks for full results or a complete crawl within MCP limits.")
|
|
21043
22121
|
};
|
|
21044
22122
|
YoutubeHarvestInputSchema = {
|
|
21045
|
-
mode:
|
|
21046
|
-
query:
|
|
21047
|
-
channelHandle:
|
|
21048
|
-
maxVideos:
|
|
22123
|
+
mode: import_zod27.z.enum(["search", "channel"]).describe("Use search for topic/keyword requests. Use channel when the user provides @handle, channel ID, or channel URL."),
|
|
22124
|
+
query: import_zod27.z.string().optional().describe("Required when mode is search. The YouTube search topic in the user\u2019s words."),
|
|
22125
|
+
channelHandle: import_zod27.z.string().optional().describe("YouTube channel handle, channel ID, or URL. Examples: @mkbhd, UC..., https://youtube.com/@mkbhd."),
|
|
22126
|
+
maxVideos: import_zod27.z.number().int().min(1).max(500).default(50).describe("Number of videos to return. Default 50, maximum 500. Use 10-25 for quick topic discovery, 50 for normal channel/search harvests, and larger values only when the user asks for full channel/history because large responses consume more context.")
|
|
21049
22127
|
};
|
|
21050
22128
|
YoutubeTranscribeInputSchema = {
|
|
21051
|
-
videoId:
|
|
21052
|
-
url:
|
|
22129
|
+
videoId: import_zod27.z.string().min(1).optional().describe("YouTube video ID, e.g. dQw4w9WgXcQ. Use only an ID returned by youtube_harvest or visible in a YouTube URL; do not invent one."),
|
|
22130
|
+
url: import_zod27.z.string().url().optional().describe("Full YouTube URL, e.g. https://www.youtube.com/watch?v=dQw4w9WgXcQ or https://youtu.be/dQw4w9WgXcQ. Use this when the user pasted a URL instead of an ID. Provide videoId or url.")
|
|
21053
22131
|
};
|
|
21054
22132
|
FacebookPageIntelInputSchema = {
|
|
21055
|
-
pageId:
|
|
21056
|
-
libraryId:
|
|
21057
|
-
query:
|
|
21058
|
-
maxAds:
|
|
21059
|
-
country:
|
|
22133
|
+
pageId: import_zod27.z.string().optional().describe("Facebook advertiser/page ID. Use only a pageId returned by facebook_ad_search/facebook_page_intel or copied from Facebook Ad Library; do not construct one yourself."),
|
|
22134
|
+
libraryId: import_zod27.z.string().optional().describe("Facebook Ad Library archive ID for a known ad or advertiser sample. Use a libraryId returned by facebook_ad_search, or a libraryId/adArchiveId visible in Ad Library."),
|
|
22135
|
+
query: import_zod27.z.string().optional().describe("Advertiser or brand name when pageId/libraryId is not known. One of pageId, libraryId, or query is required."),
|
|
22136
|
+
maxAds: import_zod27.z.number().int().min(1).max(200).default(50).describe("Maximum ads to inspect. Default 50, maximum 200. Prefer 25-50 for focused advertiser scans; use 100-200 only when the user asks for a broad ad archive sweep."),
|
|
22137
|
+
country: import_zod27.z.string().length(2).default("US").describe("Two-letter Ad Library country code. Default US. Examples: US, CA, GB, AU. Infer from the user request when they name a country.")
|
|
21060
22138
|
};
|
|
21061
22139
|
FacebookAdSearchInputSchema = {
|
|
21062
|
-
query:
|
|
21063
|
-
country:
|
|
21064
|
-
maxResults:
|
|
22140
|
+
query: import_zod27.z.string().min(1).describe("Advertiser, brand, competitor, niche, or keyword to search in Facebook Ad Library."),
|
|
22141
|
+
country: import_zod27.z.string().length(2).default("US").describe("Two-letter Ad Library country code. Default US. Examples: US, CA, GB, AU."),
|
|
22142
|
+
maxResults: import_zod27.z.number().int().min(1).max(20).default(10).describe("Maximum advertisers to return. Default 10, maximum 20. Prefer tighter search terms over maxing this out.")
|
|
21065
22143
|
};
|
|
21066
22144
|
FacebookAdTranscribeInputSchema = {
|
|
21067
|
-
videoUrl:
|
|
22145
|
+
videoUrl: import_zod27.z.string().url().describe("Direct Facebook CDN video URL from a facebook_page_intel ad result. Do not pass a public Facebook reel/post/share URL here; use facebook_video_transcribe for organic Facebook URLs.")
|
|
21068
22146
|
};
|
|
21069
22147
|
FacebookVideoTranscribeInputSchema = {
|
|
21070
|
-
url:
|
|
21071
|
-
quality:
|
|
22148
|
+
url: import_zod27.z.string().url().describe("Organic Facebook reel, video, watch, post, or share URL from facebook.com, m.facebook.com, or fb.watch. The tool renders the page, extracts the best matching public Facebook CDN MP4 URL, then transcribes it. Use this when the user pastes a normal Facebook video page URL and asks for the transcript or downloadable MP4."),
|
|
22149
|
+
quality: import_zod27.z.enum(["best", "hd", "sd"]).default("best").describe("Preferred progressive MP4 quality. Use best by default; hd prefers the highest HD progressive URL; sd forces the SD URL.")
|
|
22150
|
+
};
|
|
22151
|
+
InstagramProfileContentInputSchema = {
|
|
22152
|
+
handle: import_zod27.z.string().min(1).optional().describe("Instagram handle, with or without @. Provide handle or url. Use only public profiles unless a configured browser profile is available server-side."),
|
|
22153
|
+
url: import_zod27.z.string().url().optional().describe("Instagram profile URL, e.g. https://www.instagram.com/nasaartemis/. Provide handle or url."),
|
|
22154
|
+
maxItems: import_zod27.z.number().int().min(1).max(500).default(50).describe("Maximum profile grid post/reel/tv URLs to collect. Default 50, maximum 500. Use higher values with an authenticated browser profile when the user asks for a fuller archive."),
|
|
22155
|
+
maxScrolls: import_zod27.z.number().int().min(0).max(100).default(10).describe("Maximum pagination scroll attempts. Default 10, maximum 100. Increase when using authenticated Chromium and the profile has a long history."),
|
|
22156
|
+
scrollDelayMs: import_zod27.z.number().int().min(250).max(5e3).default(1200).describe("Delay after each pagination scroll before collecting newly loaded links. Default 1200ms. Increase to 2000-3000ms when Instagram loads slowly."),
|
|
22157
|
+
stableScrollLimit: import_zod27.z.number().int().min(1).max(10).default(4).describe("Stop after this many consecutive scrolls with no new links or scroll progress. Default 4."),
|
|
22158
|
+
browserMode: import_zod27.z.enum(["auto", "hosted", "local"]).default("auto").describe("Browser backend. auto uses the hosted browser unless local mode is configured or browserProfile is passed. local uses an imported/authenticated Chrome profile from browser_profile_import/sync. hosted forces the remote browser path."),
|
|
22159
|
+
browserProfile: import_zod27.z.string().min(1).optional().describe("Managed local browser profile name created by browser_profile_import/sync. Passing this opts the Instagram tool into local authenticated Chromium for this call.")
|
|
22160
|
+
};
|
|
22161
|
+
InstagramMediaDownloadInputSchema = {
|
|
22162
|
+
url: import_zod27.z.string().url().describe("Instagram post, reel, or tv URL, e.g. https://www.instagram.com/reel/SHORTCODE/. The tool renders the page, extracts text, image metadata, and Instagram CDN media tracks."),
|
|
22163
|
+
mediaTypes: import_zod27.z.array(import_zod27.z.enum(["image", "video", "audio"])).default(["image", "video", "audio"]).describe("Which media types to download when downloadMedia is true. Reels commonly expose separate video-only and audio-only MP4 tracks."),
|
|
22164
|
+
downloadMedia: import_zod27.z.boolean().default(true).describe("Download extracted text/media files to the MCP Scraper output directory when the API server can write files. Always returns extracted media URLs even when false."),
|
|
22165
|
+
downloadAllTracks: import_zod27.z.boolean().default(false).describe("Download every captured Instagram MP4 track instead of only the selected best video and audio tracks. Use false by default to avoid duplicate bitrates."),
|
|
22166
|
+
includeTranscript: import_zod27.z.boolean().default(false).describe("Transcribe the selected audio track when available. This adds transcription cost and may take longer."),
|
|
22167
|
+
mux: import_zod27.z.boolean().default(true).describe("When video and audio tracks are downloaded separately, try to mux them into a single MP4 if ffmpeg is available. Returns separate tracks when muxing is unavailable."),
|
|
22168
|
+
browserMode: import_zod27.z.enum(["auto", "hosted", "local"]).default("auto").describe("Browser backend. auto uses the hosted browser unless local mode is configured or browserProfile is passed. local uses an imported/authenticated Chrome profile from browser_profile_import/sync. hosted forces the remote browser path."),
|
|
22169
|
+
browserProfile: import_zod27.z.string().min(1).optional().describe("Managed local browser profile name created by browser_profile_import/sync. Passing this opts the Instagram tool into local authenticated Chromium for this call.")
|
|
21072
22170
|
};
|
|
21073
22171
|
MapsPlaceIntelInputSchema = {
|
|
21074
|
-
businessName:
|
|
21075
|
-
location:
|
|
21076
|
-
gl:
|
|
21077
|
-
hl:
|
|
21078
|
-
includeReviews:
|
|
21079
|
-
maxReviews:
|
|
22172
|
+
businessName: import_zod27.z.string().min(1).describe('Business name only. If user says "Elite Roofing Denver CO", use businessName="Elite Roofing" and location="Denver, CO".'),
|
|
22173
|
+
location: import_zod27.z.string().min(1).describe('City/region/country where the business should be searched, e.g. "Denver, CO". Infer from the user request when possible.'),
|
|
22174
|
+
gl: import_zod27.z.string().length(2).default("us").describe("Google country code inferred from location."),
|
|
22175
|
+
hl: import_zod27.z.string().length(2).default("en").describe("Language inferred from user request."),
|
|
22176
|
+
includeReviews: import_zod27.z.boolean().default(false).describe("Whether to fetch individual review cards. Use true when the user asks for reviews, customer pain, complaints, praise, themes, or review evidence."),
|
|
22177
|
+
maxReviews: import_zod27.z.number().int().min(1).max(500).default(50).describe("Max review cards to return when includeReviews is true. Default 50, maximum 500. Use 50 for normal review analysis and larger values only for deep review mining.")
|
|
21080
22178
|
};
|
|
21081
22179
|
MapsSearchInputSchema = {
|
|
21082
|
-
query:
|
|
21083
|
-
location:
|
|
21084
|
-
gl:
|
|
21085
|
-
hl:
|
|
21086
|
-
maxResults:
|
|
21087
|
-
proxyMode:
|
|
21088
|
-
proxyZip:
|
|
21089
|
-
debug:
|
|
22180
|
+
query: import_zod27.z.string().min(1).describe('Business category, niche, keyword, or search term. If the user says "roofers in Denver CO", use query="roofers" and location="Denver, CO". Do not put the location here when it can be separated.'),
|
|
22181
|
+
location: import_zod27.z.string().optional().describe('City, region, country, or service area for the Maps search, e.g. "Denver, CO". Infer from the user request when present.'),
|
|
22182
|
+
gl: import_zod27.z.string().length(2).default("us").describe("Google country code inferred from location."),
|
|
22183
|
+
hl: import_zod27.z.string().length(2).default("en").describe("Language inferred from user request."),
|
|
22184
|
+
maxResults: import_zod27.z.number().int().min(1).max(50).default(10).describe("Number of Google Maps business/profile candidates to return. Default 10. Maximum 50. Use 10 unless the user asks for more."),
|
|
22185
|
+
proxyMode: import_zod27.z.enum(["location", "configured", "none"]).default(DEFAULT_MAPS_PROXY_MODE).describe("Proxy targeting mode. Maps defaults to location so local market searches get city/state residential proxy targeting and rotation. Use configured to force the service proxy without city/ZIP targeting, and none only for local direct-network debugging."),
|
|
22186
|
+
proxyZip: import_zod27.z.string().regex(/^\d{5}$/).optional().describe("Optional US ZIP override for residential location proxy targeting. Use when the user gives a specific ZIP or city-center ZIP."),
|
|
22187
|
+
debug: import_zod27.z.boolean().default(false).describe("Include sanitized browser/proxy diagnostics when debugging Maps localization, CAPTCHA, or proxy behavior.")
|
|
21090
22188
|
};
|
|
21091
22189
|
DirectoryWorkflowInputSchema = {
|
|
21092
|
-
query:
|
|
21093
|
-
state:
|
|
21094
|
-
minPopulation:
|
|
21095
|
-
populationYear:
|
|
21096
|
-
maxCities:
|
|
21097
|
-
maxResultsPerCity:
|
|
21098
|
-
concurrency:
|
|
21099
|
-
includeZipGroups:
|
|
21100
|
-
usZipsCsvPath:
|
|
21101
|
-
saveCsv:
|
|
21102
|
-
proxyMode:
|
|
21103
|
-
proxyZip:
|
|
21104
|
-
debug:
|
|
22190
|
+
query: import_zod27.z.string().min(1).describe("Business category, niche, or keyword to search on Google Maps for every selected market, e.g. roofers, dentists, med spas. Do not include the city here."),
|
|
22191
|
+
state: import_zod27.z.string().min(2).default("TN").describe("US state abbreviation or state name used to select Census places, e.g. TN or Tennessee."),
|
|
22192
|
+
minPopulation: import_zod27.z.number().int().min(0).default(1e5).describe('Minimum Census place population for market selection. Use 100000 for "cities above 100k population".'),
|
|
22193
|
+
populationYear: import_zod27.z.number().int().min(2020).max(2025).default(2025).describe("Census population estimate year from the 2020-2025 Population Estimates Program city/place dataset."),
|
|
22194
|
+
maxCities: import_zod27.z.number().int().min(1).max(100).default(25).describe("Maximum number of markets to process after sorting by population descending."),
|
|
22195
|
+
maxResultsPerCity: import_zod27.z.number().int().min(1).max(50).default(50).describe("Google Maps business/profile candidates to collect for each city. Maximum 50."),
|
|
22196
|
+
concurrency: import_zod27.z.number().int().min(1).max(5).default(5).describe("How many city Maps searches to run in parallel. Use 5 for broad directory batches unless debugging."),
|
|
22197
|
+
includeZipGroups: import_zod27.z.boolean().default(true).describe("Attach ZIP groups from a configured US ZIPS CSV when available. Set MCP_SCRAPER_USZIPS_CSV_PATH on the API server or pass usZipsCsvPath in local/test mode."),
|
|
22198
|
+
usZipsCsvPath: import_zod27.z.string().optional().describe("Local/test-only path to a US ZIPS CSV with state_abbr, zipcode, county, city columns, such as Lead Magician tools/analytics/data/uszips.csv. Deployed APIs should use MCP_SCRAPER_USZIPS_CSV_PATH instead."),
|
|
22199
|
+
saveCsv: import_zod27.z.boolean().default(true).describe("Save a directory-ready CSV to the MCP Scraper output directory and return its path. CSV rows include source_location, result_position, business_name, review_stars, review_count, category, address, phone, hours_status, website_url, directions_url, place_url, CID fields, population, and ZIP groups."),
|
|
22200
|
+
proxyMode: import_zod27.z.enum(["location", "configured", "none"]).default(DEFAULT_MAPS_PROXY_MODE).describe("Proxy targeting mode for every city Maps search. Maps workflows default to location so each city can use city/state or ZIP-group residential proxy targeting. Use configured to force the service proxy without city/ZIP targeting, and none only for local direct-network debugging."),
|
|
22201
|
+
proxyZip: import_zod27.z.string().regex(/^\d{5}$/).optional().describe("Optional ZIP override for proxy targeting. Normally omit it so each city can use its ZIP group or city/state location."),
|
|
22202
|
+
debug: import_zod27.z.boolean().default(false).describe("Include sanitized browser/proxy diagnostics in each Maps browser session when supported.")
|
|
21105
22203
|
};
|
|
21106
|
-
RankTrackerModeSchema =
|
|
22204
|
+
RankTrackerModeSchema = import_zod27.z.enum(["maps", "organic", "ai_overview", "paa"]);
|
|
21107
22205
|
RankTrackerBlueprintInputSchema = {
|
|
21108
|
-
projectName:
|
|
21109
|
-
targetDomain:
|
|
21110
|
-
targetBusinessName:
|
|
21111
|
-
trackingModes:
|
|
21112
|
-
keywords:
|
|
21113
|
-
locations:
|
|
21114
|
-
competitors:
|
|
21115
|
-
database:
|
|
21116
|
-
scheduleCadence:
|
|
21117
|
-
customCron:
|
|
21118
|
-
timezone:
|
|
21119
|
-
includeCron:
|
|
21120
|
-
includeDashboard:
|
|
21121
|
-
includeAlerts:
|
|
21122
|
-
notes:
|
|
22206
|
+
projectName: import_zod27.z.string().min(1).optional().describe("Optional name for the rank tracker project, client, or campaign."),
|
|
22207
|
+
targetDomain: import_zod27.z.string().min(1).optional().describe("Primary domain to track in organic results, AI Overview citations, and PAA sources, e.g. example.com."),
|
|
22208
|
+
targetBusinessName: import_zod27.z.string().min(1).optional().describe("Primary Google Business Profile or brand/business name to match in Maps results. Required for reliable Maps rank tracking."),
|
|
22209
|
+
trackingModes: import_zod27.z.array(RankTrackerModeSchema).min(1).max(4).default(["maps", "organic", "ai_overview", "paa"]).describe("Rank tracker surfaces to build. maps uses directory_workflow/maps_search. organic uses search_serp. ai_overview uses search_serp/harvest_paa. paa uses harvest_paa source presence."),
|
|
22210
|
+
keywords: import_zod27.z.array(import_zod27.z.string().min(1)).max(200).default([]).describe("Seed keywords or service queries to track. Leave empty when the downstream AI should create the keyword table from user input."),
|
|
22211
|
+
locations: import_zod27.z.array(import_zod27.z.string().min(1)).max(100).default([]).describe("Markets, cities, ZIPs, or service areas to track. Use city/state strings like Denver, CO for localized SERP and Maps checks."),
|
|
22212
|
+
competitors: import_zod27.z.array(import_zod27.z.string().min(1)).max(100).default([]).describe("Optional competitor domains or business names to persist as comparison targets."),
|
|
22213
|
+
database: import_zod27.z.enum(["postgres", "neon", "supabase", "sqlite", "mysql"]).default("postgres").describe("Database family the downstream AI should target when generating migrations."),
|
|
22214
|
+
scheduleCadence: import_zod27.z.enum(["daily", "weekly", "monthly", "custom"]).default("weekly").describe("Default recurring rank check cadence for the generated cron/heartbeat plan."),
|
|
22215
|
+
customCron: import_zod27.z.string().min(1).optional().describe("Cron expression to use when scheduleCadence is custom."),
|
|
22216
|
+
timezone: import_zod27.z.string().min(1).default("UTC").describe("IANA timezone for scheduled rank checks, e.g. America/Denver."),
|
|
22217
|
+
includeCron: import_zod27.z.boolean().default(true).describe("Include a cron or heartbeat worker plan. Keep true for production rank trackers."),
|
|
22218
|
+
includeDashboard: import_zod27.z.boolean().default(true).describe("Include dashboard/reporting requirements in the generated prompt."),
|
|
22219
|
+
includeAlerts: import_zod27.z.boolean().default(true).describe("Include alert rules for rank movement and SERP feature gains/losses."),
|
|
22220
|
+
notes: import_zod27.z.string().max(4e3).optional().describe("Extra product, client, stack, or hosting requirements to include in the implementation prompt.")
|
|
21123
22221
|
};
|
|
21124
|
-
NullableString =
|
|
21125
|
-
MapsSearchAttemptOutput =
|
|
21126
|
-
attemptNumber:
|
|
21127
|
-
maxAttempts:
|
|
21128
|
-
status:
|
|
21129
|
-
outcome:
|
|
21130
|
-
willRetry:
|
|
21131
|
-
durationMs:
|
|
21132
|
-
resultCount:
|
|
22222
|
+
NullableString = import_zod27.z.string().nullable();
|
|
22223
|
+
MapsSearchAttemptOutput = import_zod27.z.object({
|
|
22224
|
+
attemptNumber: import_zod27.z.number().int().min(1),
|
|
22225
|
+
maxAttempts: import_zod27.z.number().int().min(1),
|
|
22226
|
+
status: import_zod27.z.enum(["ok", "failed"]),
|
|
22227
|
+
outcome: import_zod27.z.string(),
|
|
22228
|
+
willRetry: import_zod27.z.boolean(),
|
|
22229
|
+
durationMs: import_zod27.z.number().int().min(0),
|
|
22230
|
+
resultCount: import_zod27.z.number().int().min(0),
|
|
21133
22231
|
error: NullableString,
|
|
21134
|
-
proxyMode:
|
|
21135
|
-
proxyResolutionSource:
|
|
22232
|
+
proxyMode: import_zod27.z.enum(["location", "configured", "none"]),
|
|
22233
|
+
proxyResolutionSource: import_zod27.z.enum(["disabled", "location_reused", "location_created", "configured_fallback", "unavailable"]).nullable(),
|
|
21136
22234
|
proxyIdSuffix: NullableString,
|
|
21137
|
-
proxyTargetLevel:
|
|
22235
|
+
proxyTargetLevel: import_zod27.z.enum(["zip", "city", "state"]).nullable(),
|
|
21138
22236
|
proxyTargetLocation: NullableString,
|
|
21139
22237
|
proxyTargetZip: NullableString,
|
|
21140
22238
|
browserSessionIdSuffix: NullableString,
|
|
@@ -21143,17 +22241,17 @@ var init_mcp_tool_schemas = __esm({
|
|
|
21143
22241
|
observedRegion: NullableString
|
|
21144
22242
|
});
|
|
21145
22243
|
MapsSearchOutputSchema = {
|
|
21146
|
-
query:
|
|
21147
|
-
location:
|
|
21148
|
-
searchQuery:
|
|
21149
|
-
searchUrl:
|
|
21150
|
-
extractedAt:
|
|
21151
|
-
requestedMaxResults:
|
|
21152
|
-
resultCount:
|
|
21153
|
-
results:
|
|
21154
|
-
position:
|
|
21155
|
-
name:
|
|
21156
|
-
placeUrl:
|
|
22244
|
+
query: import_zod27.z.string(),
|
|
22245
|
+
location: import_zod27.z.string().nullable(),
|
|
22246
|
+
searchQuery: import_zod27.z.string(),
|
|
22247
|
+
searchUrl: import_zod27.z.string().url(),
|
|
22248
|
+
extractedAt: import_zod27.z.string(),
|
|
22249
|
+
requestedMaxResults: import_zod27.z.number().int().min(1).max(50),
|
|
22250
|
+
resultCount: import_zod27.z.number().int().min(0).max(50),
|
|
22251
|
+
results: import_zod27.z.array(import_zod27.z.object({
|
|
22252
|
+
position: import_zod27.z.number().int().min(1),
|
|
22253
|
+
name: import_zod27.z.string(),
|
|
22254
|
+
placeUrl: import_zod27.z.string().url(),
|
|
21157
22255
|
cid: NullableString,
|
|
21158
22256
|
cidDecimal: NullableString,
|
|
21159
22257
|
rating: NullableString,
|
|
@@ -21164,15 +22262,15 @@ var init_mcp_tool_schemas = __esm({
|
|
|
21164
22262
|
hoursStatus: NullableString,
|
|
21165
22263
|
websiteUrl: NullableString,
|
|
21166
22264
|
directionsUrl: NullableString,
|
|
21167
|
-
metadata:
|
|
22265
|
+
metadata: import_zod27.z.array(import_zod27.z.string())
|
|
21168
22266
|
})),
|
|
21169
|
-
attempts:
|
|
21170
|
-
durationMs:
|
|
22267
|
+
attempts: import_zod27.z.array(MapsSearchAttemptOutput),
|
|
22268
|
+
durationMs: import_zod27.z.number().int().min(0)
|
|
21171
22269
|
};
|
|
21172
|
-
DirectoryMapsBusinessOutput =
|
|
21173
|
-
position:
|
|
21174
|
-
name:
|
|
21175
|
-
placeUrl:
|
|
22270
|
+
DirectoryMapsBusinessOutput = import_zod27.z.object({
|
|
22271
|
+
position: import_zod27.z.number().int().min(1),
|
|
22272
|
+
name: import_zod27.z.string(),
|
|
22273
|
+
placeUrl: import_zod27.z.string().url(),
|
|
21176
22274
|
cid: NullableString,
|
|
21177
22275
|
cidDecimal: NullableString,
|
|
21178
22276
|
rating: NullableString,
|
|
@@ -21183,113 +22281,113 @@ var init_mcp_tool_schemas = __esm({
|
|
|
21183
22281
|
hoursStatus: NullableString,
|
|
21184
22282
|
websiteUrl: NullableString,
|
|
21185
22283
|
directionsUrl: NullableString,
|
|
21186
|
-
metadata:
|
|
22284
|
+
metadata: import_zod27.z.array(import_zod27.z.string())
|
|
21187
22285
|
});
|
|
21188
22286
|
DirectoryWorkflowOutputSchema = {
|
|
21189
|
-
query:
|
|
21190
|
-
state:
|
|
21191
|
-
minPopulation:
|
|
21192
|
-
populationYear:
|
|
21193
|
-
maxResultsPerCity:
|
|
21194
|
-
concurrency:
|
|
21195
|
-
censusSourceUrl:
|
|
22287
|
+
query: import_zod27.z.string(),
|
|
22288
|
+
state: import_zod27.z.string(),
|
|
22289
|
+
minPopulation: import_zod27.z.number().int().min(0),
|
|
22290
|
+
populationYear: import_zod27.z.number().int().min(2020).max(2025),
|
|
22291
|
+
maxResultsPerCity: import_zod27.z.number().int().min(1).max(50),
|
|
22292
|
+
concurrency: import_zod27.z.number().int().min(1).max(5),
|
|
22293
|
+
censusSourceUrl: import_zod27.z.string().url(),
|
|
21196
22294
|
usZipsSourcePath: NullableString,
|
|
21197
|
-
warnings:
|
|
21198
|
-
extractedAt:
|
|
21199
|
-
selectedCityCount:
|
|
21200
|
-
totalResultCount:
|
|
22295
|
+
warnings: import_zod27.z.array(import_zod27.z.string()),
|
|
22296
|
+
extractedAt: import_zod27.z.string(),
|
|
22297
|
+
selectedCityCount: import_zod27.z.number().int().min(0),
|
|
22298
|
+
totalResultCount: import_zod27.z.number().int().min(0),
|
|
21201
22299
|
csvPath: NullableString,
|
|
21202
|
-
cities:
|
|
21203
|
-
city:
|
|
21204
|
-
state:
|
|
21205
|
-
location:
|
|
21206
|
-
cityKey:
|
|
21207
|
-
censusName:
|
|
21208
|
-
population:
|
|
21209
|
-
populationYear:
|
|
21210
|
-
zips:
|
|
21211
|
-
counties:
|
|
21212
|
-
status:
|
|
22300
|
+
cities: import_zod27.z.array(import_zod27.z.object({
|
|
22301
|
+
city: import_zod27.z.string(),
|
|
22302
|
+
state: import_zod27.z.string(),
|
|
22303
|
+
location: import_zod27.z.string(),
|
|
22304
|
+
cityKey: import_zod27.z.string(),
|
|
22305
|
+
censusName: import_zod27.z.string(),
|
|
22306
|
+
population: import_zod27.z.number().int().min(0),
|
|
22307
|
+
populationYear: import_zod27.z.number().int().min(2020).max(2025),
|
|
22308
|
+
zips: import_zod27.z.array(import_zod27.z.string()),
|
|
22309
|
+
counties: import_zod27.z.array(import_zod27.z.string()),
|
|
22310
|
+
status: import_zod27.z.enum(["ok", "empty", "failed"]),
|
|
21213
22311
|
error: NullableString,
|
|
21214
|
-
resultCount:
|
|
21215
|
-
durationMs:
|
|
21216
|
-
attempts:
|
|
21217
|
-
results:
|
|
22312
|
+
resultCount: import_zod27.z.number().int().min(0),
|
|
22313
|
+
durationMs: import_zod27.z.number().int().min(0),
|
|
22314
|
+
attempts: import_zod27.z.array(MapsSearchAttemptOutput),
|
|
22315
|
+
results: import_zod27.z.array(DirectoryMapsBusinessOutput)
|
|
21218
22316
|
})),
|
|
21219
|
-
durationMs:
|
|
22317
|
+
durationMs: import_zod27.z.number().int().min(0)
|
|
21220
22318
|
};
|
|
21221
|
-
RankTrackerToolPlanOutput =
|
|
21222
|
-
tool:
|
|
21223
|
-
purpose:
|
|
22319
|
+
RankTrackerToolPlanOutput = import_zod27.z.object({
|
|
22320
|
+
tool: import_zod27.z.string(),
|
|
22321
|
+
purpose: import_zod27.z.string()
|
|
21224
22322
|
});
|
|
21225
|
-
RankTrackerTableOutput =
|
|
21226
|
-
name:
|
|
21227
|
-
purpose:
|
|
21228
|
-
keyColumns:
|
|
22323
|
+
RankTrackerTableOutput = import_zod27.z.object({
|
|
22324
|
+
name: import_zod27.z.string(),
|
|
22325
|
+
purpose: import_zod27.z.string(),
|
|
22326
|
+
keyColumns: import_zod27.z.array(import_zod27.z.string())
|
|
21229
22327
|
});
|
|
21230
|
-
RankTrackerCronJobOutput =
|
|
21231
|
-
name:
|
|
21232
|
-
purpose:
|
|
21233
|
-
modes:
|
|
21234
|
-
recommendedTools:
|
|
22328
|
+
RankTrackerCronJobOutput = import_zod27.z.object({
|
|
22329
|
+
name: import_zod27.z.string(),
|
|
22330
|
+
purpose: import_zod27.z.string(),
|
|
22331
|
+
modes: import_zod27.z.array(RankTrackerModeSchema),
|
|
22332
|
+
recommendedTools: import_zod27.z.array(import_zod27.z.string())
|
|
21235
22333
|
});
|
|
21236
22334
|
RankTrackerBlueprintOutputSchema = {
|
|
21237
|
-
projectName:
|
|
22335
|
+
projectName: import_zod27.z.string(),
|
|
21238
22336
|
targetDomain: NullableString,
|
|
21239
22337
|
targetBusinessName: NullableString,
|
|
21240
|
-
trackingModes:
|
|
21241
|
-
database:
|
|
21242
|
-
recommendedTools:
|
|
21243
|
-
tables:
|
|
21244
|
-
cron:
|
|
21245
|
-
enabled:
|
|
21246
|
-
cadence:
|
|
21247
|
-
expression:
|
|
21248
|
-
timezone:
|
|
21249
|
-
jobs:
|
|
22338
|
+
trackingModes: import_zod27.z.array(RankTrackerModeSchema),
|
|
22339
|
+
database: import_zod27.z.string(),
|
|
22340
|
+
recommendedTools: import_zod27.z.array(RankTrackerToolPlanOutput),
|
|
22341
|
+
tables: import_zod27.z.array(RankTrackerTableOutput),
|
|
22342
|
+
cron: import_zod27.z.object({
|
|
22343
|
+
enabled: import_zod27.z.boolean(),
|
|
22344
|
+
cadence: import_zod27.z.string(),
|
|
22345
|
+
expression: import_zod27.z.string(),
|
|
22346
|
+
timezone: import_zod27.z.string(),
|
|
22347
|
+
jobs: import_zod27.z.array(RankTrackerCronJobOutput)
|
|
21250
22348
|
}),
|
|
21251
|
-
metrics:
|
|
21252
|
-
implementationPrompt:
|
|
22349
|
+
metrics: import_zod27.z.array(import_zod27.z.string()),
|
|
22350
|
+
implementationPrompt: import_zod27.z.string()
|
|
21253
22351
|
};
|
|
21254
|
-
OrganicResultOutput =
|
|
21255
|
-
position:
|
|
21256
|
-
title:
|
|
21257
|
-
url:
|
|
21258
|
-
domain:
|
|
22352
|
+
OrganicResultOutput = import_zod27.z.object({
|
|
22353
|
+
position: import_zod27.z.number().int(),
|
|
22354
|
+
title: import_zod27.z.string(),
|
|
22355
|
+
url: import_zod27.z.string(),
|
|
22356
|
+
domain: import_zod27.z.string(),
|
|
21259
22357
|
snippet: NullableString
|
|
21260
22358
|
});
|
|
21261
|
-
AiOverviewOutput =
|
|
21262
|
-
detected:
|
|
22359
|
+
AiOverviewOutput = import_zod27.z.object({
|
|
22360
|
+
detected: import_zod27.z.boolean(),
|
|
21263
22361
|
text: NullableString
|
|
21264
22362
|
}).nullable();
|
|
21265
|
-
EntityIdsOutput =
|
|
21266
|
-
kgIds:
|
|
21267
|
-
cids:
|
|
21268
|
-
gcids:
|
|
22363
|
+
EntityIdsOutput = import_zod27.z.object({
|
|
22364
|
+
kgIds: import_zod27.z.array(import_zod27.z.string()),
|
|
22365
|
+
cids: import_zod27.z.array(import_zod27.z.string()),
|
|
22366
|
+
gcids: import_zod27.z.array(import_zod27.z.string())
|
|
21269
22367
|
}).nullable();
|
|
21270
22368
|
HarvestPaaOutputSchema = {
|
|
21271
|
-
query:
|
|
22369
|
+
query: import_zod27.z.string(),
|
|
21272
22370
|
location: NullableString,
|
|
21273
|
-
questionCount:
|
|
22371
|
+
questionCount: import_zod27.z.number().int().min(0),
|
|
21274
22372
|
completionStatus: NullableString,
|
|
21275
|
-
questions:
|
|
21276
|
-
question:
|
|
22373
|
+
questions: import_zod27.z.array(import_zod27.z.object({
|
|
22374
|
+
question: import_zod27.z.string(),
|
|
21277
22375
|
answer: NullableString,
|
|
21278
22376
|
sourceTitle: NullableString,
|
|
21279
22377
|
sourceSite: NullableString
|
|
21280
22378
|
})),
|
|
21281
|
-
organicResults:
|
|
22379
|
+
organicResults: import_zod27.z.array(OrganicResultOutput),
|
|
21282
22380
|
aiOverview: AiOverviewOutput,
|
|
21283
22381
|
entityIds: EntityIdsOutput,
|
|
21284
|
-
durationMs:
|
|
22382
|
+
durationMs: import_zod27.z.number().min(0).nullable()
|
|
21285
22383
|
};
|
|
21286
22384
|
SearchSerpOutputSchema = {
|
|
21287
|
-
query:
|
|
22385
|
+
query: import_zod27.z.string(),
|
|
21288
22386
|
location: NullableString,
|
|
21289
|
-
organicResults:
|
|
21290
|
-
localPack:
|
|
21291
|
-
position:
|
|
21292
|
-
name:
|
|
22387
|
+
organicResults: import_zod27.z.array(OrganicResultOutput),
|
|
22388
|
+
localPack: import_zod27.z.array(import_zod27.z.object({
|
|
22389
|
+
position: import_zod27.z.number().int(),
|
|
22390
|
+
name: import_zod27.z.string(),
|
|
21293
22391
|
rating: NullableString,
|
|
21294
22392
|
reviewCount: NullableString,
|
|
21295
22393
|
websiteUrl: NullableString
|
|
@@ -21298,31 +22396,31 @@ var init_mcp_tool_schemas = __esm({
|
|
|
21298
22396
|
entityIds: EntityIdsOutput
|
|
21299
22397
|
};
|
|
21300
22398
|
ExtractUrlOutputSchema = {
|
|
21301
|
-
url:
|
|
22399
|
+
url: import_zod27.z.string(),
|
|
21302
22400
|
title: NullableString,
|
|
21303
|
-
headings:
|
|
21304
|
-
level:
|
|
21305
|
-
text:
|
|
22401
|
+
headings: import_zod27.z.array(import_zod27.z.object({
|
|
22402
|
+
level: import_zod27.z.number().int(),
|
|
22403
|
+
text: import_zod27.z.string()
|
|
21306
22404
|
})),
|
|
21307
|
-
schemaBlockCount:
|
|
22405
|
+
schemaBlockCount: import_zod27.z.number().int().min(0),
|
|
21308
22406
|
entityName: NullableString,
|
|
21309
|
-
entityTypes:
|
|
21310
|
-
napScore:
|
|
21311
|
-
missingSchemaFields:
|
|
22407
|
+
entityTypes: import_zod27.z.array(import_zod27.z.string()),
|
|
22408
|
+
napScore: import_zod27.z.number().nullable(),
|
|
22409
|
+
missingSchemaFields: import_zod27.z.array(import_zod27.z.string()),
|
|
21312
22410
|
screenshotSaved: NullableString
|
|
21313
22411
|
};
|
|
21314
22412
|
ExtractSiteOutputSchema = {
|
|
21315
|
-
url:
|
|
21316
|
-
pageCount:
|
|
21317
|
-
pages:
|
|
21318
|
-
url:
|
|
22413
|
+
url: import_zod27.z.string(),
|
|
22414
|
+
pageCount: import_zod27.z.number().int().min(0),
|
|
22415
|
+
pages: import_zod27.z.array(import_zod27.z.object({
|
|
22416
|
+
url: import_zod27.z.string(),
|
|
21319
22417
|
title: NullableString,
|
|
21320
|
-
schemaTypes:
|
|
22418
|
+
schemaTypes: import_zod27.z.array(import_zod27.z.string())
|
|
21321
22419
|
})),
|
|
21322
|
-
durationMs:
|
|
22420
|
+
durationMs: import_zod27.z.number().min(0)
|
|
21323
22421
|
};
|
|
21324
22422
|
MapsPlaceIntelOutputSchema = {
|
|
21325
|
-
name:
|
|
22423
|
+
name: import_zod27.z.string(),
|
|
21326
22424
|
rating: NullableString,
|
|
21327
22425
|
reviewCount: NullableString,
|
|
21328
22426
|
category: NullableString,
|
|
@@ -21334,75 +22432,75 @@ var init_mcp_tool_schemas = __esm({
|
|
|
21334
22432
|
kgmid: NullableString,
|
|
21335
22433
|
cidDecimal: NullableString,
|
|
21336
22434
|
cidUrl: NullableString,
|
|
21337
|
-
lat:
|
|
21338
|
-
lng:
|
|
21339
|
-
reviewsStatus:
|
|
21340
|
-
reviewsCollected:
|
|
21341
|
-
reviewTopics:
|
|
21342
|
-
label:
|
|
21343
|
-
count:
|
|
22435
|
+
lat: import_zod27.z.number().nullable(),
|
|
22436
|
+
lng: import_zod27.z.number().nullable(),
|
|
22437
|
+
reviewsStatus: import_zod27.z.string(),
|
|
22438
|
+
reviewsCollected: import_zod27.z.number().int().min(0),
|
|
22439
|
+
reviewTopics: import_zod27.z.array(import_zod27.z.object({
|
|
22440
|
+
label: import_zod27.z.string(),
|
|
22441
|
+
count: import_zod27.z.string()
|
|
21344
22442
|
}))
|
|
21345
22443
|
};
|
|
21346
22444
|
CreditsInfoOutputSchema = {
|
|
21347
|
-
balanceCredits:
|
|
21348
|
-
matchedCost:
|
|
21349
|
-
label:
|
|
21350
|
-
credits:
|
|
21351
|
-
unit:
|
|
22445
|
+
balanceCredits: import_zod27.z.number().nullable(),
|
|
22446
|
+
matchedCost: import_zod27.z.object({
|
|
22447
|
+
label: import_zod27.z.string(),
|
|
22448
|
+
credits: import_zod27.z.number(),
|
|
22449
|
+
unit: import_zod27.z.string(),
|
|
21352
22450
|
notes: NullableString
|
|
21353
22451
|
}).nullable(),
|
|
21354
|
-
costs:
|
|
21355
|
-
key:
|
|
21356
|
-
label:
|
|
21357
|
-
credits:
|
|
21358
|
-
unit:
|
|
22452
|
+
costs: import_zod27.z.array(import_zod27.z.object({
|
|
22453
|
+
key: import_zod27.z.string(),
|
|
22454
|
+
label: import_zod27.z.string(),
|
|
22455
|
+
credits: import_zod27.z.number(),
|
|
22456
|
+
unit: import_zod27.z.string(),
|
|
21359
22457
|
notes: NullableString
|
|
21360
22458
|
})),
|
|
21361
|
-
ledger:
|
|
21362
|
-
createdAt:
|
|
21363
|
-
operation:
|
|
21364
|
-
credits:
|
|
22459
|
+
ledger: import_zod27.z.array(import_zod27.z.object({
|
|
22460
|
+
createdAt: import_zod27.z.string(),
|
|
22461
|
+
operation: import_zod27.z.string(),
|
|
22462
|
+
credits: import_zod27.z.number(),
|
|
21365
22463
|
description: NullableString
|
|
21366
22464
|
})),
|
|
21367
|
-
concurrency:
|
|
21368
|
-
currentExtraSlots:
|
|
21369
|
-
currentLimit:
|
|
21370
|
-
hasSubscription:
|
|
21371
|
-
upgrade:
|
|
21372
|
-
product:
|
|
21373
|
-
priceLabel:
|
|
21374
|
-
unitAmountUsd:
|
|
21375
|
-
currency:
|
|
21376
|
-
interval:
|
|
21377
|
-
billingUrl:
|
|
21378
|
-
terminalCommand:
|
|
21379
|
-
terminalCommandWithApiKeyEnv:
|
|
22465
|
+
concurrency: import_zod27.z.object({
|
|
22466
|
+
currentExtraSlots: import_zod27.z.number().int().min(0),
|
|
22467
|
+
currentLimit: import_zod27.z.number().int().min(1),
|
|
22468
|
+
hasSubscription: import_zod27.z.boolean(),
|
|
22469
|
+
upgrade: import_zod27.z.object({
|
|
22470
|
+
product: import_zod27.z.string(),
|
|
22471
|
+
priceLabel: import_zod27.z.string(),
|
|
22472
|
+
unitAmountUsd: import_zod27.z.number(),
|
|
22473
|
+
currency: import_zod27.z.string(),
|
|
22474
|
+
interval: import_zod27.z.string(),
|
|
22475
|
+
billingUrl: import_zod27.z.string().url(),
|
|
22476
|
+
terminalCommand: import_zod27.z.string(),
|
|
22477
|
+
terminalCommandWithApiKeyEnv: import_zod27.z.string()
|
|
21380
22478
|
})
|
|
21381
22479
|
}).nullable()
|
|
21382
22480
|
};
|
|
21383
22481
|
MapSiteUrlsOutputSchema = {
|
|
21384
|
-
startUrl:
|
|
21385
|
-
totalFound:
|
|
21386
|
-
truncated:
|
|
21387
|
-
okCount:
|
|
21388
|
-
redirectCount:
|
|
21389
|
-
brokenCount:
|
|
21390
|
-
urls:
|
|
21391
|
-
url:
|
|
21392
|
-
status:
|
|
22482
|
+
startUrl: import_zod27.z.string(),
|
|
22483
|
+
totalFound: import_zod27.z.number().int().min(0),
|
|
22484
|
+
truncated: import_zod27.z.boolean(),
|
|
22485
|
+
okCount: import_zod27.z.number().int().min(0),
|
|
22486
|
+
redirectCount: import_zod27.z.number().int().min(0),
|
|
22487
|
+
brokenCount: import_zod27.z.number().int().min(0),
|
|
22488
|
+
urls: import_zod27.z.array(import_zod27.z.object({
|
|
22489
|
+
url: import_zod27.z.string(),
|
|
22490
|
+
status: import_zod27.z.number().int().nullable()
|
|
21393
22491
|
})),
|
|
21394
|
-
durationMs:
|
|
22492
|
+
durationMs: import_zod27.z.number().min(0)
|
|
21395
22493
|
};
|
|
21396
22494
|
YoutubeHarvestOutputSchema = {
|
|
21397
|
-
mode:
|
|
21398
|
-
videoCount:
|
|
21399
|
-
channel:
|
|
22495
|
+
mode: import_zod27.z.string(),
|
|
22496
|
+
videoCount: import_zod27.z.number().int().min(0),
|
|
22497
|
+
channel: import_zod27.z.object({
|
|
21400
22498
|
title: NullableString,
|
|
21401
22499
|
subscriberCount: NullableString
|
|
21402
22500
|
}).nullable(),
|
|
21403
|
-
videos:
|
|
21404
|
-
videoId:
|
|
21405
|
-
title:
|
|
22501
|
+
videos: import_zod27.z.array(import_zod27.z.object({
|
|
22502
|
+
videoId: import_zod27.z.string(),
|
|
22503
|
+
title: import_zod27.z.string(),
|
|
21406
22504
|
channelName: NullableString,
|
|
21407
22505
|
views: NullableString,
|
|
21408
22506
|
duration: NullableString,
|
|
@@ -21410,24 +22508,24 @@ var init_mcp_tool_schemas = __esm({
|
|
|
21410
22508
|
}))
|
|
21411
22509
|
};
|
|
21412
22510
|
FacebookAdSearchOutputSchema = {
|
|
21413
|
-
query:
|
|
21414
|
-
advertiserCount:
|
|
21415
|
-
advertisers:
|
|
22511
|
+
query: import_zod27.z.string(),
|
|
22512
|
+
advertiserCount: import_zod27.z.number().int().min(0),
|
|
22513
|
+
advertisers: import_zod27.z.array(import_zod27.z.object({
|
|
21416
22514
|
name: NullableString,
|
|
21417
22515
|
pageId: NullableString,
|
|
21418
22516
|
pageUrl: NullableString,
|
|
21419
|
-
adCount:
|
|
22517
|
+
adCount: import_zod27.z.number().int().nullable(),
|
|
21420
22518
|
libraryId: NullableString,
|
|
21421
22519
|
sampleLibraryId: NullableString
|
|
21422
22520
|
}))
|
|
21423
22521
|
};
|
|
21424
22522
|
FacebookPageIntelOutputSchema = {
|
|
21425
22523
|
advertiserName: NullableString,
|
|
21426
|
-
totalAds:
|
|
21427
|
-
activeCount:
|
|
21428
|
-
videoCount:
|
|
21429
|
-
imageCount:
|
|
21430
|
-
ads:
|
|
22524
|
+
totalAds: import_zod27.z.number().int().min(0),
|
|
22525
|
+
activeCount: import_zod27.z.number().int().min(0),
|
|
22526
|
+
videoCount: import_zod27.z.number().int().min(0),
|
|
22527
|
+
imageCount: import_zod27.z.number().int().min(0),
|
|
22528
|
+
ads: import_zod27.z.array(import_zod27.z.object({
|
|
21431
22529
|
libraryId: NullableString,
|
|
21432
22530
|
status: NullableString,
|
|
21433
22531
|
creativeType: NullableString,
|
|
@@ -21440,84 +22538,179 @@ var init_mcp_tool_schemas = __esm({
|
|
|
21440
22538
|
videoUrl: NullableString,
|
|
21441
22539
|
imageUrl: NullableString,
|
|
21442
22540
|
videoPoster: NullableString,
|
|
21443
|
-
variations:
|
|
22541
|
+
variations: import_zod27.z.number().int().nullable()
|
|
21444
22542
|
}))
|
|
21445
22543
|
};
|
|
21446
22544
|
FacebookVideoTranscribeOutputSchema = {
|
|
21447
|
-
sourceUrl:
|
|
21448
|
-
pageUrl:
|
|
22545
|
+
sourceUrl: import_zod27.z.string().url(),
|
|
22546
|
+
pageUrl: import_zod27.z.string().url(),
|
|
21449
22547
|
videoId: NullableString,
|
|
21450
22548
|
ownerName: NullableString,
|
|
21451
|
-
selectedQuality:
|
|
21452
|
-
bitrate:
|
|
21453
|
-
videoDurationSec:
|
|
21454
|
-
videoUrl:
|
|
21455
|
-
wordCount:
|
|
21456
|
-
chunkCount:
|
|
21457
|
-
transcriptText:
|
|
21458
|
-
chunks:
|
|
21459
|
-
startSec:
|
|
21460
|
-
endSec:
|
|
21461
|
-
text:
|
|
22549
|
+
selectedQuality: import_zod27.z.string(),
|
|
22550
|
+
bitrate: import_zod27.z.number().int().nullable(),
|
|
22551
|
+
videoDurationSec: import_zod27.z.number().nullable(),
|
|
22552
|
+
videoUrl: import_zod27.z.string().url(),
|
|
22553
|
+
wordCount: import_zod27.z.number().int().min(0),
|
|
22554
|
+
chunkCount: import_zod27.z.number().int().min(0),
|
|
22555
|
+
transcriptText: import_zod27.z.string(),
|
|
22556
|
+
chunks: import_zod27.z.array(import_zod27.z.object({
|
|
22557
|
+
startSec: import_zod27.z.number(),
|
|
22558
|
+
endSec: import_zod27.z.number(),
|
|
22559
|
+
text: import_zod27.z.string()
|
|
22560
|
+
}))
|
|
22561
|
+
};
|
|
22562
|
+
TranscriptChunkOutput = import_zod27.z.object({
|
|
22563
|
+
startSec: import_zod27.z.number(),
|
|
22564
|
+
endSec: import_zod27.z.number(),
|
|
22565
|
+
text: import_zod27.z.string()
|
|
22566
|
+
});
|
|
22567
|
+
InstagramBrowserOutput = import_zod27.z.object({
|
|
22568
|
+
mode: import_zod27.z.enum(["hosted", "local"]),
|
|
22569
|
+
requestedMode: import_zod27.z.enum(["auto", "hosted", "local"]),
|
|
22570
|
+
profileName: NullableString,
|
|
22571
|
+
profileSource: import_zod27.z.enum(["hosted", "managed_profile", "direct_profile_dir"]),
|
|
22572
|
+
profileDirConfigured: import_zod27.z.boolean(),
|
|
22573
|
+
executablePathConfigured: import_zod27.z.boolean()
|
|
22574
|
+
});
|
|
22575
|
+
InstagramPaginationOutput = import_zod27.z.object({
|
|
22576
|
+
maxItems: import_zod27.z.number().int().min(1).max(500),
|
|
22577
|
+
maxScrolls: import_zod27.z.number().int().min(0).max(100),
|
|
22578
|
+
attemptedScrolls: import_zod27.z.number().int().min(0),
|
|
22579
|
+
stableScrolls: import_zod27.z.number().int().min(0),
|
|
22580
|
+
stableScrollLimit: import_zod27.z.number().int().min(1).max(10),
|
|
22581
|
+
scrollDelayMs: import_zod27.z.number().int().min(250).max(5e3),
|
|
22582
|
+
reachedMaxItems: import_zod27.z.boolean(),
|
|
22583
|
+
reachedReportedPostCount: import_zod27.z.boolean(),
|
|
22584
|
+
finalScrollHeight: import_zod27.z.number().int().nullable(),
|
|
22585
|
+
stoppedReason: import_zod27.z.enum(["max_items", "reported_post_count", "stable_scrolls", "max_scrolls", "no_scrolls"]),
|
|
22586
|
+
stages: import_zod27.z.array(import_zod27.z.object({
|
|
22587
|
+
stage: import_zod27.z.string(),
|
|
22588
|
+
itemCount: import_zod27.z.number().int().min(0),
|
|
22589
|
+
addedCount: import_zod27.z.number().int().min(0),
|
|
22590
|
+
scrollY: import_zod27.z.number().nullable(),
|
|
22591
|
+
scrollHeight: import_zod27.z.number().nullable()
|
|
22592
|
+
}))
|
|
22593
|
+
});
|
|
22594
|
+
InstagramProfileContentOutputSchema = {
|
|
22595
|
+
handle: import_zod27.z.string(),
|
|
22596
|
+
profileUrl: import_zod27.z.string().url(),
|
|
22597
|
+
pageUrl: import_zod27.z.string().url(),
|
|
22598
|
+
browser: InstagramBrowserOutput,
|
|
22599
|
+
profileName: NullableString,
|
|
22600
|
+
reportedPostCount: import_zod27.z.number().int().nullable(),
|
|
22601
|
+
reportedPostCountText: NullableString,
|
|
22602
|
+
followerCountText: NullableString,
|
|
22603
|
+
followingCountText: NullableString,
|
|
22604
|
+
collectedContentCount: import_zod27.z.number().int().min(0),
|
|
22605
|
+
typeCounts: import_zod27.z.object({
|
|
22606
|
+
post: import_zod27.z.number().int().min(0),
|
|
22607
|
+
reel: import_zod27.z.number().int().min(0),
|
|
22608
|
+
tv: import_zod27.z.number().int().min(0)
|
|
22609
|
+
}),
|
|
22610
|
+
pagination: InstagramPaginationOutput,
|
|
22611
|
+
limited: import_zod27.z.boolean(),
|
|
22612
|
+
limitations: import_zod27.z.array(import_zod27.z.string()),
|
|
22613
|
+
items: import_zod27.z.array(import_zod27.z.object({
|
|
22614
|
+
url: import_zod27.z.string().url(),
|
|
22615
|
+
type: import_zod27.z.enum(["post", "reel", "tv"]),
|
|
22616
|
+
shortcode: import_zod27.z.string(),
|
|
22617
|
+
anchorText: NullableString,
|
|
22618
|
+
firstSeenStage: import_zod27.z.string()
|
|
21462
22619
|
}))
|
|
21463
22620
|
};
|
|
21464
|
-
|
|
21465
|
-
|
|
21466
|
-
|
|
21467
|
-
|
|
22621
|
+
InstagramMediaTrackOutput = import_zod27.z.object({
|
|
22622
|
+
url: import_zod27.z.string().url(),
|
|
22623
|
+
streamType: import_zod27.z.enum(["video", "audio", "unknown"]),
|
|
22624
|
+
bitrate: import_zod27.z.number().int().nullable(),
|
|
22625
|
+
durationSec: import_zod27.z.number().nullable(),
|
|
22626
|
+
vencodeTag: NullableString,
|
|
22627
|
+
width: import_zod27.z.number().int().nullable(),
|
|
22628
|
+
height: import_zod27.z.number().int().nullable()
|
|
21468
22629
|
});
|
|
22630
|
+
InstagramDownloadOutput = import_zod27.z.object({
|
|
22631
|
+
kind: import_zod27.z.enum(["text", "image", "video", "audio", "muxed_video"]),
|
|
22632
|
+
url: import_zod27.z.string().url().nullable(),
|
|
22633
|
+
savedPath: NullableString,
|
|
22634
|
+
sizeBytes: import_zod27.z.number().int().nullable(),
|
|
22635
|
+
mimeType: NullableString,
|
|
22636
|
+
error: NullableString
|
|
22637
|
+
});
|
|
22638
|
+
InstagramMediaDownloadOutputSchema = {
|
|
22639
|
+
sourceUrl: import_zod27.z.string().url(),
|
|
22640
|
+
pageUrl: import_zod27.z.string().url(),
|
|
22641
|
+
browser: InstagramBrowserOutput,
|
|
22642
|
+
type: import_zod27.z.enum(["post", "reel", "tv"]).nullable(),
|
|
22643
|
+
shortcode: NullableString,
|
|
22644
|
+
ownerName: NullableString,
|
|
22645
|
+
caption: NullableString,
|
|
22646
|
+
imageUrl: import_zod27.z.string().url().nullable(),
|
|
22647
|
+
trackCount: import_zod27.z.number().int().min(0),
|
|
22648
|
+
selectedVideoTrack: InstagramMediaTrackOutput.nullable(),
|
|
22649
|
+
selectedAudioTrack: InstagramMediaTrackOutput.nullable(),
|
|
22650
|
+
downloads: import_zod27.z.array(InstagramDownloadOutput),
|
|
22651
|
+
outputDir: NullableString,
|
|
22652
|
+
warnings: import_zod27.z.array(import_zod27.z.string()),
|
|
22653
|
+
limitations: import_zod27.z.array(import_zod27.z.string()),
|
|
22654
|
+
transcript: import_zod27.z.object({
|
|
22655
|
+
wordCount: import_zod27.z.number().int().min(0),
|
|
22656
|
+
chunkCount: import_zod27.z.number().int().min(0),
|
|
22657
|
+
durationMs: import_zod27.z.number().nullable(),
|
|
22658
|
+
transcriptText: import_zod27.z.string(),
|
|
22659
|
+
chunks: import_zod27.z.array(TranscriptChunkOutput)
|
|
22660
|
+
}).nullable()
|
|
22661
|
+
};
|
|
21469
22662
|
YoutubeTranscribeOutputSchema = {
|
|
21470
22663
|
videoId: NullableString,
|
|
21471
22664
|
url: NullableString,
|
|
21472
|
-
wordCount:
|
|
21473
|
-
chunkCount:
|
|
21474
|
-
durationMs:
|
|
21475
|
-
transcriptText:
|
|
21476
|
-
chunks:
|
|
21477
|
-
resolvedInputs:
|
|
22665
|
+
wordCount: import_zod27.z.number().int().min(0),
|
|
22666
|
+
chunkCount: import_zod27.z.number().int().min(0),
|
|
22667
|
+
durationMs: import_zod27.z.number().nullable(),
|
|
22668
|
+
transcriptText: import_zod27.z.string(),
|
|
22669
|
+
chunks: import_zod27.z.array(TranscriptChunkOutput),
|
|
22670
|
+
resolvedInputs: import_zod27.z.object({
|
|
21478
22671
|
videoId: NullableString,
|
|
21479
22672
|
url: NullableString
|
|
21480
22673
|
})
|
|
21481
22674
|
};
|
|
21482
22675
|
FacebookAdTranscribeOutputSchema = {
|
|
21483
|
-
videoUrl:
|
|
21484
|
-
wordCount:
|
|
21485
|
-
chunkCount:
|
|
21486
|
-
durationMs:
|
|
21487
|
-
transcriptText:
|
|
21488
|
-
chunks:
|
|
21489
|
-
resolvedInputs:
|
|
21490
|
-
videoUrl:
|
|
22676
|
+
videoUrl: import_zod27.z.string().url(),
|
|
22677
|
+
wordCount: import_zod27.z.number().int().min(0),
|
|
22678
|
+
chunkCount: import_zod27.z.number().int().min(0),
|
|
22679
|
+
durationMs: import_zod27.z.number().nullable(),
|
|
22680
|
+
transcriptText: import_zod27.z.string(),
|
|
22681
|
+
chunks: import_zod27.z.array(TranscriptChunkOutput),
|
|
22682
|
+
resolvedInputs: import_zod27.z.object({
|
|
22683
|
+
videoUrl: import_zod27.z.string().url()
|
|
21491
22684
|
})
|
|
21492
22685
|
};
|
|
21493
22686
|
CaptureSerpSnapshotOutputSchema = {
|
|
21494
|
-
schemaVersion:
|
|
21495
|
-
status:
|
|
22687
|
+
schemaVersion: import_zod27.z.literal("serp-intelligence.capture.v1"),
|
|
22688
|
+
status: import_zod27.z.string(),
|
|
21496
22689
|
query: NullableString,
|
|
21497
22690
|
location: NullableString,
|
|
21498
22691
|
capturedAt: NullableString,
|
|
21499
|
-
resultCount:
|
|
22692
|
+
resultCount: import_zod27.z.number().int().min(0).nullable(),
|
|
21500
22693
|
snapshotId: NullableString,
|
|
21501
|
-
resolvedInputs:
|
|
21502
|
-
artifacts:
|
|
21503
|
-
diagnostics:
|
|
21504
|
-
providerPayload:
|
|
22694
|
+
resolvedInputs: import_zod27.z.record(import_zod27.z.unknown()),
|
|
22695
|
+
artifacts: import_zod27.z.array(import_zod27.z.record(import_zod27.z.unknown())),
|
|
22696
|
+
diagnostics: import_zod27.z.record(import_zod27.z.unknown()).nullable(),
|
|
22697
|
+
providerPayload: import_zod27.z.record(import_zod27.z.unknown())
|
|
21505
22698
|
};
|
|
21506
22699
|
CaptureSerpPageSnapshotsOutputSchema = {
|
|
21507
|
-
schemaVersion:
|
|
21508
|
-
status:
|
|
21509
|
-
count:
|
|
21510
|
-
failedCount:
|
|
21511
|
-
captures:
|
|
21512
|
-
resolvedInputs:
|
|
21513
|
-
diagnostics:
|
|
21514
|
-
providerPayload:
|
|
22700
|
+
schemaVersion: import_zod27.z.literal("serp-intelligence.page-snapshots.v1"),
|
|
22701
|
+
status: import_zod27.z.string(),
|
|
22702
|
+
count: import_zod27.z.number().int().min(0),
|
|
22703
|
+
failedCount: import_zod27.z.number().int().min(0),
|
|
22704
|
+
captures: import_zod27.z.array(import_zod27.z.record(import_zod27.z.unknown())),
|
|
22705
|
+
resolvedInputs: import_zod27.z.record(import_zod27.z.unknown()),
|
|
22706
|
+
diagnostics: import_zod27.z.record(import_zod27.z.unknown()).nullable(),
|
|
22707
|
+
providerPayload: import_zod27.z.record(import_zod27.z.unknown())
|
|
21515
22708
|
};
|
|
21516
22709
|
CreditsInfoInputSchema = {
|
|
21517
|
-
item:
|
|
21518
|
-
includeLedger:
|
|
22710
|
+
item: import_zod27.z.string().optional().describe('Optional tool, action, or feature to look up, e.g. "maps reviews", "extract_url", "YouTube transcription", or "concurrency"'),
|
|
22711
|
+
includeLedger: import_zod27.z.boolean().default(false).describe("Whether to include recent credit ledger entries")
|
|
21519
22712
|
};
|
|
21520
|
-
WorkflowIdSchema2 =
|
|
22713
|
+
WorkflowIdSchema2 = import_zod27.z.enum([
|
|
21521
22714
|
"directory",
|
|
21522
22715
|
"agent-packet",
|
|
21523
22716
|
"local-competitive-audit",
|
|
@@ -21527,129 +22720,129 @@ var init_mcp_tool_schemas = __esm({
|
|
|
21527
22720
|
"ai-overview-language"
|
|
21528
22721
|
]);
|
|
21529
22722
|
WorkflowListInputSchema = {
|
|
21530
|
-
includeRecipes:
|
|
22723
|
+
includeRecipes: import_zod27.z.boolean().default(true).describe("Include high-level AI-facing recipes such as market analysis, ICP research, forum/review research, brand design briefings, CRO audits, positioning briefs, content gaps, and AI search visibility audits.")
|
|
21531
22724
|
};
|
|
21532
22725
|
WorkflowSuggestInputSchema = {
|
|
21533
|
-
goal:
|
|
21534
|
-
query:
|
|
21535
|
-
keyword:
|
|
21536
|
-
domain:
|
|
21537
|
-
url:
|
|
21538
|
-
location:
|
|
21539
|
-
state:
|
|
21540
|
-
maxSuggestions:
|
|
22726
|
+
goal: import_zod27.z.string().min(1).describe('The user goal or job to route, e.g. "market analysis for roofers in Tennessee", "ICP research for med spas", "CRO audit for this URL", or "brand design briefing".'),
|
|
22727
|
+
query: import_zod27.z.string().optional().describe("Business category, niche, or Maps query when known."),
|
|
22728
|
+
keyword: import_zod27.z.string().optional().describe("Search keyword, audience problem, or content topic when known."),
|
|
22729
|
+
domain: import_zod27.z.string().optional().describe("Target domain or brand domain when known."),
|
|
22730
|
+
url: import_zod27.z.string().url().optional().describe("Target URL when the workflow should inspect a specific page."),
|
|
22731
|
+
location: import_zod27.z.string().optional().describe("City/region/country for localized research, e.g. Denver, CO."),
|
|
22732
|
+
state: import_zod27.z.string().optional().describe("US state abbreviation or name for state-wide market research."),
|
|
22733
|
+
maxSuggestions: import_zod27.z.number().int().min(1).max(8).default(3).describe("Number of matching workflow recipes to return.")
|
|
21541
22734
|
};
|
|
21542
22735
|
WorkflowRunInputSchema = {
|
|
21543
22736
|
workflowId: WorkflowIdSchema2.describe("Workflow to run: directory, agent-packet, local-competitive-audit, map-comparison, serp-comparison, paa-expansion-brief, ai-overview-language. Use only these values; call workflow_list or workflow_suggest first when unsure."),
|
|
21544
|
-
input:
|
|
21545
|
-
webhookUrl:
|
|
22737
|
+
input: import_zod27.z.record(import_zod27.z.unknown()).default({}).describe("Workflow-specific input object. Examples: agent-packet uses {keyword, domain?, location?, maxQuestions?}; local-competitive-audit uses {query, state, minPopulation?, maxCities?, maxResultsPerCity?, hydrateTop?, maxReviews?}; serp-comparison uses {keyword, domain?, url?, location?, extractTop?}."),
|
|
22738
|
+
webhookUrl: import_zod27.z.string().url().optional().describe("Optional HTTPS webhook to receive the completed hosted workflow run event.")
|
|
21546
22739
|
};
|
|
21547
22740
|
WorkflowStepInputSchema = {
|
|
21548
|
-
runId:
|
|
22741
|
+
runId: import_zod27.z.string().min(1).describe("Workflow run id returned by workflow_run, workflow_step, or workflow_status. Use only a returned runId; do not construct one yourself. Advances the run by exactly one step (one logical leg, e.g. one live harvest).")
|
|
21549
22742
|
};
|
|
21550
22743
|
WorkflowStatusInputSchema = {
|
|
21551
|
-
runId:
|
|
22744
|
+
runId: import_zod27.z.string().min(1).describe("Workflow run id returned by workflow_run, workflow_step, or workflow_status. Use only a returned runId; do not construct one yourself.")
|
|
21552
22745
|
};
|
|
21553
22746
|
WorkflowArtifactReadInputSchema = {
|
|
21554
|
-
runId:
|
|
21555
|
-
artifactId:
|
|
21556
|
-
maxBytes:
|
|
22747
|
+
runId: import_zod27.z.string().min(1).describe("Workflow run id returned by workflow_run, workflow_step, or workflow_status. Use only a returned runId; do not construct one yourself."),
|
|
22748
|
+
artifactId: import_zod27.z.string().min(1).describe("Artifact id from the run artifact list returned by workflow_run, workflow_step, or workflow_status. Use only a returned artifactId; do not construct one yourself."),
|
|
22749
|
+
maxBytes: import_zod27.z.number().int().min(1e3).max(1e6).default(2e5).describe("Maximum bytes of artifact text to return inline. Use lower values for large CSV/JSON artifacts; call again with the downloadUrl if needed outside MCP.")
|
|
21557
22750
|
};
|
|
21558
|
-
WorkflowRecipeOutput =
|
|
21559
|
-
id:
|
|
21560
|
-
title:
|
|
21561
|
-
description:
|
|
21562
|
-
primaryWorkflowId:
|
|
21563
|
-
recommendedTools:
|
|
21564
|
-
requiredInputs:
|
|
21565
|
-
optionalInputs:
|
|
21566
|
-
produces:
|
|
21567
|
-
runHint:
|
|
22751
|
+
WorkflowRecipeOutput = import_zod27.z.object({
|
|
22752
|
+
id: import_zod27.z.string(),
|
|
22753
|
+
title: import_zod27.z.string(),
|
|
22754
|
+
description: import_zod27.z.string(),
|
|
22755
|
+
primaryWorkflowId: import_zod27.z.string().nullable(),
|
|
22756
|
+
recommendedTools: import_zod27.z.array(import_zod27.z.string()),
|
|
22757
|
+
requiredInputs: import_zod27.z.array(import_zod27.z.string()),
|
|
22758
|
+
optionalInputs: import_zod27.z.array(import_zod27.z.string()),
|
|
22759
|
+
produces: import_zod27.z.array(import_zod27.z.string()),
|
|
22760
|
+
runHint: import_zod27.z.string()
|
|
21568
22761
|
});
|
|
21569
|
-
WorkflowDefinitionOutput =
|
|
21570
|
-
id:
|
|
21571
|
-
title:
|
|
21572
|
-
description:
|
|
22762
|
+
WorkflowDefinitionOutput = import_zod27.z.object({
|
|
22763
|
+
id: import_zod27.z.string(),
|
|
22764
|
+
title: import_zod27.z.string(),
|
|
22765
|
+
description: import_zod27.z.string()
|
|
21573
22766
|
});
|
|
21574
|
-
WorkflowArtifactOutput =
|
|
22767
|
+
WorkflowArtifactOutput = import_zod27.z.record(import_zod27.z.unknown());
|
|
21575
22768
|
WorkflowListOutputSchema = {
|
|
21576
|
-
workflows:
|
|
21577
|
-
recipes:
|
|
22769
|
+
workflows: import_zod27.z.array(WorkflowDefinitionOutput),
|
|
22770
|
+
recipes: import_zod27.z.array(WorkflowRecipeOutput)
|
|
21578
22771
|
};
|
|
21579
22772
|
WorkflowSuggestOutputSchema = {
|
|
21580
|
-
goal:
|
|
21581
|
-
suggestions:
|
|
22773
|
+
goal: import_zod27.z.string(),
|
|
22774
|
+
suggestions: import_zod27.z.array(WorkflowRecipeOutput)
|
|
21582
22775
|
};
|
|
21583
22776
|
WorkflowRunOutputSchema = {
|
|
21584
|
-
workflowId:
|
|
21585
|
-
input:
|
|
21586
|
-
run:
|
|
21587
|
-
summary:
|
|
21588
|
-
step:
|
|
21589
|
-
nextStep:
|
|
21590
|
-
done:
|
|
21591
|
-
artifacts:
|
|
22777
|
+
workflowId: import_zod27.z.string(),
|
|
22778
|
+
input: import_zod27.z.record(import_zod27.z.unknown()),
|
|
22779
|
+
run: import_zod27.z.record(import_zod27.z.unknown()).optional(),
|
|
22780
|
+
summary: import_zod27.z.record(import_zod27.z.unknown()).optional(),
|
|
22781
|
+
step: import_zod27.z.record(import_zod27.z.unknown()).optional(),
|
|
22782
|
+
nextStep: import_zod27.z.record(import_zod27.z.unknown()).nullable().optional(),
|
|
22783
|
+
done: import_zod27.z.boolean().optional(),
|
|
22784
|
+
artifacts: import_zod27.z.array(WorkflowArtifactOutput)
|
|
21592
22785
|
};
|
|
21593
22786
|
WorkflowStepOutputSchema = {
|
|
21594
|
-
runId:
|
|
21595
|
-
run:
|
|
21596
|
-
summary:
|
|
21597
|
-
step:
|
|
21598
|
-
nextStep:
|
|
21599
|
-
done:
|
|
21600
|
-
artifacts:
|
|
22787
|
+
runId: import_zod27.z.string(),
|
|
22788
|
+
run: import_zod27.z.record(import_zod27.z.unknown()).optional(),
|
|
22789
|
+
summary: import_zod27.z.record(import_zod27.z.unknown()).nullable().optional(),
|
|
22790
|
+
step: import_zod27.z.record(import_zod27.z.unknown()).optional(),
|
|
22791
|
+
nextStep: import_zod27.z.record(import_zod27.z.unknown()).nullable().optional(),
|
|
22792
|
+
done: import_zod27.z.boolean(),
|
|
22793
|
+
artifacts: import_zod27.z.array(WorkflowArtifactOutput)
|
|
21601
22794
|
};
|
|
21602
22795
|
WorkflowStatusOutputSchema = {
|
|
21603
|
-
run:
|
|
21604
|
-
artifacts:
|
|
22796
|
+
run: import_zod27.z.record(import_zod27.z.unknown()).optional(),
|
|
22797
|
+
artifacts: import_zod27.z.array(WorkflowArtifactOutput)
|
|
21605
22798
|
};
|
|
21606
22799
|
WorkflowArtifactReadOutputSchema = {
|
|
21607
|
-
runId:
|
|
21608
|
-
artifactId:
|
|
21609
|
-
contentType:
|
|
21610
|
-
bytes:
|
|
21611
|
-
truncated:
|
|
21612
|
-
text:
|
|
22800
|
+
runId: import_zod27.z.string(),
|
|
22801
|
+
artifactId: import_zod27.z.string(),
|
|
22802
|
+
contentType: import_zod27.z.string(),
|
|
22803
|
+
bytes: import_zod27.z.number().int().min(0),
|
|
22804
|
+
truncated: import_zod27.z.boolean(),
|
|
22805
|
+
text: import_zod27.z.string()
|
|
21613
22806
|
};
|
|
21614
22807
|
SearchSerpInputSchema = {
|
|
21615
|
-
query:
|
|
21616
|
-
location:
|
|
21617
|
-
gl:
|
|
21618
|
-
hl:
|
|
21619
|
-
device:
|
|
21620
|
-
proxyMode:
|
|
21621
|
-
proxyZip:
|
|
21622
|
-
debug:
|
|
21623
|
-
pages:
|
|
22808
|
+
query: import_zod27.z.string().min(1).describe('Core search topic only. Separate location when possible. If user says "best dentist in Brooklyn NY serp", use query="best dentist" and location="Brooklyn, NY".'),
|
|
22809
|
+
location: import_zod27.z.string().optional().describe("City, region, or country for geo-targeted results, inferred from user request when present."),
|
|
22810
|
+
gl: import_zod27.z.string().length(2).default("us").describe("Google country code inferred from location or user language."),
|
|
22811
|
+
hl: import_zod27.z.string().default("en").describe("Google interface/content language inferred from user request."),
|
|
22812
|
+
device: import_zod27.z.enum(["desktop", "mobile"]).default("desktop").describe("SERP device context. Use desktop by default; use mobile only when the user asks for mobile rankings."),
|
|
22813
|
+
proxyMode: import_zod27.z.enum(["location", "configured", "none"]).default(DEFAULT_PROXY_MODE).describe("Proxy targeting mode. Default configured uses the service proxy without city/ZIP targeting for the highest general SERP success rate. Use location only when the user explicitly needs city/ZIP-targeted residential proxy evidence. Use none only for direct-network debugging."),
|
|
22814
|
+
proxyZip: import_zod27.z.string().regex(/^\d{5}$/).optional().describe("Optional US ZIP override for residential location proxy targeting. Use only with proxyMode location when the user gives a specific ZIP or city-center targeting needs to be forced."),
|
|
22815
|
+
debug: import_zod27.z.boolean().default(false).describe("Include sanitized browser/session/location diagnostics in the response. Use true when debugging localization, CAPTCHA, or proxy behavior."),
|
|
22816
|
+
pages: import_zod27.z.number().int().min(1).max(2).default(1).describe("Number of result pages to fetch (1\u20132)")
|
|
21624
22817
|
};
|
|
21625
22818
|
CaptureSerpSnapshotInputSchema = {
|
|
21626
|
-
query:
|
|
21627
|
-
location:
|
|
21628
|
-
gl:
|
|
21629
|
-
hl:
|
|
21630
|
-
device:
|
|
21631
|
-
proxyMode:
|
|
21632
|
-
proxyZip:
|
|
21633
|
-
pages:
|
|
21634
|
-
debug:
|
|
21635
|
-
includePageSnapshots:
|
|
21636
|
-
pageSnapshotLimit:
|
|
22819
|
+
query: import_zod27.z.string().min(1).describe("Core search query to capture as a structured SERP Intelligence snapshot. Separate the place into location when the user gives a city, region, country, or ZIP."),
|
|
22820
|
+
location: import_zod27.z.string().optional().describe("City, region, country, or service area used for localized Google results. MCP Scraper records location evidence; UULE alone is not proof of localization."),
|
|
22821
|
+
gl: import_zod27.z.string().length(2).default("us").describe("Google country code inferred from the requested market, e.g. us, gb, ca, au."),
|
|
22822
|
+
hl: import_zod27.z.string().default("en").describe("Google interface/content language inferred from the user request."),
|
|
22823
|
+
device: import_zod27.z.enum(["desktop", "mobile"]).default("desktop").describe("SERP device context. Use mobile only when the user asks for mobile rankings or mobile SERP evidence."),
|
|
22824
|
+
proxyMode: import_zod27.z.enum(["location", "configured", "none"]).default(DEFAULT_PROXY_MODE).describe("Proxy behavior for capture. Default configured uses the service proxy without city/ZIP targeting for the highest general capture success rate. Use location only when the user explicitly needs city/ZIP-targeted residential proxy evidence. Use none only for direct-network debugging."),
|
|
22825
|
+
proxyZip: import_zod27.z.string().regex(/^\d{5}$/).optional().describe("Optional US ZIP override for residential location proxy targeting. Use only with proxyMode location when a precise city-center or ZIP proxy is needed."),
|
|
22826
|
+
pages: import_zod27.z.number().int().min(1).max(2).default(1).describe("Number of Google result pages to capture. Use 1 normally and 2 only when the user needs deeper ranking evidence."),
|
|
22827
|
+
debug: import_zod27.z.boolean().default(false).describe("Include sanitized browser, proxy, and location diagnostics. Use true when debugging localization, CAPTCHA, proxy selection, or capture reliability."),
|
|
22828
|
+
includePageSnapshots: import_zod27.z.boolean().default(false).describe("Also capture ranking-page snapshots for selected SERP URLs through the same product capture path."),
|
|
22829
|
+
pageSnapshotLimit: import_zod27.z.number().int().min(0).max(10).default(0).describe("Maximum ranking-page snapshots to capture when includePageSnapshots is true. Use 0 when only SERP evidence is needed.")
|
|
21637
22830
|
};
|
|
21638
22831
|
ScreenshotInputSchema = {
|
|
21639
|
-
url:
|
|
21640
|
-
device:
|
|
21641
|
-
allowLocal:
|
|
22832
|
+
url: import_zod27.z.string().url().describe("URL to capture as a full-page screenshot. Use http or https. Pass allowLocal: true to capture localhost or private-network URLs during development."),
|
|
22833
|
+
device: import_zod27.z.enum(["desktop", "mobile"]).default("desktop").describe("Viewport profile. desktop = 1440\xD7900. mobile = 390\xD7844. Use desktop by default; use mobile when the user asks for a mobile view."),
|
|
22834
|
+
allowLocal: import_zod27.z.boolean().default(false).describe("Allow localhost and private-network URLs (127.x, 192.168.x, 10.x, etc.). For local development only \u2014 not for production use.")
|
|
21642
22835
|
};
|
|
21643
22836
|
CaptureSerpPageSnapshotsInputSchema = {
|
|
21644
|
-
urls:
|
|
21645
|
-
targets:
|
|
21646
|
-
url:
|
|
21647
|
-
sourceKind:
|
|
21648
|
-
sourcePosition:
|
|
22837
|
+
urls: import_zod27.z.array(import_zod27.z.string().url()).min(1).max(25).describe("Public HTTP/HTTPS URLs to capture as SERP Intelligence page snapshots. Do not pass localhost, private IPs, file URLs, or internal admin URLs."),
|
|
22838
|
+
targets: import_zod27.z.array(import_zod27.z.object({
|
|
22839
|
+
url: import_zod27.z.string().url().describe("Public HTTP/HTTPS URL to capture."),
|
|
22840
|
+
sourceKind: import_zod27.z.enum(["organic", "ai_citation", "local_pack_website", "configured_target", "site_subject"]).default("configured_target").describe("Why this page is being captured for SERP Intelligence evidence."),
|
|
22841
|
+
sourcePosition: import_zod27.z.number().int().min(1).optional().describe("Ranking or citation position when the page came from SERP evidence.")
|
|
21649
22842
|
}).strict()).min(1).max(25).optional().describe("Structured page snapshot targets. Use this instead of urls when source kind or position should be preserved."),
|
|
21650
|
-
maxConcurrency:
|
|
21651
|
-
timeoutMs:
|
|
21652
|
-
debug:
|
|
22843
|
+
maxConcurrency: import_zod27.z.number().int().min(1).max(5).default(2).describe("Parallel page captures. Use 2 normally; higher values can increase proxy/browser pressure."),
|
|
22844
|
+
timeoutMs: import_zod27.z.number().int().min(1e3).max(6e4).default(15e3).describe("Per-page capture timeout in milliseconds. Increase for slow pages; timeout artifacts are returned as structured capture failures."),
|
|
22845
|
+
debug: import_zod27.z.boolean().default(false).describe("Include sanitized browser/proxy diagnostics for page snapshot debugging. Use true for capture, network, or proxy troubleshooting.")
|
|
21653
22846
|
};
|
|
21654
22847
|
}
|
|
21655
22848
|
});
|
|
@@ -22004,8 +23197,8 @@ function localPlanningToolAnnotations(title) {
|
|
|
22004
23197
|
}
|
|
22005
23198
|
function listSavedReports() {
|
|
22006
23199
|
try {
|
|
22007
|
-
const dir =
|
|
22008
|
-
return (0,
|
|
23200
|
+
const dir = outputBaseDir2();
|
|
23201
|
+
return (0, import_node_fs8.readdirSync)(dir).filter((f) => f.endsWith(".md")).map((f) => ({ filename: f, mtimeMs: (0, import_node_fs8.statSync)((0, import_node_path12.join)(dir, f)).mtimeMs })).sort((a, b) => b.mtimeMs - a.mtimeMs).slice(0, 100);
|
|
22009
23202
|
} catch {
|
|
22010
23203
|
return [];
|
|
22011
23204
|
}
|
|
@@ -22029,9 +23222,9 @@ function registerSavedReportResources(server) {
|
|
|
22029
23222
|
},
|
|
22030
23223
|
async (uri, variables) => {
|
|
22031
23224
|
const requested = Array.isArray(variables.filename) ? variables.filename[0] : variables.filename;
|
|
22032
|
-
const filename = (0,
|
|
23225
|
+
const filename = (0, import_node_path12.basename)(decodeURIComponent(String(requested ?? "")));
|
|
22033
23226
|
if (!filename.endsWith(".md")) throw new Error("Only saved .md reports can be read");
|
|
22034
|
-
const text = (0,
|
|
23227
|
+
const text = (0, import_node_fs8.readFileSync)((0, import_node_path12.join)(outputBaseDir2(), filename), "utf8");
|
|
22035
23228
|
return { contents: [{ uri: uri.href, mimeType: "text/markdown", text }] };
|
|
22036
23229
|
}
|
|
22037
23230
|
);
|
|
@@ -22123,6 +23316,20 @@ function registerPaaExtractorMcpTools(server, executor, options = {}) {
|
|
|
22123
23316
|
outputSchema: FacebookVideoTranscribeOutputSchema,
|
|
22124
23317
|
annotations: liveWebToolAnnotations("Facebook Organic Video Transcription")
|
|
22125
23318
|
}, async (input) => formatFacebookVideoTranscribe(await executor.facebookVideoTranscribe(input), input));
|
|
23319
|
+
server.registerTool("instagram_profile_content", {
|
|
23320
|
+
title: "Instagram Profile Content Discovery",
|
|
23321
|
+
description: withReportNote("Discover Instagram profile grid content links for a handle or profile URL. Use this when the user wants a person or brand account content inventory before selecting posts/reels to download. Returns profile stats, collected post/reel/tv URLs, shortcodes, type counts, browser mode, pagination attempts, stop reason, and limitations. For authenticated collection, first use browser_profile_import/sync, then pass browserProfile and raise maxItems/maxScrolls for longer profile pagination."),
|
|
23322
|
+
inputSchema: InstagramProfileContentInputSchema,
|
|
23323
|
+
outputSchema: InstagramProfileContentOutputSchema,
|
|
23324
|
+
annotations: liveWebToolAnnotations("Instagram Profile Content Discovery")
|
|
23325
|
+
}, async (input) => formatInstagramProfileContent(await executor.instagramProfileContent(input), input));
|
|
23326
|
+
server.registerTool("instagram_media_download", {
|
|
23327
|
+
title: "Instagram Post/Reel Media Download",
|
|
23328
|
+
description: withReportNote("Extract and download media from one Instagram post, reel, or tv URL. Use after instagram_profile_content or when the user gives a specific Instagram URL and wants the image, caption/text, reel audio/video tracks, optional muxed MP4, or optional transcript. Reels commonly expose separate video-only and audio-only MP4 tracks; this tool selects the best video and audio tracks and attempts muxing when ffmpeg is available. Pass browserProfile from browser_profile_import/sync when the media is login-gated."),
|
|
23329
|
+
inputSchema: InstagramMediaDownloadInputSchema,
|
|
23330
|
+
outputSchema: InstagramMediaDownloadOutputSchema,
|
|
23331
|
+
annotations: liveWebToolAnnotations("Instagram Post/Reel Media Download")
|
|
23332
|
+
}, async (input) => formatInstagramMediaDownload(await executor.instagramMediaDownload(input), input));
|
|
22126
23333
|
server.registerTool("maps_place_intel", {
|
|
22127
23334
|
title: "Google Maps Business Profile Details",
|
|
22128
23335
|
description: withReportNote('Extract Google Maps business intelligence for one known/named business: rating, review count, category, address, phone, website, hours, booking URL, review histogram, review topics, about attributes, entity IDs, and optional review cards. Do not use this for category searches, local market prospect lists, or requests for multiple GMB/GBP profiles; use maps_search first for those. Split business name from location (e.g. "Elite Roofing Denver CO" => businessName "Elite Roofing", location "Denver, CO"). Pass includeReviews true when the user asks for reviews/customer pain.'),
|
|
@@ -22207,13 +23414,13 @@ function registerPaaExtractorMcpTools(server, executor, options = {}) {
|
|
|
22207
23414
|
}
|
|
22208
23415
|
}, async (input) => formatCreditsInfo(await executor.creditsInfo(input), input));
|
|
22209
23416
|
}
|
|
22210
|
-
var import_mcp,
|
|
23417
|
+
var import_mcp, import_node_fs8, import_node_path12;
|
|
22211
23418
|
var init_paa_mcp_server = __esm({
|
|
22212
23419
|
"src/mcp/paa-mcp-server.ts"() {
|
|
22213
23420
|
"use strict";
|
|
22214
23421
|
import_mcp = require("@modelcontextprotocol/sdk/server/mcp.js");
|
|
22215
|
-
|
|
22216
|
-
|
|
23422
|
+
import_node_fs8 = require("fs");
|
|
23423
|
+
import_node_path12 = require("path");
|
|
22217
23424
|
init_version();
|
|
22218
23425
|
init_mcp_response_formatter();
|
|
22219
23426
|
init_mcp_tool_schemas();
|
|
@@ -22402,6 +23609,12 @@ var init_http_mcp_tool_executor = __esm({
|
|
|
22402
23609
|
facebookVideoTranscribe(input) {
|
|
22403
23610
|
return this.call("/facebook/video-transcribe", input, this.httpTimeoutOverrideMs ?? 24e4);
|
|
22404
23611
|
}
|
|
23612
|
+
instagramProfileContent(input) {
|
|
23613
|
+
return this.call("/instagram/profile-content", input, this.httpTimeoutOverrideMs ?? 24e4);
|
|
23614
|
+
}
|
|
23615
|
+
instagramMediaDownload(input) {
|
|
23616
|
+
return this.call("/instagram/media-download", input, this.httpTimeoutOverrideMs ?? 3e5);
|
|
23617
|
+
}
|
|
22405
23618
|
mapsPlaceIntel(input) {
|
|
22406
23619
|
return this.call("/maps/place", input);
|
|
22407
23620
|
}
|
|
@@ -22495,11 +23708,11 @@ function registerSerpIntelligenceCaptureTools(server, executor) {
|
|
|
22495
23708
|
annotations: liveWebToolAnnotations("SERP Intelligence Page Snapshots")
|
|
22496
23709
|
}, async (input) => formatCaptureSerpPageSnapshots(await executor.captureSerpPageSnapshots(input), input));
|
|
22497
23710
|
}
|
|
22498
|
-
var
|
|
23711
|
+
var import_hono10, import_webStandardStreamableHttp, mcpApp;
|
|
22499
23712
|
var init_mcp_routes = __esm({
|
|
22500
23713
|
"src/mcp/mcp-routes.ts"() {
|
|
22501
23714
|
"use strict";
|
|
22502
|
-
|
|
23715
|
+
import_hono10 = require("hono");
|
|
22503
23716
|
import_webStandardStreamableHttp = require("@modelcontextprotocol/sdk/server/webStandardStreamableHttp.js");
|
|
22504
23717
|
init_paa_mcp_server();
|
|
22505
23718
|
init_http_mcp_tool_executor();
|
|
@@ -22507,7 +23720,7 @@ var init_mcp_routes = __esm({
|
|
|
22507
23720
|
init_db();
|
|
22508
23721
|
init_mcp_tool_schemas();
|
|
22509
23722
|
configureReportSaving(false);
|
|
22510
|
-
mcpApp = new
|
|
23723
|
+
mcpApp = new import_hono10.Hono();
|
|
22511
23724
|
mcpApp.all("/", async (c) => {
|
|
22512
23725
|
try {
|
|
22513
23726
|
const keyOrError = await requireMcpCallerKey(c);
|
|
@@ -23371,19 +24584,19 @@ var init_classify = __esm({
|
|
|
23371
24584
|
});
|
|
23372
24585
|
|
|
23373
24586
|
// src/services/fanout/export.ts
|
|
23374
|
-
function
|
|
23375
|
-
return process.env.MCP_SCRAPER_OUTPUT_DIR?.trim() || (0,
|
|
24587
|
+
function outputBaseDir3() {
|
|
24588
|
+
return process.env.MCP_SCRAPER_OUTPUT_DIR?.trim() || (0, import_node_path13.join)((0, import_node_os8.homedir)(), "Downloads", "mcp-scraper");
|
|
23376
24589
|
}
|
|
23377
24590
|
function safe(value) {
|
|
23378
24591
|
return value.replace(/[^a-zA-Z0-9._-]+/g, "-").replace(/^-+|-+$/g, "").slice(0, 80) || "capture";
|
|
23379
24592
|
}
|
|
23380
24593
|
function writeTable(path6, rows, delimiter) {
|
|
23381
|
-
(0,
|
|
24594
|
+
(0, import_node_fs9.writeFileSync)(path6, import_papaparse3.default.unparse(rows.length ? rows : [{}], { delimiter }));
|
|
23382
24595
|
}
|
|
23383
24596
|
function exportFanout(enriched) {
|
|
23384
24597
|
const stamp = safe(enriched.capturedAt.replace(/[:.]/g, "-"));
|
|
23385
|
-
const dir = (0,
|
|
23386
|
-
(0,
|
|
24598
|
+
const dir = (0, import_node_path13.join)(outputBaseDir3(), "fanout", `${stamp}-${safe(enriched.platform)}`);
|
|
24599
|
+
(0, import_node_fs9.mkdirSync)(dir, { recursive: true });
|
|
23387
24600
|
const queryRows = enriched.queries.map((q, i) => ({ index: i + 1, query: q }));
|
|
23388
24601
|
const citationRows = enriched.aggregates.citationOrder.map((c) => {
|
|
23389
24602
|
const src = enriched.citedUrls.find((s) => s.url === c.url);
|
|
@@ -23393,21 +24606,21 @@ function exportFanout(enriched) {
|
|
|
23393
24606
|
const domainRows = enriched.aggregates.topSites.map((d) => ({ domain: d.domain, count: d.count, cited: d.cited, timesCited: d.timesCited, siteType: d.siteType }));
|
|
23394
24607
|
const paths = {
|
|
23395
24608
|
dir,
|
|
23396
|
-
json: (0,
|
|
23397
|
-
queriesCsv: (0,
|
|
23398
|
-
queriesTsv: (0,
|
|
23399
|
-
citationsCsv: (0,
|
|
23400
|
-
sourcesCsv: (0,
|
|
23401
|
-
domainsCsv: (0,
|
|
23402
|
-
report: (0,
|
|
24609
|
+
json: (0, import_node_path13.join)(dir, "fanout.json"),
|
|
24610
|
+
queriesCsv: (0, import_node_path13.join)(dir, "queries.csv"),
|
|
24611
|
+
queriesTsv: (0, import_node_path13.join)(dir, "queries.tsv"),
|
|
24612
|
+
citationsCsv: (0, import_node_path13.join)(dir, "citations.csv"),
|
|
24613
|
+
sourcesCsv: (0, import_node_path13.join)(dir, "sources.csv"),
|
|
24614
|
+
domainsCsv: (0, import_node_path13.join)(dir, "domains.csv"),
|
|
24615
|
+
report: (0, import_node_path13.join)(dir, "report.html")
|
|
23403
24616
|
};
|
|
23404
|
-
(0,
|
|
24617
|
+
(0, import_node_fs9.writeFileSync)(paths.json, JSON.stringify(enriched, null, 2));
|
|
23405
24618
|
writeTable(paths.queriesCsv, queryRows, ",");
|
|
23406
24619
|
writeTable(paths.queriesTsv, queryRows, " ");
|
|
23407
24620
|
writeTable(paths.citationsCsv, citationRows, ",");
|
|
23408
24621
|
writeTable(paths.sourcesCsv, sourceRows2, ",");
|
|
23409
24622
|
writeTable(paths.domainsCsv, domainRows, ",");
|
|
23410
|
-
(0,
|
|
24623
|
+
(0, import_node_fs9.writeFileSync)(paths.report, renderReportHtml(enriched));
|
|
23411
24624
|
return paths;
|
|
23412
24625
|
}
|
|
23413
24626
|
function esc(s) {
|
|
@@ -23485,13 +24698,13 @@ render();
|
|
|
23485
24698
|
</script>
|
|
23486
24699
|
</body></html>`;
|
|
23487
24700
|
}
|
|
23488
|
-
var
|
|
24701
|
+
var import_node_fs9, import_node_os8, import_node_path13, import_papaparse3;
|
|
23489
24702
|
var init_export = __esm({
|
|
23490
24703
|
"src/services/fanout/export.ts"() {
|
|
23491
24704
|
"use strict";
|
|
23492
|
-
|
|
23493
|
-
|
|
23494
|
-
|
|
24705
|
+
import_node_fs9 = require("fs");
|
|
24706
|
+
import_node_os8 = require("os");
|
|
24707
|
+
import_node_path13 = require("path");
|
|
23495
24708
|
import_papaparse3 = __toESM(require("papaparse"), 1);
|
|
23496
24709
|
}
|
|
23497
24710
|
});
|
|
@@ -23983,7 +25196,7 @@ async function loadOpenSession(id, userId) {
|
|
|
23983
25196
|
return row;
|
|
23984
25197
|
}
|
|
23985
25198
|
function buildBrowserAgentRoutes() {
|
|
23986
|
-
const app2 = new
|
|
25199
|
+
const app2 = new import_hono11.Hono();
|
|
23987
25200
|
app2.use("*", async (c, next) => {
|
|
23988
25201
|
await migrateBrowserAgent();
|
|
23989
25202
|
return next();
|
|
@@ -24340,11 +25553,11 @@ function buildBrowserAgentRoutes() {
|
|
|
24340
25553
|
});
|
|
24341
25554
|
return app2;
|
|
24342
25555
|
}
|
|
24343
|
-
var
|
|
25556
|
+
var import_hono11, auth, DEFAULT_BROWSER_SESSION_LOCK_TTL_SECONDS;
|
|
24344
25557
|
var init_browser_agent_routes = __esm({
|
|
24345
25558
|
"src/api/browser-agent-routes.ts"() {
|
|
24346
25559
|
"use strict";
|
|
24347
|
-
|
|
25560
|
+
import_hono11 = require("hono");
|
|
24348
25561
|
init_api_auth();
|
|
24349
25562
|
init_errors();
|
|
24350
25563
|
init_db();
|
|
@@ -24561,16 +25774,16 @@ var init_browser_agent_console = __esm({
|
|
|
24561
25774
|
});
|
|
24562
25775
|
|
|
24563
25776
|
// src/api/stripe-routes.ts
|
|
24564
|
-
var import_stripe,
|
|
25777
|
+
var import_stripe, import_hono12, stripe, stripeApp;
|
|
24565
25778
|
var init_stripe_routes = __esm({
|
|
24566
25779
|
"src/api/stripe-routes.ts"() {
|
|
24567
25780
|
"use strict";
|
|
24568
25781
|
import_stripe = __toESM(require("stripe"), 1);
|
|
24569
|
-
|
|
25782
|
+
import_hono12 = require("hono");
|
|
24570
25783
|
init_db();
|
|
24571
25784
|
init_rates();
|
|
24572
25785
|
stripe = new import_stripe.default(process.env.STRIPE_SECRET_KEY, { apiVersion: "2026-02-25.clover" });
|
|
24573
|
-
stripeApp = new
|
|
25786
|
+
stripeApp = new import_hono12.Hono();
|
|
24574
25787
|
stripeApp.post("/webhooks", async (c) => {
|
|
24575
25788
|
const sig = c.req.header("stripe-signature");
|
|
24576
25789
|
const body = await c.req.text();
|
|
@@ -24671,31 +25884,31 @@ var init_site_audit_worker = __esm({
|
|
|
24671
25884
|
});
|
|
24672
25885
|
|
|
24673
25886
|
// src/api/billing-schemas.ts
|
|
24674
|
-
var
|
|
25887
|
+
var import_zod28, BillingCheckoutBodySchema, FreeCreditBreakdownSchema, BillingBalanceResponseSchema, MonthlyRefreshSweepResultSchema;
|
|
24675
25888
|
var init_billing_schemas = __esm({
|
|
24676
25889
|
"src/api/billing-schemas.ts"() {
|
|
24677
25890
|
"use strict";
|
|
24678
|
-
|
|
24679
|
-
BillingCheckoutBodySchema =
|
|
24680
|
-
priceId:
|
|
25891
|
+
import_zod28 = require("zod");
|
|
25892
|
+
BillingCheckoutBodySchema = import_zod28.z.object({
|
|
25893
|
+
priceId: import_zod28.z.string().min(1)
|
|
24681
25894
|
});
|
|
24682
|
-
FreeCreditBreakdownSchema =
|
|
24683
|
-
signup_grant_mc:
|
|
24684
|
-
monthly_refresh_mc:
|
|
24685
|
-
total_free_mc:
|
|
24686
|
-
signup_grant_credits:
|
|
24687
|
-
monthly_refresh_credits:
|
|
24688
|
-
total_free_credits:
|
|
25895
|
+
FreeCreditBreakdownSchema = import_zod28.z.object({
|
|
25896
|
+
signup_grant_mc: import_zod28.z.number().int().nonnegative(),
|
|
25897
|
+
monthly_refresh_mc: import_zod28.z.number().int().nonnegative(),
|
|
25898
|
+
total_free_mc: import_zod28.z.number().int().nonnegative(),
|
|
25899
|
+
signup_grant_credits: import_zod28.z.number().nonnegative(),
|
|
25900
|
+
monthly_refresh_credits: import_zod28.z.number().nonnegative(),
|
|
25901
|
+
total_free_credits: import_zod28.z.number().nonnegative()
|
|
24689
25902
|
});
|
|
24690
|
-
BillingBalanceResponseSchema =
|
|
24691
|
-
balance_mc:
|
|
24692
|
-
balance_credits:
|
|
25903
|
+
BillingBalanceResponseSchema = import_zod28.z.object({
|
|
25904
|
+
balance_mc: import_zod28.z.number().int().nonnegative(),
|
|
25905
|
+
balance_credits: import_zod28.z.number().nonnegative(),
|
|
24693
25906
|
free_credits: FreeCreditBreakdownSchema,
|
|
24694
|
-
ledger:
|
|
25907
|
+
ledger: import_zod28.z.array(import_zod28.z.any())
|
|
24695
25908
|
});
|
|
24696
|
-
MonthlyRefreshSweepResultSchema =
|
|
24697
|
-
usersRefreshed:
|
|
24698
|
-
totalMcGranted:
|
|
25909
|
+
MonthlyRefreshSweepResultSchema = import_zod28.z.object({
|
|
25910
|
+
usersRefreshed: import_zod28.z.number().int().nonnegative(),
|
|
25911
|
+
totalMcGranted: import_zod28.z.number().int().nonnegative()
|
|
24699
25912
|
});
|
|
24700
25913
|
}
|
|
24701
25914
|
});
|
|
@@ -25106,7 +26319,7 @@ async function checkHarvestLimits(user, reuseLockId) {
|
|
|
25106
26319
|
}
|
|
25107
26320
|
return null;
|
|
25108
26321
|
}
|
|
25109
|
-
var import_resend,
|
|
26322
|
+
var import_resend, import_hono13, import_hono14, import_factory6, import_cookie, import_stripe2, secureCookies, isProduction2, sessionCookieOptions, requireAllowedOrigin, auth2, adminAuth, sessionAuth, app, STRIPE_API_VERSION, SYNC_HARVEST_TIMEOUT_OVERRIDE_MS;
|
|
25110
26323
|
var init_server = __esm({
|
|
25111
26324
|
"src/api/server.ts"() {
|
|
25112
26325
|
"use strict";
|
|
@@ -25124,14 +26337,15 @@ var init_server = __esm({
|
|
|
25124
26337
|
init_media_extractor();
|
|
25125
26338
|
init_site_mapper();
|
|
25126
26339
|
init_site_extractor();
|
|
25127
|
-
|
|
25128
|
-
|
|
26340
|
+
import_hono13 = require("hono");
|
|
26341
|
+
import_hono14 = require("inngest/hono");
|
|
25129
26342
|
init_client();
|
|
25130
26343
|
init_site_audit();
|
|
25131
26344
|
init_site_audit_routes();
|
|
25132
26345
|
init_youtube_routes();
|
|
25133
26346
|
init_screenshot_routes();
|
|
25134
26347
|
init_facebook_ad_routes();
|
|
26348
|
+
init_instagram_routes();
|
|
25135
26349
|
init_maps_routes();
|
|
25136
26350
|
init_directory_routes();
|
|
25137
26351
|
init_workflow_routes();
|
|
@@ -25199,7 +26413,7 @@ var init_server = __esm({
|
|
|
25199
26413
|
c.set("sessionUser", { ...refreshed, balance_mc: balanceMc });
|
|
25200
26414
|
return next();
|
|
25201
26415
|
});
|
|
25202
|
-
app = new
|
|
26416
|
+
app = new import_hono13.Hono();
|
|
25203
26417
|
STRIPE_API_VERSION = "2026-02-25.clover";
|
|
25204
26418
|
app.use("*", async (c, next) => {
|
|
25205
26419
|
await next();
|
|
@@ -25578,7 +26792,10 @@ var init_server = __esm({
|
|
|
25578
26792
|
const device = screenshotDevice === "mobile" ? "mobile" : "desktop";
|
|
25579
26793
|
const [result, pageData] = await Promise.all([
|
|
25580
26794
|
extractKpo({ url: canonicalUrl, kernelApiKey }),
|
|
25581
|
-
screenshot2 || extractBranding ? capturePageData(canonicalUrl, { kernelApiKey, device, screenshot: !!screenshot2, branding: !!extractBranding }).catch(() =>
|
|
26795
|
+
screenshot2 || extractBranding ? capturePageData(canonicalUrl, { kernelApiKey, device, screenshot: !!screenshot2, branding: !!extractBranding }).catch((err) => {
|
|
26796
|
+
console.error("[extract-url] capturePageData failed:", err instanceof Error ? err.message : err);
|
|
26797
|
+
return null;
|
|
26798
|
+
}) : null
|
|
25582
26799
|
]);
|
|
25583
26800
|
const screenshotBuf = pageData?.screenshot ?? null;
|
|
25584
26801
|
const brandingData = pageData?.branding ?? null;
|
|
@@ -25586,7 +26803,7 @@ var init_server = __esm({
|
|
|
25586
26803
|
if (screenshotBuf) {
|
|
25587
26804
|
screenshotMeta = { base64: screenshotBuf.toString("base64"), sizeBytes: screenshotBuf.length, device };
|
|
25588
26805
|
}
|
|
25589
|
-
const mediaMeta = downloadMedia ? await harvestPageMedia(result.bodyHtml, canonicalUrl, { types: mediaTypes ?? ["image", "video", "audio"] }) : null;
|
|
26806
|
+
const mediaMeta = downloadMedia ? await harvestPageMedia(result.bodyHtml, canonicalUrl, { types: mediaTypes ?? ["image", "video", "audio"], outputDir: null }) : null;
|
|
25590
26807
|
await logRequestEvent({ userId: user.id, source: "extract_url", status: "done", query: canonicalUrl, resultCount: result.headings.length, result });
|
|
25591
26808
|
return c.json({ ...result, screenshot: screenshotMeta, branding: brandingData, media: mediaMeta });
|
|
25592
26809
|
} catch (err) {
|
|
@@ -25889,11 +27106,12 @@ var init_server = __esm({
|
|
|
25889
27106
|
]);
|
|
25890
27107
|
return c.json({ drained: results.length, results, sweepResult, workflowDispatch: workflowDispatchResult });
|
|
25891
27108
|
});
|
|
25892
|
-
app.on(["GET", "POST", "PUT"], "/api/inngest", (0,
|
|
27109
|
+
app.on(["GET", "POST", "PUT"], "/api/inngest", (0, import_hono14.serve)({ client: inngest, functions: [siteAuditFn] }));
|
|
25893
27110
|
app.route("/api/internal/site-architecture-auditor", siteAuditApp);
|
|
25894
27111
|
app.route("/youtube", youtubeApp);
|
|
25895
27112
|
app.route("/screenshot", screenshotApp);
|
|
25896
27113
|
app.route("/facebook", facebookAdApp);
|
|
27114
|
+
app.route("/instagram", instagramApp);
|
|
25897
27115
|
app.route("/maps", mapsApp);
|
|
25898
27116
|
app.route("/directory", directoryApp);
|
|
25899
27117
|
app.route("/workflows", workflowApp);
|
|
@@ -26010,10 +27228,10 @@ ${ATTRIBUTION_PIXEL_SCRIPT}
|
|
|
26010
27228
|
});
|
|
26011
27229
|
|
|
26012
27230
|
// bin/api-server.ts
|
|
26013
|
-
var
|
|
27231
|
+
var import_node_fs10 = require("fs");
|
|
26014
27232
|
function loadDotEnv() {
|
|
26015
27233
|
try {
|
|
26016
|
-
for (const line of (0,
|
|
27234
|
+
for (const line of (0, import_node_fs10.readFileSync)(".env", "utf8").split("\n")) {
|
|
26017
27235
|
const eq = line.indexOf("=");
|
|
26018
27236
|
if (eq < 1 || line.trimStart().startsWith("#")) continue;
|
|
26019
27237
|
const k = line.slice(0, eq).trim();
|