mcp-scraper 0.3.0 → 0.3.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +4 -1
- package/dist/bin/api-server.cjs +2047 -831
- package/dist/bin/api-server.cjs.map +1 -1
- package/dist/bin/api-server.js +2 -2
- package/dist/bin/browser-agent-stdio-server.cjs +3 -3
- package/dist/bin/browser-agent-stdio-server.cjs.map +1 -1
- package/dist/bin/browser-agent-stdio-server.js +2 -2
- package/dist/bin/mcp-scraper-cli.cjs +1 -1
- package/dist/bin/mcp-scraper-cli.cjs.map +1 -1
- package/dist/bin/mcp-scraper-cli.js +1 -1
- package/dist/bin/mcp-scraper-combined-stdio-server.cjs +350 -5
- package/dist/bin/mcp-scraper-combined-stdio-server.cjs.map +1 -1
- package/dist/bin/mcp-scraper-combined-stdio-server.js +4 -4
- package/dist/bin/mcp-scraper-install.cjs +3 -3
- package/dist/bin/mcp-scraper-install.cjs.map +1 -1
- package/dist/bin/mcp-scraper-install.js +2 -2
- package/dist/bin/mcp-stdio-server.cjs +346 -1
- package/dist/bin/mcp-stdio-server.cjs.map +1 -1
- package/dist/bin/mcp-stdio-server.js +2 -2
- package/dist/bin/paa-harvest.cjs +3 -1
- package/dist/bin/paa-harvest.cjs.map +1 -1
- package/dist/bin/paa-harvest.js +1 -1
- package/dist/{chunk-SXTXMFEQ.js → chunk-AZ5PKU4F.js} +34 -1
- package/dist/chunk-AZ5PKU4F.js.map +1 -0
- package/dist/{chunk-Q4STSM63.js → chunk-C3FGVJWH.js} +3 -3
- package/dist/chunk-C3FGVJWH.js.map +1 -0
- package/dist/{chunk-DH4H3F6S.js → chunk-HRTCMLZB.js} +4 -4
- package/dist/chunk-HRTCMLZB.js.map +1 -0
- package/dist/{chunk-SIXJ6EVS.js → chunk-KO223PSJ.js} +347 -2
- package/dist/chunk-KO223PSJ.js.map +1 -0
- package/dist/{chunk-IPW4LFOT.js → chunk-UWSG3C5J.js} +4 -2
- package/dist/chunk-UWSG3C5J.js.map +1 -0
- package/dist/chunk-VB65G7PS.js +7 -0
- package/dist/chunk-VB65G7PS.js.map +1 -0
- package/dist/index.cjs +3 -1
- package/dist/index.cjs.map +1 -1
- package/dist/index.js +1 -1
- package/dist/{server-MOP7NIUK.js → server-N5LM3PEO.js} +997 -253
- package/dist/server-N5LM3PEO.js.map +1 -0
- package/dist/{worker-SLQ375UG.js → worker-56IXWOQU.js} +3 -3
- package/docs/mcp-tool-craft-lint.generated.md +5 -2
- package/docs/mcp-tool-manifest.generated.json +85 -9
- package/package.json +1 -1
- package/dist/chunk-DH4H3F6S.js.map +0 -1
- package/dist/chunk-IPW4LFOT.js.map +0 -1
- package/dist/chunk-PGWJ2EJ3.js +0 -7
- package/dist/chunk-PGWJ2EJ3.js.map +0 -1
- package/dist/chunk-Q4STSM63.js.map +0 -1
- package/dist/chunk-SIXJ6EVS.js.map +0 -1
- package/dist/chunk-SXTXMFEQ.js.map +0 -1
- package/dist/server-MOP7NIUK.js.map +0 -1
- /package/dist/{worker-SLQ375UG.js.map → worker-56IXWOQU.js.map} +0 -0
package/dist/bin/api-server.cjs
CHANGED
|
@@ -4098,9 +4098,9 @@ async function extractBrandingFromPage(page) {
|
|
|
4098
4098
|
colors: {
|
|
4099
4099
|
primary: primary && !isTransparentOrWhite(primary) ? primary : null,
|
|
4100
4100
|
accent: accent && !isTransparentOrWhite(accent) ? accent : null,
|
|
4101
|
-
background: background
|
|
4102
|
-
text: text
|
|
4103
|
-
heading: heading
|
|
4101
|
+
background: background ?? null,
|
|
4102
|
+
text: text ?? null,
|
|
4103
|
+
heading: heading ?? null
|
|
4104
4104
|
},
|
|
4105
4105
|
fonts: {
|
|
4106
4106
|
body: firstFont(raw.bodyFont ?? ""),
|
|
@@ -4372,8 +4372,8 @@ async function downloadAsset(url, destDir, filename) {
|
|
|
4372
4372
|
}
|
|
4373
4373
|
const writer = (0, import_node_fs.createWriteStream)(dest);
|
|
4374
4374
|
await (0, import_promises2.pipeline)(import_node_stream.Readable.fromWeb(res.body), writer);
|
|
4375
|
-
const { statSync:
|
|
4376
|
-
const sizeBytes =
|
|
4375
|
+
const { statSync: statSync3 } = await import("fs");
|
|
4376
|
+
const sizeBytes = statSync3(dest).size;
|
|
4377
4377
|
return { savedPath: dest, sizeBytes, mimeType };
|
|
4378
4378
|
}
|
|
4379
4379
|
async function harvestPageMedia(html, pageUrl, options = {}) {
|
|
@@ -8590,8 +8590,8 @@ var init_site_audit_middleware = __esm({
|
|
|
8590
8590
|
|
|
8591
8591
|
// src/api/site-audit-routes.ts
|
|
8592
8592
|
function isPathInside(root, target) {
|
|
8593
|
-
const
|
|
8594
|
-
return
|
|
8593
|
+
const relative2 = import_node_path2.default.relative(root, target);
|
|
8594
|
+
return relative2 === "" || !relative2.startsWith("..") && !import_node_path2.default.isAbsolute(relative2);
|
|
8595
8595
|
}
|
|
8596
8596
|
var import_node_path2, import_node_os2, import_hono, import_zod11, siteAuditApp;
|
|
8597
8597
|
var init_site_audit_routes = __esm({
|
|
@@ -8827,6 +8827,9 @@ var init_rates = __esm({
|
|
|
8827
8827
|
maps_review: 50,
|
|
8828
8828
|
fb_search: 50,
|
|
8829
8829
|
fb_transcribe: 50,
|
|
8830
|
+
instagram_profile: 100,
|
|
8831
|
+
instagram_media: 100,
|
|
8832
|
+
instagram_transcribe: 50,
|
|
8830
8833
|
browser_minute: 4e3
|
|
8831
8834
|
};
|
|
8832
8835
|
MC_PER_BROWSER_MS = MC_COSTS.browser_minute / 6e4;
|
|
@@ -8927,6 +8930,30 @@ var init_rates = __esm({
|
|
|
8927
8930
|
unit: "per call",
|
|
8928
8931
|
notes: "Whisper transcription of Facebook ad video via fal.ai."
|
|
8929
8932
|
},
|
|
8933
|
+
{
|
|
8934
|
+
key: "instagram_profile",
|
|
8935
|
+
label: "Instagram profile content discovery",
|
|
8936
|
+
aliases: ["instagram_profile_content", "instagram profile", "instagram content list", "ig profile"],
|
|
8937
|
+
credits: mcToCredits(MC_COSTS.instagram_profile),
|
|
8938
|
+
unit: "per profile scan",
|
|
8939
|
+
notes: "Browser extraction of public Instagram profile grid links. Complete history may require a logged-in profile."
|
|
8940
|
+
},
|
|
8941
|
+
{
|
|
8942
|
+
key: "instagram_media",
|
|
8943
|
+
label: "Instagram media download",
|
|
8944
|
+
aliases: ["instagram_media_download", "instagram reel download", "instagram post download", "ig media"],
|
|
8945
|
+
credits: mcToCredits(MC_COSTS.instagram_media),
|
|
8946
|
+
unit: "per post or reel",
|
|
8947
|
+
notes: "Extracts post text, image URLs, and reel audio/video tracks, with local downloads when the server can write files."
|
|
8948
|
+
},
|
|
8949
|
+
{
|
|
8950
|
+
key: "instagram_transcribe",
|
|
8951
|
+
label: "Instagram media transcription",
|
|
8952
|
+
aliases: ["instagram transcript", "instagram reel transcript", "ig transcribe"],
|
|
8953
|
+
credits: mcToCredits(MC_COSTS.instagram_transcribe),
|
|
8954
|
+
unit: "per call",
|
|
8955
|
+
notes: "Whisper transcription of selected Instagram audio/video media via fal.ai."
|
|
8956
|
+
},
|
|
8930
8957
|
{
|
|
8931
8958
|
key: "browser_minute",
|
|
8932
8959
|
label: "Interactive browser session",
|
|
@@ -8985,6 +9012,12 @@ var init_rates = __esm({
|
|
|
8985
9012
|
FB_TRANSCRIBE: "fb_transcribe",
|
|
8986
9013
|
FB_SEARCH_REFUND: "fb_search_refund",
|
|
8987
9014
|
FB_TRANSCRIBE_REFUND: "fb_transcribe_refund",
|
|
9015
|
+
INSTAGRAM_PROFILE: "instagram_profile",
|
|
9016
|
+
INSTAGRAM_PROFILE_REFUND: "instagram_profile_refund",
|
|
9017
|
+
INSTAGRAM_MEDIA: "instagram_media",
|
|
9018
|
+
INSTAGRAM_MEDIA_REFUND: "instagram_media_refund",
|
|
9019
|
+
INSTAGRAM_TRANSCRIBE: "instagram_transcribe",
|
|
9020
|
+
INSTAGRAM_TRANSCRIBE_REFUND: "instagram_transcribe_refund",
|
|
8988
9021
|
BROWSER_SESSION: "browser_session"
|
|
8989
9022
|
};
|
|
8990
9023
|
}
|
|
@@ -9297,7 +9330,9 @@ var init_BrowserDriver = __esm({
|
|
|
9297
9330
|
}
|
|
9298
9331
|
const launchOpts = {
|
|
9299
9332
|
headless: config.headless,
|
|
9300
|
-
proxy: config.proxy ? { server: config.proxy } : void 0
|
|
9333
|
+
proxy: config.proxy ? { server: config.proxy } : void 0,
|
|
9334
|
+
...config.browserExecutablePath ? { executablePath: config.browserExecutablePath } : {},
|
|
9335
|
+
...config.ignoreDefaultArgs ? { ignoreDefaultArgs: config.ignoreDefaultArgs } : {}
|
|
9301
9336
|
};
|
|
9302
9337
|
const ctxOpts = rankCheckContextOptions(config);
|
|
9303
9338
|
if (config.profileDir) {
|
|
@@ -10283,8 +10318,8 @@ async function fetchViaKernelInnertube(videoId) {
|
|
|
10283
10318
|
const tracks = data?.captions?.playerCaptionsTracklistRenderer?.captionTracks ?? [];
|
|
10284
10319
|
if (tracks.length === 0) return null;
|
|
10285
10320
|
const track = tracks.find((t) => t.languageCode === "en") ?? tracks[0];
|
|
10286
|
-
const
|
|
10287
|
-
const xmlResp = await fetch(`${track.baseUrl}${
|
|
10321
|
+
const sep2 = track.baseUrl.includes("?") ? "&" : "?";
|
|
10322
|
+
const xmlResp = await fetch(`${track.baseUrl}${sep2}fmt=json3`);
|
|
10288
10323
|
return xmlResp.ok ? xmlResp.text() : null;
|
|
10289
10324
|
}, videoId);
|
|
10290
10325
|
if (!xml) return null;
|
|
@@ -12492,15 +12527,8 @@ var init_schemas3 = __esm({
|
|
|
12492
12527
|
}
|
|
12493
12528
|
});
|
|
12494
12529
|
|
|
12495
|
-
// src/
|
|
12496
|
-
function
|
|
12497
|
-
return { error_code: "invalid_request", message };
|
|
12498
|
-
}
|
|
12499
|
-
function isAllowedFacebookOrganicUrl(url) {
|
|
12500
|
-
const hostname = url.hostname.toLowerCase().replace(/^www\./, "").replace(/^m\./, "");
|
|
12501
|
-
return hostname === "facebook.com" || hostname === "fb.watch";
|
|
12502
|
-
}
|
|
12503
|
-
function facebookTranscriptMarkdown(title, text, chunks, durationMs) {
|
|
12530
|
+
// src/services/media-transcription.ts
|
|
12531
|
+
function transcriptMarkdown(title, text, chunks, durationMs) {
|
|
12504
12532
|
const fmtTs2 = (s) => `${Math.floor(s / 60)}:${String(Math.floor(s % 60)).padStart(2, "0")}`;
|
|
12505
12533
|
const lines = [title, "", `*Transcribed in ${(durationMs / 1e3).toFixed(1)}s*`, "", "## Full Text", "", text, ""];
|
|
12506
12534
|
if (chunks.length) {
|
|
@@ -12511,10 +12539,10 @@ function facebookTranscriptMarkdown(title, text, chunks, durationMs) {
|
|
|
12511
12539
|
}
|
|
12512
12540
|
return lines.join("\n");
|
|
12513
12541
|
}
|
|
12514
|
-
async function
|
|
12542
|
+
async function transcribeMediaUrl(mediaUrl, markdownTitle = "# Media Transcript") {
|
|
12515
12543
|
const startMs = Date.now();
|
|
12516
12544
|
const result = await import_client3.fal.subscribe("fal-ai/wizper", {
|
|
12517
|
-
input: { audio_url:
|
|
12545
|
+
input: { audio_url: mediaUrl, task: "transcribe", language: "en" },
|
|
12518
12546
|
logs: false,
|
|
12519
12547
|
pollInterval: 3e3
|
|
12520
12548
|
});
|
|
@@ -12526,9 +12554,28 @@ async function transcribeFacebookVideoUrl(videoUrl, markdownTitle = "# Facebook
|
|
|
12526
12554
|
text,
|
|
12527
12555
|
chunks,
|
|
12528
12556
|
durationMs,
|
|
12529
|
-
markdown:
|
|
12557
|
+
markdown: transcriptMarkdown(markdownTitle, text, chunks, durationMs)
|
|
12530
12558
|
};
|
|
12531
12559
|
}
|
|
12560
|
+
var import_client3;
|
|
12561
|
+
var init_media_transcription = __esm({
|
|
12562
|
+
"src/services/media-transcription.ts"() {
|
|
12563
|
+
"use strict";
|
|
12564
|
+
import_client3 = require("@fal-ai/client");
|
|
12565
|
+
}
|
|
12566
|
+
});
|
|
12567
|
+
|
|
12568
|
+
// src/api/facebook-ad-routes.ts
|
|
12569
|
+
function invalidRequest(message) {
|
|
12570
|
+
return { error_code: "invalid_request", message };
|
|
12571
|
+
}
|
|
12572
|
+
function isAllowedFacebookOrganicUrl(url) {
|
|
12573
|
+
const hostname = url.hostname.toLowerCase().replace(/^www\./, "").replace(/^m\./, "");
|
|
12574
|
+
return hostname === "facebook.com" || hostname === "fb.watch";
|
|
12575
|
+
}
|
|
12576
|
+
async function transcribeFacebookVideoUrl(videoUrl, markdownTitle = "# Facebook Video Transcript") {
|
|
12577
|
+
return transcribeMediaUrl(videoUrl, markdownTitle);
|
|
12578
|
+
}
|
|
12532
12579
|
async function detectSoftBlock(driver) {
|
|
12533
12580
|
const page = driver.getPage();
|
|
12534
12581
|
const bodyText = await page.evaluate(() => document.body?.innerText ?? "").catch(() => "");
|
|
@@ -12557,7 +12604,7 @@ async function kernelLaunchOptsResidential() {
|
|
|
12557
12604
|
}
|
|
12558
12605
|
return { headless: true, kernelApiKey: browserServiceApiKey(), kernelProxyId: proxyId, viewport: { width: 1280, height: 900 }, locale: "en-US" };
|
|
12559
12606
|
}
|
|
12560
|
-
var import_hono4, import_zod16,
|
|
12607
|
+
var import_hono4, import_zod16, import_client4, FacebookAdBodySchema, FacebookPageIntelBodySchema, FacebookTranscribeBodySchema, FacebookVideoTranscribeBodySchema, FacebookSearchBodySchema, FacebookMediaBodySchema, facebookAdApp, ALLOWED_MEDIA_HOSTS;
|
|
12561
12608
|
var init_facebook_ad_routes = __esm({
|
|
12562
12609
|
"src/api/facebook-ad-routes.ts"() {
|
|
12563
12610
|
"use strict";
|
|
@@ -12572,10 +12619,11 @@ var init_facebook_ad_routes = __esm({
|
|
|
12572
12619
|
init_FacebookOrganicVideoExtractor();
|
|
12573
12620
|
init_kernel_proxy_resolver();
|
|
12574
12621
|
init_schemas3();
|
|
12575
|
-
|
|
12622
|
+
import_client4 = require("@fal-ai/client");
|
|
12576
12623
|
init_api_auth();
|
|
12577
12624
|
init_url_utils();
|
|
12578
12625
|
init_concurrency_gates();
|
|
12626
|
+
init_media_transcription();
|
|
12579
12627
|
FacebookAdBodySchema = import_zod16.z.object({
|
|
12580
12628
|
url: import_zod16.z.string().trim().optional(),
|
|
12581
12629
|
libraryId: import_zod16.z.string().trim().optional(),
|
|
@@ -12714,7 +12762,7 @@ var init_facebook_ad_routes = __esm({
|
|
|
12714
12762
|
metadata: { videoUrl }
|
|
12715
12763
|
});
|
|
12716
12764
|
if (!gate.ok) return c.json(concurrencyLimitExceededResponse(gate), 429, { "Retry-After": String(gate.retryAfterSeconds) });
|
|
12717
|
-
|
|
12765
|
+
import_client4.fal.config({ credentials: process.env.FAL_KEY });
|
|
12718
12766
|
let debited = false;
|
|
12719
12767
|
try {
|
|
12720
12768
|
const { ok, balance_mc } = await debitMc(fbUser.id, MC_COSTS.fb_transcribe, LedgerOperation.FB_TRANSCRIBE, videoUrl);
|
|
@@ -12753,7 +12801,7 @@ var init_facebook_ad_routes = __esm({
|
|
|
12753
12801
|
metadata: { url: sourceUrl.href }
|
|
12754
12802
|
});
|
|
12755
12803
|
if (!gate.ok) return c.json(concurrencyLimitExceededResponse(gate), 429, { "Retry-After": String(gate.retryAfterSeconds) });
|
|
12756
|
-
|
|
12804
|
+
import_client4.fal.config({ credentials: process.env.FAL_KEY });
|
|
12757
12805
|
const driver = new BrowserDriver();
|
|
12758
12806
|
let debited = false;
|
|
12759
12807
|
try {
|
|
@@ -12930,6 +12978,824 @@ var init_facebook_ad_routes = __esm({
|
|
|
12930
12978
|
}
|
|
12931
12979
|
});
|
|
12932
12980
|
|
|
12981
|
+
// src/lib/chrome-profiles.ts
|
|
12982
|
+
function recommendedKernelProfileName(value) {
|
|
12983
|
+
return value.trim().toLowerCase().replace(/@/g, "-").replace(/[^a-z0-9]+/g, "-").replace(/^-+|-+$/g, "").slice(0, 80) || "browser-profile";
|
|
12984
|
+
}
|
|
12985
|
+
var import_promises4, import_node_os3, import_node_path5;
|
|
12986
|
+
var init_chrome_profiles = __esm({
|
|
12987
|
+
"src/lib/chrome-profiles.ts"() {
|
|
12988
|
+
"use strict";
|
|
12989
|
+
import_promises4 = require("fs/promises");
|
|
12990
|
+
import_node_os3 = require("os");
|
|
12991
|
+
import_node_path5 = require("path");
|
|
12992
|
+
}
|
|
12993
|
+
});
|
|
12994
|
+
|
|
12995
|
+
// src/lib/local-browser-profiles.ts
|
|
12996
|
+
function defaultImportedBrowserProfilesDir() {
|
|
12997
|
+
return process.env.MCP_SCRAPER_BROWSER_PROFILE_BASE_DIR?.trim() || (0, import_node_path6.join)((0, import_node_os4.homedir)(), ".mcp-scraper", "browser-profiles");
|
|
12998
|
+
}
|
|
12999
|
+
function defaultChromeExecutablePath() {
|
|
13000
|
+
const envPath = process.env.MCP_SCRAPER_BROWSER_EXECUTABLE?.trim() || process.env.CHROME_PATH?.trim();
|
|
13001
|
+
if (envPath) return envPath;
|
|
13002
|
+
const candidates = process.platform === "darwin" ? ["/Applications/Google Chrome.app/Contents/MacOS/Google Chrome"] : process.platform === "win32" ? [
|
|
13003
|
+
(0, import_node_path6.join)(process.env.PROGRAMFILES ?? "C:\\Program Files", "Google", "Chrome", "Application", "chrome.exe"),
|
|
13004
|
+
(0, import_node_path6.join)(process.env["PROGRAMFILES(X86)"] ?? "C:\\Program Files (x86)", "Google", "Chrome", "Application", "chrome.exe")
|
|
13005
|
+
] : ["/usr/bin/google-chrome", "/usr/bin/google-chrome-stable", "/usr/bin/chromium-browser", "/usr/bin/chromium"];
|
|
13006
|
+
return candidates.find((candidate) => (0, import_node_fs3.existsSync)(candidate));
|
|
13007
|
+
}
|
|
13008
|
+
function localBrowserModeEnabled() {
|
|
13009
|
+
return process.env.MCP_SCRAPER_BROWSER_MODE?.trim().toLowerCase() === "local";
|
|
13010
|
+
}
|
|
13011
|
+
function importedProfileName(value) {
|
|
13012
|
+
return recommendedKernelProfileName(value);
|
|
13013
|
+
}
|
|
13014
|
+
function importedProfileRoot(name, outputDir = defaultImportedBrowserProfilesDir()) {
|
|
13015
|
+
return (0, import_node_path6.join)(outputDir, importedProfileName(name));
|
|
13016
|
+
}
|
|
13017
|
+
function importedProfileManifestPath(name, outputDir = defaultImportedBrowserProfilesDir()) {
|
|
13018
|
+
return (0, import_node_path6.join)(importedProfileRoot(name, outputDir), "manifest.json");
|
|
13019
|
+
}
|
|
13020
|
+
async function readExistingManifest(path6) {
|
|
13021
|
+
try {
|
|
13022
|
+
return JSON.parse(await (0, import_promises5.readFile)(path6, "utf8"));
|
|
13023
|
+
} catch {
|
|
13024
|
+
return null;
|
|
13025
|
+
}
|
|
13026
|
+
}
|
|
13027
|
+
async function loadImportedBrowserProfile(name = process.env.MCP_SCRAPER_BROWSER_PROFILE?.trim() || "default", outputDir = defaultImportedBrowserProfilesDir()) {
|
|
13028
|
+
const profileName = importedProfileName(name);
|
|
13029
|
+
const manifestPath = importedProfileManifestPath(profileName, outputDir);
|
|
13030
|
+
const manifest = await readExistingManifest(manifestPath);
|
|
13031
|
+
if (!manifest) {
|
|
13032
|
+
throw new Error(
|
|
13033
|
+
`Imported browser profile "${profileName}" was not found. Run: mcp-scraper-cli browser import-chrome --name ${profileName}`
|
|
13034
|
+
);
|
|
13035
|
+
}
|
|
13036
|
+
return manifest;
|
|
13037
|
+
}
|
|
13038
|
+
var import_node_fs3, import_promises5, import_node_os4, import_node_path6;
|
|
13039
|
+
var init_local_browser_profiles = __esm({
|
|
13040
|
+
"src/lib/local-browser-profiles.ts"() {
|
|
13041
|
+
"use strict";
|
|
13042
|
+
import_node_fs3 = require("fs");
|
|
13043
|
+
import_promises5 = require("fs/promises");
|
|
13044
|
+
import_node_os4 = require("os");
|
|
13045
|
+
import_node_path6 = require("path");
|
|
13046
|
+
init_chrome_profiles();
|
|
13047
|
+
}
|
|
13048
|
+
});
|
|
13049
|
+
|
|
13050
|
+
// src/extractor/InstagramContentExtractor.ts
|
|
13051
|
+
function htmlDecode2(value) {
|
|
13052
|
+
return value.replace(/&/g, "&").replace(/"/g, '"').replace(/'/g, "'").replace(/</g, "<").replace(/>/g, ">");
|
|
13053
|
+
}
|
|
13054
|
+
function decodeEscapedValue2(raw) {
|
|
13055
|
+
let out = raw;
|
|
13056
|
+
try {
|
|
13057
|
+
out = JSON.parse(`"${raw.replace(/"/g, '\\"')}"`);
|
|
13058
|
+
} catch {
|
|
13059
|
+
out = raw.replace(/\\u0025/g, "%").replace(/\\u0026/g, "&").replace(/\\u003D/g, "=").replace(/\\u003d/g, "=").replace(/\\"/g, '"').replace(/\\\//g, "/");
|
|
13060
|
+
}
|
|
13061
|
+
return htmlDecode2(out);
|
|
13062
|
+
}
|
|
13063
|
+
function metadataContent2(html, key) {
|
|
13064
|
+
const escaped = key.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
|
13065
|
+
const re = new RegExp(`<meta\\s+[^>]*(?:property|name)=["']${escaped}["'][^>]*content=["']([^"']+)["'][^>]*>`, "i");
|
|
13066
|
+
const m = html.match(re);
|
|
13067
|
+
return m ? htmlDecode2(m[1]).trim() : null;
|
|
13068
|
+
}
|
|
13069
|
+
function instagramContentTypeFromPath(rawPath) {
|
|
13070
|
+
const match = rawPath.match(CONTENT_PATH_RE);
|
|
13071
|
+
if (!match) return null;
|
|
13072
|
+
const type = match[1].toLowerCase() === "p" ? "post" : match[1].toLowerCase();
|
|
13073
|
+
return { type, shortcode: match[2] };
|
|
13074
|
+
}
|
|
13075
|
+
function normalizeInstagramProfileInput(input) {
|
|
13076
|
+
const raw = input.trim();
|
|
13077
|
+
if (!raw) throw new Error("Instagram handle or profile URL is required");
|
|
13078
|
+
if (/^https?:\/\//i.test(raw) || /^www\.instagram\.com/i.test(raw) || /^instagram\.com/i.test(raw)) {
|
|
13079
|
+
const parsed = new URL(/^https?:\/\//i.test(raw) ? raw : `https://${raw}`);
|
|
13080
|
+
const host = parsed.hostname.toLowerCase().replace(/^www\./, "").replace(/^m\./, "");
|
|
13081
|
+
if (host !== "instagram.com") throw new Error("Profile URL must be on instagram.com");
|
|
13082
|
+
const first = parsed.pathname.split("/").filter(Boolean)[0];
|
|
13083
|
+
if (!first || ["p", "reel", "tv", "stories", "explore"].includes(first.toLowerCase())) {
|
|
13084
|
+
throw new Error("Profile URL must point to an Instagram profile, not a post, reel, story, or explore page");
|
|
13085
|
+
}
|
|
13086
|
+
return { handle: first, profileUrl: `https://www.instagram.com/${first}/` };
|
|
13087
|
+
}
|
|
13088
|
+
const handle = raw.replace(/^@/, "").replace(/^\/+|\/+$/g, "");
|
|
13089
|
+
if (!/^[A-Za-z0-9._]{1,30}$/.test(handle)) throw new Error("Invalid Instagram handle");
|
|
13090
|
+
return { handle, profileUrl: `https://www.instagram.com/${handle}/` };
|
|
13091
|
+
}
|
|
13092
|
+
function isAllowedInstagramPageUrl(url) {
|
|
13093
|
+
const host = url.hostname.toLowerCase().replace(/^www\./, "").replace(/^m\./, "");
|
|
13094
|
+
return host === "instagram.com";
|
|
13095
|
+
}
|
|
13096
|
+
function normalizeInstagramMediaUrl(raw) {
|
|
13097
|
+
try {
|
|
13098
|
+
const decoded = decodeEscapedValue2(raw);
|
|
13099
|
+
const parsed = new URL(decoded);
|
|
13100
|
+
if (!CDN_HOST_RE.test(parsed.hostname.toLowerCase())) return null;
|
|
13101
|
+
parsed.searchParams.delete("bytestart");
|
|
13102
|
+
parsed.searchParams.delete("byteend");
|
|
13103
|
+
return parsed.href;
|
|
13104
|
+
} catch {
|
|
13105
|
+
return null;
|
|
13106
|
+
}
|
|
13107
|
+
}
|
|
13108
|
+
function parseInstagramEfg(url) {
|
|
13109
|
+
try {
|
|
13110
|
+
const efg = new URL(url).searchParams.get("efg");
|
|
13111
|
+
if (!efg) return null;
|
|
13112
|
+
return JSON.parse(Buffer.from(efg, "base64").toString("utf8"));
|
|
13113
|
+
} catch {
|
|
13114
|
+
return null;
|
|
13115
|
+
}
|
|
13116
|
+
}
|
|
13117
|
+
function trackFromUrl(url) {
|
|
13118
|
+
const normalized = normalizeInstagramMediaUrl(url);
|
|
13119
|
+
if (!normalized) return null;
|
|
13120
|
+
const parsed = parseInstagramEfg(normalized);
|
|
13121
|
+
const tag = typeof parsed?.vencode_tag === "string" ? parsed.vencode_tag : null;
|
|
13122
|
+
const streamType = tag?.toLowerCase().includes("audio") ? "audio" : normalized.includes(".mp4") ? "video" : "unknown";
|
|
13123
|
+
return {
|
|
13124
|
+
url: normalized,
|
|
13125
|
+
streamType,
|
|
13126
|
+
bitrate: typeof parsed?.bitrate === "number" ? parsed.bitrate : null,
|
|
13127
|
+
durationSec: typeof parsed?.duration_s === "number" ? parsed.duration_s : null,
|
|
13128
|
+
vencodeTag: tag,
|
|
13129
|
+
width: typeof parsed?.width === "number" ? parsed.width : null,
|
|
13130
|
+
height: typeof parsed?.height === "number" ? parsed.height : null
|
|
13131
|
+
};
|
|
13132
|
+
}
|
|
13133
|
+
function extractInstagramMp4UrlsFromText(text) {
|
|
13134
|
+
const urls = [];
|
|
13135
|
+
const seen = /* @__PURE__ */ new Set();
|
|
13136
|
+
const patterns = [
|
|
13137
|
+
/https?:\\?\/\\?\/[^"'<>\s]+?\.mp4(?:\?[^"'<>\s]+)?/g,
|
|
13138
|
+
/https?:%5C?\/%5C?\/[^"'<>\s]+?\.mp4(?:\?[^"'<>\s]+)?/g
|
|
13139
|
+
];
|
|
13140
|
+
for (const pattern of patterns) {
|
|
13141
|
+
let match;
|
|
13142
|
+
while ((match = pattern.exec(text)) !== null) {
|
|
13143
|
+
const normalized = normalizeInstagramMediaUrl(match[0].replace(/%5C/gi, "\\"));
|
|
13144
|
+
if (!normalized || seen.has(normalized)) continue;
|
|
13145
|
+
seen.add(normalized);
|
|
13146
|
+
urls.push(normalized);
|
|
13147
|
+
}
|
|
13148
|
+
}
|
|
13149
|
+
return urls;
|
|
13150
|
+
}
|
|
13151
|
+
function instagramTracksFromUrls(urls) {
|
|
13152
|
+
const seen = /* @__PURE__ */ new Set();
|
|
13153
|
+
const tracks = [];
|
|
13154
|
+
for (const raw of urls) {
|
|
13155
|
+
const track = trackFromUrl(raw);
|
|
13156
|
+
if (!track || seen.has(track.url)) continue;
|
|
13157
|
+
seen.add(track.url);
|
|
13158
|
+
tracks.push(track);
|
|
13159
|
+
}
|
|
13160
|
+
return tracks;
|
|
13161
|
+
}
|
|
13162
|
+
function selectInstagramTracks(tracks) {
|
|
13163
|
+
const selectedVideoTrack = tracks.filter((track) => track.streamType === "video").sort((a, b) => {
|
|
13164
|
+
const aPixels = (a.width ?? 0) * (a.height ?? 0);
|
|
13165
|
+
const bPixels = (b.width ?? 0) * (b.height ?? 0);
|
|
13166
|
+
if (aPixels !== bPixels) return bPixels - aPixels;
|
|
13167
|
+
return (b.bitrate ?? 0) - (a.bitrate ?? 0);
|
|
13168
|
+
})[0] ?? null;
|
|
13169
|
+
const selectedAudioTrack = tracks.filter((track) => track.streamType === "audio").sort((a, b) => (b.bitrate ?? 0) - (a.bitrate ?? 0))[0] ?? null;
|
|
13170
|
+
return { selectedVideoTrack, selectedAudioTrack };
|
|
13171
|
+
}
|
|
13172
|
+
function parseCount(text) {
|
|
13173
|
+
if (!text) return null;
|
|
13174
|
+
const match = text.match(/([\d,.]+)\s*posts?/i);
|
|
13175
|
+
if (!match) return null;
|
|
13176
|
+
const value = Number(match[1].replace(/[,.]/g, ""));
|
|
13177
|
+
return Number.isFinite(value) ? value : null;
|
|
13178
|
+
}
|
|
13179
|
+
function captionFromMetadata(metadata) {
|
|
13180
|
+
const raw = metadata.ogDescription ?? metadata.description;
|
|
13181
|
+
if (!raw) return null;
|
|
13182
|
+
const quoted = raw.match(/:\s*"([\s\S]+)"\.?\s*$/);
|
|
13183
|
+
if (quoted?.[1]) return quoted[1].trim();
|
|
13184
|
+
return raw.trim();
|
|
13185
|
+
}
|
|
13186
|
+
function ownerFromMetadata(metadata) {
|
|
13187
|
+
const title = metadata.ogTitle ?? metadata.twitterTitle ?? metadata.title;
|
|
13188
|
+
if (!title) return null;
|
|
13189
|
+
const match = title.match(/^(.+?)\s+on Instagram:/i) ?? title.match(/^(.+?)\s+\(@[^)]+\)\s+•\s+Instagram/i);
|
|
13190
|
+
return match?.[1]?.trim() || null;
|
|
13191
|
+
}
|
|
13192
|
+
async function collectInstagramProfileContentFromPage(page, opts) {
|
|
13193
|
+
const maxItems = Math.min(500, Math.max(1, opts.maxItems ?? 50));
|
|
13194
|
+
const maxScrolls = Math.min(100, Math.max(0, opts.maxScrolls ?? 10));
|
|
13195
|
+
const scrollDelayMs = Math.min(5e3, Math.max(250, opts.scrollDelayMs ?? 1200));
|
|
13196
|
+
const stableScrollLimit = Math.min(10, Math.max(1, opts.stableScrollLimit ?? 4));
|
|
13197
|
+
const seen = /* @__PURE__ */ new Set();
|
|
13198
|
+
const items = [];
|
|
13199
|
+
const stages = [];
|
|
13200
|
+
let attemptedScrolls = 0;
|
|
13201
|
+
let stableScrolls = 0;
|
|
13202
|
+
let stoppedReason = null;
|
|
13203
|
+
const collect = async (stage) => {
|
|
13204
|
+
const before = items.length;
|
|
13205
|
+
const links = await page.evaluate(() => Array.from(document.querySelectorAll("a[href]")).map((a) => ({
|
|
13206
|
+
href: a.href,
|
|
13207
|
+
text: (a.textContent ?? "").replace(/\s+/g, " ").trim()
|
|
13208
|
+
})));
|
|
13209
|
+
for (const link of links) {
|
|
13210
|
+
let parsed;
|
|
13211
|
+
try {
|
|
13212
|
+
parsed = new URL(link.href);
|
|
13213
|
+
} catch {
|
|
13214
|
+
continue;
|
|
13215
|
+
}
|
|
13216
|
+
const found = instagramContentTypeFromPath(parsed.pathname);
|
|
13217
|
+
if (!found) continue;
|
|
13218
|
+
const normalized = `https://www.instagram.com/${found.type === "post" ? "p" : found.type}/${found.shortcode}/`;
|
|
13219
|
+
if (seen.has(normalized)) continue;
|
|
13220
|
+
seen.add(normalized);
|
|
13221
|
+
items.push({
|
|
13222
|
+
url: normalized,
|
|
13223
|
+
type: found.type,
|
|
13224
|
+
shortcode: found.shortcode,
|
|
13225
|
+
anchorText: link.text || null,
|
|
13226
|
+
firstSeenStage: stage
|
|
13227
|
+
});
|
|
13228
|
+
if (items.length >= maxItems) return items.length - before;
|
|
13229
|
+
}
|
|
13230
|
+
return items.length - before;
|
|
13231
|
+
};
|
|
13232
|
+
const readBodyText = async () => page.evaluate(() => document.body?.innerText ?? "").catch(() => "");
|
|
13233
|
+
const profileLines = (text) => text.split("\n").map((line) => line.trim()).filter(Boolean);
|
|
13234
|
+
const profilePostCountText = (text) => profileLines(text).find((line) => /\bposts?\b/i.test(line) && /[\d,.]/.test(line)) ?? null;
|
|
13235
|
+
const scrollMetrics = async () => page.evaluate(() => ({
|
|
13236
|
+
scrollY: typeof window.scrollY === "number" ? window.scrollY : null,
|
|
13237
|
+
scrollHeight: Math.max(
|
|
13238
|
+
document.body?.scrollHeight ?? 0,
|
|
13239
|
+
document.documentElement?.scrollHeight ?? 0
|
|
13240
|
+
) || null
|
|
13241
|
+
})).catch(() => ({ scrollY: null, scrollHeight: null }));
|
|
13242
|
+
const recordStage = async (stage, addedCount) => {
|
|
13243
|
+
const metrics = await scrollMetrics();
|
|
13244
|
+
stages.push({
|
|
13245
|
+
stage,
|
|
13246
|
+
itemCount: items.length,
|
|
13247
|
+
addedCount,
|
|
13248
|
+
scrollY: metrics.scrollY,
|
|
13249
|
+
scrollHeight: metrics.scrollHeight
|
|
13250
|
+
});
|
|
13251
|
+
};
|
|
13252
|
+
const advancePagination = async () => page.evaluate(() => {
|
|
13253
|
+
const contentLinks = Array.from(document.querySelectorAll('a[href*="/p/"], a[href*="/reel/"], a[href*="/tv/"]'));
|
|
13254
|
+
const last = contentLinks[contentLinks.length - 1];
|
|
13255
|
+
if (last) last.scrollIntoView({ block: "end", inline: "nearest" });
|
|
13256
|
+
const distance = Math.max(window.innerHeight * 0.9, 700);
|
|
13257
|
+
window.scrollBy(0, distance);
|
|
13258
|
+
return {
|
|
13259
|
+
scrollY: typeof window.scrollY === "number" ? window.scrollY : null,
|
|
13260
|
+
scrollHeight: Math.max(
|
|
13261
|
+
document.body?.scrollHeight ?? 0,
|
|
13262
|
+
document.documentElement?.scrollHeight ?? 0
|
|
13263
|
+
) || null
|
|
13264
|
+
};
|
|
13265
|
+
}).catch(() => ({ scrollY: null, scrollHeight: null }));
|
|
13266
|
+
await page.waitForLoadState("domcontentloaded", { timeout: 15e3 }).catch(() => {
|
|
13267
|
+
});
|
|
13268
|
+
await page.waitForTimeout(1500);
|
|
13269
|
+
await recordStage("initial", await collect("initial"));
|
|
13270
|
+
const initialReportedPostCount = parseCount(profilePostCountText(await readBodyText()));
|
|
13271
|
+
for (let i = 0; i < maxScrolls && items.length < maxItems && stableScrolls < stableScrollLimit; i++) {
|
|
13272
|
+
const before = items.length;
|
|
13273
|
+
const beforeMetrics = await scrollMetrics();
|
|
13274
|
+
attemptedScrolls = i + 1;
|
|
13275
|
+
await advancePagination();
|
|
13276
|
+
await page.keyboard.press("PageDown").catch(() => {
|
|
13277
|
+
});
|
|
13278
|
+
await page.waitForLoadState("networkidle", { timeout: 2500 }).catch(() => {
|
|
13279
|
+
});
|
|
13280
|
+
await page.waitForTimeout(scrollDelayMs);
|
|
13281
|
+
const added = await collect(`scroll_${i + 1}`);
|
|
13282
|
+
await recordStage(`scroll_${i + 1}`, added);
|
|
13283
|
+
const afterMetrics = await scrollMetrics();
|
|
13284
|
+
const advanced = (afterMetrics.scrollHeight ?? 0) > (beforeMetrics.scrollHeight ?? 0) || (afterMetrics.scrollY ?? 0) > (beforeMetrics.scrollY ?? 0);
|
|
13285
|
+
stableScrolls = items.length === before && !advanced ? stableScrolls + 1 : 0;
|
|
13286
|
+
if (initialReportedPostCount != null && items.length >= initialReportedPostCount) {
|
|
13287
|
+
stoppedReason = "reported_post_count";
|
|
13288
|
+
break;
|
|
13289
|
+
}
|
|
13290
|
+
if (items.length >= maxItems) {
|
|
13291
|
+
stoppedReason = "max_items";
|
|
13292
|
+
break;
|
|
13293
|
+
}
|
|
13294
|
+
if (stableScrolls >= stableScrollLimit) {
|
|
13295
|
+
stoppedReason = "stable_scrolls";
|
|
13296
|
+
break;
|
|
13297
|
+
}
|
|
13298
|
+
}
|
|
13299
|
+
const bodyText = await readBodyText();
|
|
13300
|
+
const lines = profileLines(bodyText);
|
|
13301
|
+
const reportedPostCountText = lines.find((line) => /\bposts?\b/i.test(line) && /[\d,.]/.test(line)) ?? null;
|
|
13302
|
+
const followerCountText = lines.find((line) => /\bfollowers?\b/i.test(line)) ?? null;
|
|
13303
|
+
const followingCountText = lines.find((line) => /\bfollowing\b/i.test(line)) ?? null;
|
|
13304
|
+
const profileName = lines.find((line) => line && line.toLowerCase() !== opts.handle.toLowerCase()) ?? null;
|
|
13305
|
+
const typeCounts = {
|
|
13306
|
+
post: items.filter((item) => item.type === "post").length,
|
|
13307
|
+
reel: items.filter((item) => item.type === "reel").length,
|
|
13308
|
+
tv: items.filter((item) => item.type === "tv").length
|
|
13309
|
+
};
|
|
13310
|
+
const reportedPostCount = parseCount(reportedPostCountText);
|
|
13311
|
+
const reachedMaxItems = items.length >= maxItems;
|
|
13312
|
+
const reachedReportedPostCount = reportedPostCount != null && items.length >= reportedPostCount;
|
|
13313
|
+
if (!stoppedReason) {
|
|
13314
|
+
stoppedReason = reachedMaxItems ? "max_items" : reachedReportedPostCount ? "reported_post_count" : maxScrolls === 0 ? "no_scrolls" : attemptedScrolls >= maxScrolls ? "max_scrolls" : "stable_scrolls";
|
|
13315
|
+
}
|
|
13316
|
+
const finalMetrics = await scrollMetrics();
|
|
13317
|
+
const pagination = {
|
|
13318
|
+
maxItems,
|
|
13319
|
+
maxScrolls,
|
|
13320
|
+
attemptedScrolls,
|
|
13321
|
+
stableScrolls,
|
|
13322
|
+
stableScrollLimit,
|
|
13323
|
+
scrollDelayMs,
|
|
13324
|
+
reachedMaxItems,
|
|
13325
|
+
reachedReportedPostCount,
|
|
13326
|
+
finalScrollHeight: finalMetrics.scrollHeight,
|
|
13327
|
+
stoppedReason,
|
|
13328
|
+
stages
|
|
13329
|
+
};
|
|
13330
|
+
const limitations = [];
|
|
13331
|
+
if (/log in|sign up/i.test(bodyText)) limitations.push("Login or sign-up prompts were visible during extraction.");
|
|
13332
|
+
if (reachedMaxItems && !reachedReportedPostCount) limitations.push("Collection stopped at maxItems before the full profile history was proven.");
|
|
13333
|
+
if (reportedPostCount != null && items.length < reportedPostCount && !reachedMaxItems) limitations.push("Collected item count is lower than the profile-reported post count.");
|
|
13334
|
+
if (stoppedReason === "stable_scrolls" && !reachedReportedPostCount) limitations.push("Pagination stopped after repeated stable scrolls without new content.");
|
|
13335
|
+
if (stoppedReason === "max_scrolls" && !reachedReportedPostCount) limitations.push("Pagination reached maxScrolls before the full profile history was proven.");
|
|
13336
|
+
return {
|
|
13337
|
+
handle: opts.handle,
|
|
13338
|
+
profileUrl: opts.profileUrl,
|
|
13339
|
+
pageUrl: page.url(),
|
|
13340
|
+
profileName,
|
|
13341
|
+
bio: null,
|
|
13342
|
+
reportedPostCount,
|
|
13343
|
+
reportedPostCountText,
|
|
13344
|
+
followerCountText,
|
|
13345
|
+
followingCountText,
|
|
13346
|
+
collectedContentCount: items.length,
|
|
13347
|
+
typeCounts,
|
|
13348
|
+
pagination,
|
|
13349
|
+
items,
|
|
13350
|
+
limited: limitations.length > 0,
|
|
13351
|
+
limitations,
|
|
13352
|
+
extractedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
13353
|
+
};
|
|
13354
|
+
}
|
|
13355
|
+
async function extractInstagramMediaFromPage(page, sourceUrl, capturedMediaUrls = []) {
|
|
13356
|
+
await page.waitForLoadState("domcontentloaded", { timeout: 15e3 }).catch(() => {
|
|
13357
|
+
});
|
|
13358
|
+
await page.waitForTimeout(2500);
|
|
13359
|
+
const html = await page.content();
|
|
13360
|
+
const bodyText = await page.evaluate(() => document.body?.innerText ?? "").catch(() => "");
|
|
13361
|
+
const metadata = {
|
|
13362
|
+
title: await page.title().catch(() => null),
|
|
13363
|
+
description: metadataContent2(html, "description"),
|
|
13364
|
+
ogTitle: metadataContent2(html, "og:title"),
|
|
13365
|
+
ogDescription: metadataContent2(html, "og:description"),
|
|
13366
|
+
ogImage: metadataContent2(html, "og:image"),
|
|
13367
|
+
ogUrl: metadataContent2(html, "og:url"),
|
|
13368
|
+
twitterImage: metadataContent2(html, "twitter:image"),
|
|
13369
|
+
twitterTitle: metadataContent2(html, "twitter:title")
|
|
13370
|
+
};
|
|
13371
|
+
const parsedPath = (() => {
|
|
13372
|
+
try {
|
|
13373
|
+
return instagramContentTypeFromPath(new URL(page.url()).pathname) ?? instagramContentTypeFromPath(new URL(sourceUrl).pathname);
|
|
13374
|
+
} catch {
|
|
13375
|
+
return null;
|
|
13376
|
+
}
|
|
13377
|
+
})();
|
|
13378
|
+
const htmlUrls = extractInstagramMp4UrlsFromText(html);
|
|
13379
|
+
const tracks = instagramTracksFromUrls([...capturedMediaUrls, ...htmlUrls]);
|
|
13380
|
+
const selected = selectInstagramTracks(tracks);
|
|
13381
|
+
const imageUrl = metadata.ogImage ?? metadata.twitterImage;
|
|
13382
|
+
const limitations = [];
|
|
13383
|
+
if (/log in|sign up/i.test(bodyText)) limitations.push("Login or sign-up prompts were visible during extraction.");
|
|
13384
|
+
if (parsedPath?.type === "reel" && tracks.length === 0) limitations.push("No Instagram MP4 tracks were captured; retry with a logged-in browser profile or longer wait.");
|
|
13385
|
+
return {
|
|
13386
|
+
sourceUrl,
|
|
13387
|
+
pageUrl: page.url(),
|
|
13388
|
+
type: parsedPath?.type ?? null,
|
|
13389
|
+
shortcode: parsedPath?.shortcode ?? null,
|
|
13390
|
+
ownerName: ownerFromMetadata(metadata),
|
|
13391
|
+
caption: captionFromMetadata(metadata),
|
|
13392
|
+
bodyText,
|
|
13393
|
+
metadata,
|
|
13394
|
+
imageUrl,
|
|
13395
|
+
tracks,
|
|
13396
|
+
selectedVideoTrack: selected.selectedVideoTrack,
|
|
13397
|
+
selectedAudioTrack: selected.selectedAudioTrack,
|
|
13398
|
+
limitations,
|
|
13399
|
+
extractedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
13400
|
+
};
|
|
13401
|
+
}
|
|
13402
|
+
var CONTENT_PATH_RE, CDN_HOST_RE;
|
|
13403
|
+
var init_InstagramContentExtractor = __esm({
|
|
13404
|
+
"src/extractor/InstagramContentExtractor.ts"() {
|
|
13405
|
+
"use strict";
|
|
13406
|
+
CONTENT_PATH_RE = /^\/(?:[^/]+\/)?(p|reel|tv)\/([A-Za-z0-9_-]+)\/?/i;
|
|
13407
|
+
CDN_HOST_RE = /(^|\.)((cdninstagram\.com)|(scontent[^/]*\.cdninstagram\.com))$/i;
|
|
13408
|
+
}
|
|
13409
|
+
});
|
|
13410
|
+
|
|
13411
|
+
// src/api/instagram-routes.ts
|
|
13412
|
+
function invalidRequest2(message) {
|
|
13413
|
+
return { error_code: "invalid_request", message };
|
|
13414
|
+
}
|
|
13415
|
+
function directProfileDir() {
|
|
13416
|
+
return process.env.MCP_SCRAPER_BROWSER_PROFILE_DIR?.trim() || void 0;
|
|
13417
|
+
}
|
|
13418
|
+
function configuredProfileName(profile) {
|
|
13419
|
+
const directDir = directProfileDir();
|
|
13420
|
+
return profile?.trim() || process.env.MCP_SCRAPER_BROWSER_PROFILE?.trim() || (directDir ? (0, import_node_path7.basename)(directDir) : "default");
|
|
13421
|
+
}
|
|
13422
|
+
function launchExecutablePath(manifestExecutablePath) {
|
|
13423
|
+
const configured = process.env.MCP_SCRAPER_BROWSER_EXECUTABLE?.trim() || manifestExecutablePath || defaultChromeExecutablePath();
|
|
13424
|
+
return configured && (0, import_node_fs4.existsSync)(configured) ? configured : void 0;
|
|
13425
|
+
}
|
|
13426
|
+
function ignoreDefaultArgsFor(executablePath) {
|
|
13427
|
+
if (!executablePath) return void 0;
|
|
13428
|
+
if (process.platform !== "darwin") return void 0;
|
|
13429
|
+
if (!/Google Chrome\.app\/Contents\/MacOS\/Google Chrome$/.test(executablePath)) return void 0;
|
|
13430
|
+
return ["--password-store=basic", "--use-mock-keychain"];
|
|
13431
|
+
}
|
|
13432
|
+
async function kernelLaunchOptsResidential2() {
|
|
13433
|
+
let proxyId = browserServiceProxyId();
|
|
13434
|
+
try {
|
|
13435
|
+
const resolution2 = await resolveKernelProxyId({
|
|
13436
|
+
kernelApiKey: browserServiceApiKey(),
|
|
13437
|
+
proxyMode: DEFAULT_PROXY_MODE,
|
|
13438
|
+
configuredKernelProxyId: browserServiceProxyId(),
|
|
13439
|
+
gl: "us"
|
|
13440
|
+
});
|
|
13441
|
+
if (resolution2.kernelProxyId) proxyId = resolution2.kernelProxyId;
|
|
13442
|
+
} catch {
|
|
13443
|
+
proxyId = browserServiceProxyId();
|
|
13444
|
+
}
|
|
13445
|
+
return {
|
|
13446
|
+
headless: true,
|
|
13447
|
+
kernelApiKey: browserServiceApiKey(),
|
|
13448
|
+
kernelProxyId: proxyId,
|
|
13449
|
+
viewport: { width: 1280, height: 900 },
|
|
13450
|
+
locale: "en-US"
|
|
13451
|
+
};
|
|
13452
|
+
}
|
|
13453
|
+
async function resolveInstagramLaunch(input) {
|
|
13454
|
+
const requestedMode = input.browserMode ?? "auto";
|
|
13455
|
+
const directDir = directProfileDir();
|
|
13456
|
+
const useLocal = requestedMode === "local" || requestedMode === "auto" && (Boolean(input.browserProfile) || Boolean(directDir) || localBrowserModeEnabled());
|
|
13457
|
+
if (!useLocal) {
|
|
13458
|
+
return {
|
|
13459
|
+
config: await kernelLaunchOptsResidential2(),
|
|
13460
|
+
browser: {
|
|
13461
|
+
mode: "hosted",
|
|
13462
|
+
requestedMode,
|
|
13463
|
+
profileName: null,
|
|
13464
|
+
profileSource: "hosted",
|
|
13465
|
+
profileDirConfigured: false,
|
|
13466
|
+
executablePathConfigured: false
|
|
13467
|
+
}
|
|
13468
|
+
};
|
|
13469
|
+
}
|
|
13470
|
+
const requestedName = configuredProfileName(input.browserProfile);
|
|
13471
|
+
if (directDir && (!input.browserProfile || requestedName === process.env.MCP_SCRAPER_BROWSER_PROFILE?.trim())) {
|
|
13472
|
+
const executablePath2 = launchExecutablePath(null);
|
|
13473
|
+
return {
|
|
13474
|
+
config: {
|
|
13475
|
+
headless: false,
|
|
13476
|
+
profileDir: directDir,
|
|
13477
|
+
browserExecutablePath: executablePath2,
|
|
13478
|
+
ignoreDefaultArgs: ignoreDefaultArgsFor(executablePath2),
|
|
13479
|
+
viewport: { width: 1280, height: 900 },
|
|
13480
|
+
locale: "en-US"
|
|
13481
|
+
},
|
|
13482
|
+
browser: {
|
|
13483
|
+
mode: "local",
|
|
13484
|
+
requestedMode,
|
|
13485
|
+
profileName: requestedName,
|
|
13486
|
+
profileSource: "direct_profile_dir",
|
|
13487
|
+
profileDirConfigured: true,
|
|
13488
|
+
executablePathConfigured: Boolean(executablePath2)
|
|
13489
|
+
}
|
|
13490
|
+
};
|
|
13491
|
+
}
|
|
13492
|
+
const manifest = await loadImportedBrowserProfile(requestedName);
|
|
13493
|
+
const executablePath = launchExecutablePath(manifest.browserExecutablePath);
|
|
13494
|
+
return {
|
|
13495
|
+
config: {
|
|
13496
|
+
headless: false,
|
|
13497
|
+
profileDir: manifest.userDataDir,
|
|
13498
|
+
browserExecutablePath: executablePath,
|
|
13499
|
+
ignoreDefaultArgs: ignoreDefaultArgsFor(executablePath),
|
|
13500
|
+
viewport: { width: 1280, height: 900 },
|
|
13501
|
+
locale: "en-US"
|
|
13502
|
+
},
|
|
13503
|
+
browser: {
|
|
13504
|
+
mode: "local",
|
|
13505
|
+
requestedMode,
|
|
13506
|
+
profileName: manifest.name,
|
|
13507
|
+
profileSource: "managed_profile",
|
|
13508
|
+
profileDirConfigured: true,
|
|
13509
|
+
executablePathConfigured: Boolean(executablePath)
|
|
13510
|
+
}
|
|
13511
|
+
};
|
|
13512
|
+
}
|
|
13513
|
+
function outputBaseDir() {
|
|
13514
|
+
return process.env.MCP_SCRAPER_OUTPUT_DIR?.trim() || (0, import_node_path7.join)((0, import_node_os5.homedir)(), "Downloads", "mcp-scraper");
|
|
13515
|
+
}
|
|
13516
|
+
function safeFilePart(input) {
|
|
13517
|
+
return input.replace(/^https?:\/\//, "").replace(/[^a-z0-9._-]+/gi, "-").replace(/^-+|-+$/g, "").slice(0, 80) || "instagram";
|
|
13518
|
+
}
|
|
13519
|
+
function mediaOutputDir(shortcode, sourceUrl) {
|
|
13520
|
+
const stamp = (/* @__PURE__ */ new Date()).toISOString().replace(/[:.]/g, "-").slice(0, 19);
|
|
13521
|
+
const slug = shortcode ? `instagram-${shortcode}` : safeFilePart(sourceUrl);
|
|
13522
|
+
const dir = (0, import_node_path7.join)(outputBaseDir(), "instagram", `${stamp}-${slug}`);
|
|
13523
|
+
(0, import_node_fs4.mkdirSync)(dir, { recursive: true });
|
|
13524
|
+
return dir;
|
|
13525
|
+
}
|
|
13526
|
+
async function downloadToFile(url, destPath, referer) {
|
|
13527
|
+
const res = await fetch(url, {
|
|
13528
|
+
headers: {
|
|
13529
|
+
"User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36",
|
|
13530
|
+
"Referer": referer
|
|
13531
|
+
},
|
|
13532
|
+
signal: AbortSignal.timeout(9e4)
|
|
13533
|
+
});
|
|
13534
|
+
if (!res.ok) throw new Error(`HTTP ${res.status}`);
|
|
13535
|
+
if (!res.body) throw new Error("Empty response body");
|
|
13536
|
+
await (0, import_promises6.pipeline)(import_node_stream2.Readable.fromWeb(res.body), (0, import_node_fs4.createWriteStream)(destPath));
|
|
13537
|
+
return {
|
|
13538
|
+
savedPath: destPath,
|
|
13539
|
+
sizeBytes: (0, import_node_fs4.statSync)(destPath).size,
|
|
13540
|
+
mimeType: res.headers.get("content-type")?.split(";")[0]?.trim() ?? null
|
|
13541
|
+
};
|
|
13542
|
+
}
|
|
13543
|
+
function extFromMime(mimeType, fallback) {
|
|
13544
|
+
const map = {
|
|
13545
|
+
"image/jpeg": "jpg",
|
|
13546
|
+
"image/png": "png",
|
|
13547
|
+
"image/webp": "webp",
|
|
13548
|
+
"image/gif": "gif",
|
|
13549
|
+
"video/mp4": "mp4",
|
|
13550
|
+
"audio/mp4": "mp4",
|
|
13551
|
+
"audio/aac": "aac"
|
|
13552
|
+
};
|
|
13553
|
+
return mimeType ? map[mimeType] ?? fallback : fallback;
|
|
13554
|
+
}
|
|
13555
|
+
function runFfmpegMux(videoPath, audioPath, outPath) {
|
|
13556
|
+
return new Promise((resolve) => {
|
|
13557
|
+
const child = (0, import_node_child_process2.spawn)("ffmpeg", ["-y", "-i", videoPath, "-i", audioPath, "-c", "copy", "-shortest", outPath], {
|
|
13558
|
+
stdio: ["ignore", "ignore", "pipe"]
|
|
13559
|
+
});
|
|
13560
|
+
let stderr = "";
|
|
13561
|
+
child.stderr.on("data", (chunk) => {
|
|
13562
|
+
stderr += String(chunk);
|
|
13563
|
+
});
|
|
13564
|
+
child.on("error", (err) => resolve({ ok: false, error: err.message }));
|
|
13565
|
+
child.on("close", (code) => resolve({ ok: code === 0, error: code === 0 ? null : stderr.trim() || `ffmpeg exited ${code}` }));
|
|
13566
|
+
});
|
|
13567
|
+
}
|
|
13568
|
+
function trackFilename(track, index, shortcode) {
|
|
13569
|
+
const label = shortcode ?? "instagram";
|
|
13570
|
+
const type = track.streamType === "audio" ? "audio" : track.streamType === "video" ? "video" : "track";
|
|
13571
|
+
const bitrate = track.bitrate ? `-${track.bitrate}` : "";
|
|
13572
|
+
return `${label}-${type}-${index}${bitrate}.mp4`;
|
|
13573
|
+
}
|
|
13574
|
+
var import_hono5, import_zod17, import_node_fs4, import_node_os5, import_node_path7, import_node_stream2, import_promises6, import_node_child_process2, InstagramBrowserModeSchema, InstagramBrowserOptionsSchema, InstagramProfileContentBodySchema, InstagramMediaTypeSchema, InstagramMediaDownloadBodySchema, instagramApp;
|
|
13575
|
+
var init_instagram_routes = __esm({
|
|
13576
|
+
"src/api/instagram-routes.ts"() {
|
|
13577
|
+
"use strict";
|
|
13578
|
+
import_hono5 = require("hono");
|
|
13579
|
+
import_zod17 = require("zod");
|
|
13580
|
+
import_node_fs4 = require("fs");
|
|
13581
|
+
import_node_os5 = require("os");
|
|
13582
|
+
import_node_path7 = require("path");
|
|
13583
|
+
import_node_stream2 = require("stream");
|
|
13584
|
+
import_promises6 = require("stream/promises");
|
|
13585
|
+
import_node_child_process2 = require("child_process");
|
|
13586
|
+
init_browser_service_env();
|
|
13587
|
+
init_BrowserDriver();
|
|
13588
|
+
init_kernel_proxy_resolver();
|
|
13589
|
+
init_schemas3();
|
|
13590
|
+
init_api_auth();
|
|
13591
|
+
init_url_utils();
|
|
13592
|
+
init_db();
|
|
13593
|
+
init_rates();
|
|
13594
|
+
init_concurrency_gates();
|
|
13595
|
+
init_local_browser_profiles();
|
|
13596
|
+
init_InstagramContentExtractor();
|
|
13597
|
+
init_media_transcription();
|
|
13598
|
+
InstagramBrowserModeSchema = import_zod17.z.enum(["auto", "hosted", "local"]);
|
|
13599
|
+
InstagramBrowserOptionsSchema = {
|
|
13600
|
+
browserMode: InstagramBrowserModeSchema.default("auto"),
|
|
13601
|
+
browserProfile: import_zod17.z.string().trim().min(1).optional()
|
|
13602
|
+
};
|
|
13603
|
+
InstagramProfileContentBodySchema = import_zod17.z.object({
|
|
13604
|
+
handle: import_zod17.z.string().trim().optional(),
|
|
13605
|
+
url: import_zod17.z.string().trim().optional(),
|
|
13606
|
+
maxItems: import_zod17.z.number().int().min(1).max(500).default(50),
|
|
13607
|
+
maxScrolls: import_zod17.z.number().int().min(0).max(100).default(10),
|
|
13608
|
+
scrollDelayMs: import_zod17.z.number().int().min(250).max(5e3).default(1200),
|
|
13609
|
+
stableScrollLimit: import_zod17.z.number().int().min(1).max(10).default(4),
|
|
13610
|
+
...InstagramBrowserOptionsSchema
|
|
13611
|
+
}).refine((d) => !!d.handle || !!d.url, {
|
|
13612
|
+
message: "handle or url is required"
|
|
13613
|
+
});
|
|
13614
|
+
InstagramMediaTypeSchema = import_zod17.z.enum(["image", "video", "audio"]);
|
|
13615
|
+
InstagramMediaDownloadBodySchema = import_zod17.z.object({
|
|
13616
|
+
url: import_zod17.z.string().trim().min(1, "url is required"),
|
|
13617
|
+
mediaTypes: import_zod17.z.array(InstagramMediaTypeSchema).default(["image", "video", "audio"]),
|
|
13618
|
+
downloadMedia: import_zod17.z.boolean().default(true),
|
|
13619
|
+
downloadAllTracks: import_zod17.z.boolean().default(false),
|
|
13620
|
+
includeTranscript: import_zod17.z.boolean().default(false),
|
|
13621
|
+
mux: import_zod17.z.boolean().default(true),
|
|
13622
|
+
...InstagramBrowserOptionsSchema
|
|
13623
|
+
});
|
|
13624
|
+
instagramApp = new import_hono5.Hono();
|
|
13625
|
+
instagramApp.post("/profile-content", createApiKeyAuth(), async (c) => {
|
|
13626
|
+
const raw = await c.req.json().catch(() => ({}));
|
|
13627
|
+
const parsed = InstagramProfileContentBodySchema.safeParse(raw);
|
|
13628
|
+
if (!parsed.success) return c.json(invalidRequest2(parsed.error.issues[0]?.message ?? "Invalid request"), 400);
|
|
13629
|
+
const body = parsed.data;
|
|
13630
|
+
let target;
|
|
13631
|
+
try {
|
|
13632
|
+
target = normalizeInstagramProfileInput(body.url ?? body.handle ?? "");
|
|
13633
|
+
} catch (err) {
|
|
13634
|
+
return c.json(invalidRequest2(err instanceof Error ? err.message : String(err)), 400);
|
|
13635
|
+
}
|
|
13636
|
+
const user = c.get("user");
|
|
13637
|
+
const gate = await acquireConcurrencyGate(user, "instagram_profile_content", {
|
|
13638
|
+
reuseLockId: c.req.header("x-mcp-scraper-concurrency-lock"),
|
|
13639
|
+
metadata: { handle: target.handle }
|
|
13640
|
+
});
|
|
13641
|
+
if (!gate.ok) return c.json(concurrencyLimitExceededResponse(gate), 429, { "Retry-After": String(gate.retryAfterSeconds) });
|
|
13642
|
+
const driver = new BrowserDriver();
|
|
13643
|
+
let debited = false;
|
|
13644
|
+
try {
|
|
13645
|
+
const { ok, balance_mc } = await debitMc(user.id, MC_COSTS.instagram_profile, LedgerOperation.INSTAGRAM_PROFILE, target.profileUrl);
|
|
13646
|
+
if (!ok) return c.json(insufficientBalanceResponse(balance_mc, MC_COSTS.instagram_profile), 402);
|
|
13647
|
+
debited = true;
|
|
13648
|
+
const launch = await resolveInstagramLaunch(body);
|
|
13649
|
+
await driver.launch(launch.config);
|
|
13650
|
+
await driver.navigateTo(target.profileUrl);
|
|
13651
|
+
const page = driver.getPage();
|
|
13652
|
+
const result = await collectInstagramProfileContentFromPage(page, {
|
|
13653
|
+
...target,
|
|
13654
|
+
maxItems: body.maxItems,
|
|
13655
|
+
maxScrolls: body.maxScrolls,
|
|
13656
|
+
scrollDelayMs: body.scrollDelayMs,
|
|
13657
|
+
stableScrollLimit: body.stableScrollLimit
|
|
13658
|
+
});
|
|
13659
|
+
const response = { ...result, browser: launch.browser };
|
|
13660
|
+
await logRequestEvent({ userId: user.id, source: "instagram_profile_content", status: "done", query: target.profileUrl, resultCount: result.items.length, result: response });
|
|
13661
|
+
return c.json(response);
|
|
13662
|
+
} catch (err) {
|
|
13663
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
13664
|
+
if (debited) await creditMc(user.id, MC_COSTS.instagram_profile, LedgerOperation.INSTAGRAM_PROFILE_REFUND, "failed call");
|
|
13665
|
+
await logRequestEvent({ userId: user.id, source: "instagram_profile_content", status: "failed", query: target.profileUrl, error: msg });
|
|
13666
|
+
if (msg.toLowerCase().includes("blocked") || msg.toLowerCase().includes("captcha")) return c.json({ error: msg }, 503);
|
|
13667
|
+
return c.json({ error: msg }, 500);
|
|
13668
|
+
} finally {
|
|
13669
|
+
await driver.close();
|
|
13670
|
+
await releaseConcurrencyGate(gate.lockId);
|
|
13671
|
+
}
|
|
13672
|
+
});
|
|
13673
|
+
instagramApp.post("/media-download", createApiKeyAuth(), async (c) => {
|
|
13674
|
+
const raw = await c.req.json().catch(() => ({}));
|
|
13675
|
+
const parsed = InstagramMediaDownloadBodySchema.safeParse(raw);
|
|
13676
|
+
if (!parsed.success) return c.json(invalidRequest2(parsed.error.issues[0]?.message ?? "Invalid request"), 400);
|
|
13677
|
+
const body = parsed.data;
|
|
13678
|
+
const urlCheck = await validatePublicHttpUrl(body.url, { field: "url", requireHttps: false });
|
|
13679
|
+
if (urlCheck.error) return c.json(invalidRequest2(urlCheck.error), 400);
|
|
13680
|
+
const sourceUrl = urlCheck.parsed;
|
|
13681
|
+
if (!isAllowedInstagramPageUrl(sourceUrl)) return c.json(invalidRequest2("url must be an Instagram post, reel, or tv URL"), 400);
|
|
13682
|
+
const user = c.get("user");
|
|
13683
|
+
const gate = await acquireConcurrencyGate(user, "instagram_media_download", {
|
|
13684
|
+
reuseLockId: c.req.header("x-mcp-scraper-concurrency-lock"),
|
|
13685
|
+
metadata: { url: sourceUrl.href }
|
|
13686
|
+
});
|
|
13687
|
+
if (!gate.ok) return c.json(concurrencyLimitExceededResponse(gate), 429, { "Retry-After": String(gate.retryAfterSeconds) });
|
|
13688
|
+
const driver = new BrowserDriver();
|
|
13689
|
+
let mediaDebited = false;
|
|
13690
|
+
let transcriptDebited = false;
|
|
13691
|
+
try {
|
|
13692
|
+
const { ok, balance_mc } = await debitMc(user.id, MC_COSTS.instagram_media, LedgerOperation.INSTAGRAM_MEDIA, sourceUrl.href);
|
|
13693
|
+
if (!ok) return c.json(insufficientBalanceResponse(balance_mc, MC_COSTS.instagram_media), 402);
|
|
13694
|
+
mediaDebited = true;
|
|
13695
|
+
const launch = await resolveInstagramLaunch(body);
|
|
13696
|
+
await driver.launch(launch.config);
|
|
13697
|
+
const page = driver.getPage();
|
|
13698
|
+
const capturedMediaUrls = [];
|
|
13699
|
+
page.on("response", (response) => {
|
|
13700
|
+
const url = normalizeInstagramMediaUrl(response.url());
|
|
13701
|
+
if (!url) return;
|
|
13702
|
+
const headers = response.headers();
|
|
13703
|
+
const contentType = headers["content-type"] ?? "";
|
|
13704
|
+
if (contentType.includes("video/mp4") || url.includes(".mp4")) capturedMediaUrls.push(url);
|
|
13705
|
+
});
|
|
13706
|
+
await driver.navigateTo(sourceUrl.href);
|
|
13707
|
+
const extraction = await extractInstagramMediaFromPage(page, sourceUrl.href, capturedMediaUrls);
|
|
13708
|
+
const mediaTypes = new Set(body.mediaTypes);
|
|
13709
|
+
const downloads = [];
|
|
13710
|
+
const warnings = [];
|
|
13711
|
+
let outputDir = null;
|
|
13712
|
+
if (body.downloadMedia) {
|
|
13713
|
+
outputDir = mediaOutputDir(extraction.shortcode, sourceUrl.href);
|
|
13714
|
+
const textPath = (0, import_node_path7.join)(outputDir, `${extraction.shortcode ?? "instagram"}-text.txt`);
|
|
13715
|
+
(0, import_node_fs4.writeFileSync)(textPath, [
|
|
13716
|
+
`URL: ${extraction.pageUrl}`,
|
|
13717
|
+
extraction.caption ? `Caption: ${extraction.caption}` : "",
|
|
13718
|
+
"",
|
|
13719
|
+
extraction.bodyText
|
|
13720
|
+
].filter(Boolean).join("\n"), "utf8");
|
|
13721
|
+
downloads.push({ kind: "text", url: null, savedPath: textPath, sizeBytes: (0, import_node_fs4.statSync)(textPath).size, mimeType: "text/plain", error: null });
|
|
13722
|
+
if (mediaTypes.has("image") && extraction.imageUrl) {
|
|
13723
|
+
const imagePathBase = (0, import_node_path7.join)(outputDir, `${extraction.shortcode ?? "instagram"}-image`);
|
|
13724
|
+
try {
|
|
13725
|
+
const downloaded = await downloadToFile(extraction.imageUrl, imagePathBase, extraction.pageUrl);
|
|
13726
|
+
const ext = extFromMime(downloaded.mimeType, "jpg");
|
|
13727
|
+
const finalPath = downloaded.savedPath.endsWith(`.${ext}`) ? downloaded.savedPath : `${downloaded.savedPath}.${ext}`;
|
|
13728
|
+
if (finalPath !== downloaded.savedPath) {
|
|
13729
|
+
const { renameSync } = await import("fs");
|
|
13730
|
+
renameSync(downloaded.savedPath, finalPath);
|
|
13731
|
+
}
|
|
13732
|
+
downloads.push({ kind: "image", url: extraction.imageUrl, savedPath: finalPath, sizeBytes: (0, import_node_fs4.statSync)(finalPath).size, mimeType: downloaded.mimeType, error: null });
|
|
13733
|
+
} catch (err) {
|
|
13734
|
+
downloads.push({ kind: "image", url: extraction.imageUrl, savedPath: null, sizeBytes: null, mimeType: null, error: err instanceof Error ? err.message : String(err) });
|
|
13735
|
+
}
|
|
13736
|
+
}
|
|
13737
|
+
const tracksToDownload = body.downloadAllTracks ? extraction.tracks : [extraction.selectedVideoTrack, extraction.selectedAudioTrack].filter((track) => Boolean(track));
|
|
13738
|
+
const savedByUrl = /* @__PURE__ */ new Map();
|
|
13739
|
+
for (const [index, track] of tracksToDownload.entries()) {
|
|
13740
|
+
if (track.streamType === "video" && !mediaTypes.has("video")) continue;
|
|
13741
|
+
if (track.streamType === "audio" && !mediaTypes.has("audio")) continue;
|
|
13742
|
+
const kind = track.streamType === "audio" ? "audio" : "video";
|
|
13743
|
+
const target = (0, import_node_path7.join)(outputDir, trackFilename(track, index, extraction.shortcode));
|
|
13744
|
+
try {
|
|
13745
|
+
const downloaded = await downloadToFile(track.url, target, extraction.pageUrl);
|
|
13746
|
+
downloads.push({ kind, url: track.url, savedPath: downloaded.savedPath, sizeBytes: downloaded.sizeBytes, mimeType: downloaded.mimeType, error: null });
|
|
13747
|
+
savedByUrl.set(track.url, downloaded.savedPath);
|
|
13748
|
+
} catch (err) {
|
|
13749
|
+
downloads.push({ kind, url: track.url, savedPath: null, sizeBytes: null, mimeType: null, error: err instanceof Error ? err.message : String(err) });
|
|
13750
|
+
}
|
|
13751
|
+
}
|
|
13752
|
+
if (body.mux && mediaTypes.has("video") && mediaTypes.has("audio") && extraction.selectedVideoTrack && extraction.selectedAudioTrack && savedByUrl.has(extraction.selectedVideoTrack.url) && savedByUrl.has(extraction.selectedAudioTrack.url)) {
|
|
13753
|
+
const outPath = (0, import_node_path7.join)(outputDir, `${extraction.shortcode ?? "instagram"}-muxed.mp4`);
|
|
13754
|
+
const muxed = await runFfmpegMux(savedByUrl.get(extraction.selectedVideoTrack.url), savedByUrl.get(extraction.selectedAudioTrack.url), outPath);
|
|
13755
|
+
if (muxed.ok) {
|
|
13756
|
+
downloads.push({ kind: "muxed_video", url: null, savedPath: outPath, sizeBytes: (0, import_node_fs4.statSync)(outPath).size, mimeType: "video/mp4", error: null });
|
|
13757
|
+
} else {
|
|
13758
|
+
warnings.push(`Mux skipped/failed: ${muxed.error}`);
|
|
13759
|
+
}
|
|
13760
|
+
}
|
|
13761
|
+
}
|
|
13762
|
+
let transcript = null;
|
|
13763
|
+
if (body.includeTranscript) {
|
|
13764
|
+
const audioUrl = extraction.selectedAudioTrack?.url ?? extraction.selectedVideoTrack?.url;
|
|
13765
|
+
if (!audioUrl) {
|
|
13766
|
+
warnings.push("Transcript requested, but no audio/video track was available.");
|
|
13767
|
+
} else {
|
|
13768
|
+
const tx = await debitMc(user.id, MC_COSTS.instagram_transcribe, LedgerOperation.INSTAGRAM_TRANSCRIBE, sourceUrl.href);
|
|
13769
|
+
if (!tx.ok) return c.json(insufficientBalanceResponse(tx.balance_mc, MC_COSTS.instagram_transcribe), 402);
|
|
13770
|
+
transcriptDebited = true;
|
|
13771
|
+
transcript = await transcribeMediaUrl(audioUrl, "# Instagram Media Transcript");
|
|
13772
|
+
}
|
|
13773
|
+
}
|
|
13774
|
+
const result = {
|
|
13775
|
+
...extraction,
|
|
13776
|
+
browser: launch.browser,
|
|
13777
|
+
outputDir,
|
|
13778
|
+
downloads,
|
|
13779
|
+
warnings,
|
|
13780
|
+
transcript
|
|
13781
|
+
};
|
|
13782
|
+
await logRequestEvent({ userId: user.id, source: "instagram_media_download", status: "done", query: sourceUrl.href, resultCount: downloads.length, result });
|
|
13783
|
+
return c.json(result);
|
|
13784
|
+
} catch (err) {
|
|
13785
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
13786
|
+
if (transcriptDebited) await creditMc(user.id, MC_COSTS.instagram_transcribe, LedgerOperation.INSTAGRAM_TRANSCRIBE_REFUND, "failed call");
|
|
13787
|
+
if (mediaDebited) await creditMc(user.id, MC_COSTS.instagram_media, LedgerOperation.INSTAGRAM_MEDIA_REFUND, "failed call");
|
|
13788
|
+
await logRequestEvent({ userId: user.id, source: "instagram_media_download", status: "failed", query: sourceUrl.href, error: msg });
|
|
13789
|
+
if (msg.toLowerCase().includes("blocked") || msg.toLowerCase().includes("captcha")) return c.json({ error: msg }, 503);
|
|
13790
|
+
return c.json({ error: msg }, 500);
|
|
13791
|
+
} finally {
|
|
13792
|
+
await driver.close();
|
|
13793
|
+
await releaseConcurrencyGate(gate.lockId);
|
|
13794
|
+
}
|
|
13795
|
+
});
|
|
13796
|
+
}
|
|
13797
|
+
});
|
|
13798
|
+
|
|
12933
13799
|
// src/extractor/MapsNavigator.ts
|
|
12934
13800
|
var MapsNavigator;
|
|
12935
13801
|
var init_MapsNavigator = __esm({
|
|
@@ -13117,7 +13983,7 @@ var init_MapsReviewCollector = __esm({
|
|
|
13117
13983
|
});
|
|
13118
13984
|
|
|
13119
13985
|
// src/extractor/MapsExtractor.ts
|
|
13120
|
-
var
|
|
13986
|
+
var import_zod18, MapsExtractor;
|
|
13121
13987
|
var init_MapsExtractor = __esm({
|
|
13122
13988
|
"src/extractor/MapsExtractor.ts"() {
|
|
13123
13989
|
"use strict";
|
|
@@ -13125,7 +13991,7 @@ var init_MapsExtractor = __esm({
|
|
|
13125
13991
|
init_selectors();
|
|
13126
13992
|
init_MapsNavigator();
|
|
13127
13993
|
init_MapsReviewCollector();
|
|
13128
|
-
|
|
13994
|
+
import_zod18 = require("zod");
|
|
13129
13995
|
MapsExtractor = class {
|
|
13130
13996
|
constructor(driver) {
|
|
13131
13997
|
this.driver = driver;
|
|
@@ -13311,7 +14177,7 @@ var init_MapsExtractor = __esm({
|
|
|
13311
14177
|
});
|
|
13312
14178
|
return rows;
|
|
13313
14179
|
}, { hoursTable: MapsSelectors.hoursTable, hoursTableAlt: MapsSelectors.hoursTableAlt });
|
|
13314
|
-
const result =
|
|
14180
|
+
const result = import_zod18.z.array(RawMapsHoursRowSchema).safeParse(raw);
|
|
13315
14181
|
if (!result.success) {
|
|
13316
14182
|
console.warn("[MapsExtractor] hours parse failed", result.error.flatten());
|
|
13317
14183
|
return [];
|
|
@@ -13375,7 +14241,7 @@ var init_MapsExtractor = __esm({
|
|
|
13375
14241
|
});
|
|
13376
14242
|
return results;
|
|
13377
14243
|
});
|
|
13378
|
-
const result =
|
|
14244
|
+
const result = import_zod18.z.array(RawMapsAboutAttributeSchema).safeParse(raw);
|
|
13379
14245
|
if (!result.success) {
|
|
13380
14246
|
console.warn("[MapsExtractor] about parse failed", result.error.flatten());
|
|
13381
14247
|
return [];
|
|
@@ -13767,11 +14633,11 @@ function mapsErrorResponse(c, err, errorCode) {
|
|
|
13767
14633
|
attempts: rotationError?.attempts ?? void 0
|
|
13768
14634
|
}, retryable ? 503 : 500);
|
|
13769
14635
|
}
|
|
13770
|
-
var
|
|
14636
|
+
var import_hono6, mapsApp;
|
|
13771
14637
|
var init_maps_routes = __esm({
|
|
13772
14638
|
"src/api/maps-routes.ts"() {
|
|
13773
14639
|
"use strict";
|
|
13774
|
-
|
|
14640
|
+
import_hono6 = require("hono");
|
|
13775
14641
|
init_db();
|
|
13776
14642
|
init_rates();
|
|
13777
14643
|
init_MapsExtractor();
|
|
@@ -13782,7 +14648,7 @@ var init_maps_routes = __esm({
|
|
|
13782
14648
|
init_browser_service_env();
|
|
13783
14649
|
init_maps_search_rotation();
|
|
13784
14650
|
init_concurrency_gates();
|
|
13785
|
-
mapsApp = new
|
|
14651
|
+
mapsApp = new import_hono6.Hono();
|
|
13786
14652
|
mapsApp.post("/search", createApiKeyAuth(), async (c) => {
|
|
13787
14653
|
const user = c.get("user");
|
|
13788
14654
|
const body = await c.req.json().catch(() => ({}));
|
|
@@ -14058,17 +14924,17 @@ function reportTitle(full) {
|
|
|
14058
14924
|
const title = full.split("\n").find((line) => line.startsWith("# "));
|
|
14059
14925
|
return title?.replace(/^#\s+/, "").trim() || "MCP Scraper Report";
|
|
14060
14926
|
}
|
|
14061
|
-
function
|
|
14062
|
-
return process.env.MCP_SCRAPER_OUTPUT_DIR?.trim() || (0,
|
|
14927
|
+
function outputBaseDir2() {
|
|
14928
|
+
return process.env.MCP_SCRAPER_OUTPUT_DIR?.trim() || (0, import_node_path8.join)((0, import_node_os6.homedir)(), "Downloads", "mcp-scraper");
|
|
14063
14929
|
}
|
|
14064
14930
|
function saveFullReport(full) {
|
|
14065
14931
|
if (!reportSavingEnabled || process.env.MCP_SCRAPER_SAVE_REPORTS === "false") return null;
|
|
14066
|
-
const outDir =
|
|
14932
|
+
const outDir = outputBaseDir2();
|
|
14067
14933
|
try {
|
|
14068
|
-
(0,
|
|
14934
|
+
(0, import_node_fs5.mkdirSync)(outDir, { recursive: true });
|
|
14069
14935
|
const stamp = (/* @__PURE__ */ new Date()).toISOString().replace(/[:.]/g, "-");
|
|
14070
|
-
const file = (0,
|
|
14071
|
-
(0,
|
|
14936
|
+
const file = (0, import_node_path8.join)(outDir, `${stamp}-${slugifyReportName(reportTitle(full))}.md`);
|
|
14937
|
+
(0, import_node_fs5.writeFileSync)(file, full, "utf8");
|
|
14072
14938
|
return file;
|
|
14073
14939
|
} catch {
|
|
14074
14940
|
return null;
|
|
@@ -14077,12 +14943,12 @@ function saveFullReport(full) {
|
|
|
14077
14943
|
function persistScreenshotLocally(base64, url) {
|
|
14078
14944
|
if (!reportSavingEnabled || process.env.MCP_SCRAPER_SAVE_REPORTS === "false") return null;
|
|
14079
14945
|
try {
|
|
14080
|
-
const dir = (0,
|
|
14081
|
-
(0,
|
|
14946
|
+
const dir = (0, import_node_path8.join)(outputBaseDir2(), "screenshots");
|
|
14947
|
+
(0, import_node_fs5.mkdirSync)(dir, { recursive: true });
|
|
14082
14948
|
const stamp = (/* @__PURE__ */ new Date()).toISOString().replace(/[:.]/g, "-");
|
|
14083
14949
|
const slug = url.replace(/^https?:\/\//, "").replace(/[^a-z0-9]+/gi, "-").replace(/^-+|-+$/g, "").slice(0, 60);
|
|
14084
|
-
const filePath = (0,
|
|
14085
|
-
(0,
|
|
14950
|
+
const filePath = (0, import_node_path8.join)(dir, `${stamp}-${slug}.png`);
|
|
14951
|
+
(0, import_node_fs5.writeFileSync)(filePath, Buffer.from(base64, "base64"));
|
|
14086
14952
|
return filePath;
|
|
14087
14953
|
} catch {
|
|
14088
14954
|
return null;
|
|
@@ -15371,6 +16237,216 @@ ${chunkRows}` : "",
|
|
|
15371
16237
|
}
|
|
15372
16238
|
};
|
|
15373
16239
|
}
|
|
16240
|
+
function structuredInstagramBrowser(raw) {
|
|
16241
|
+
const browser = raw && typeof raw === "object" ? raw : {};
|
|
16242
|
+
const mode = browser.mode === "local" ? "local" : "hosted";
|
|
16243
|
+
const requestedMode = browser.requestedMode === "local" || browser.requestedMode === "hosted" || browser.requestedMode === "auto" ? browser.requestedMode : "auto";
|
|
16244
|
+
const profileSource = browser.profileSource === "managed_profile" || browser.profileSource === "direct_profile_dir" ? browser.profileSource : "hosted";
|
|
16245
|
+
return {
|
|
16246
|
+
mode,
|
|
16247
|
+
requestedMode,
|
|
16248
|
+
profileName: typeof browser.profileName === "string" ? browser.profileName : null,
|
|
16249
|
+
profileSource,
|
|
16250
|
+
profileDirConfigured: browser.profileDirConfigured === true,
|
|
16251
|
+
executablePathConfigured: browser.executablePathConfigured === true
|
|
16252
|
+
};
|
|
16253
|
+
}
|
|
16254
|
+
function structuredInstagramPagination(raw, input) {
|
|
16255
|
+
const pagination = raw && typeof raw === "object" ? raw : {};
|
|
16256
|
+
const stages = Array.isArray(pagination.stages) ? pagination.stages : [];
|
|
16257
|
+
const stoppedReason = typeof pagination.stoppedReason === "string" && ["max_items", "reported_post_count", "stable_scrolls", "max_scrolls", "no_scrolls"].includes(pagination.stoppedReason) ? pagination.stoppedReason : "no_scrolls";
|
|
16258
|
+
return {
|
|
16259
|
+
maxItems: typeof pagination.maxItems === "number" ? pagination.maxItems : Number(input.maxItems ?? 50),
|
|
16260
|
+
maxScrolls: typeof pagination.maxScrolls === "number" ? pagination.maxScrolls : Number(input.maxScrolls ?? 10),
|
|
16261
|
+
attemptedScrolls: typeof pagination.attemptedScrolls === "number" ? pagination.attemptedScrolls : 0,
|
|
16262
|
+
stableScrolls: typeof pagination.stableScrolls === "number" ? pagination.stableScrolls : 0,
|
|
16263
|
+
stableScrollLimit: typeof pagination.stableScrollLimit === "number" ? pagination.stableScrollLimit : Number(input.stableScrollLimit ?? 4),
|
|
16264
|
+
scrollDelayMs: typeof pagination.scrollDelayMs === "number" ? pagination.scrollDelayMs : Number(input.scrollDelayMs ?? 1200),
|
|
16265
|
+
reachedMaxItems: pagination.reachedMaxItems === true,
|
|
16266
|
+
reachedReportedPostCount: pagination.reachedReportedPostCount === true,
|
|
16267
|
+
finalScrollHeight: typeof pagination.finalScrollHeight === "number" ? pagination.finalScrollHeight : null,
|
|
16268
|
+
stoppedReason,
|
|
16269
|
+
stages: stages.map((stage) => {
|
|
16270
|
+
const row = stage && typeof stage === "object" ? stage : {};
|
|
16271
|
+
return {
|
|
16272
|
+
stage: String(row.stage ?? ""),
|
|
16273
|
+
itemCount: typeof row.itemCount === "number" ? row.itemCount : 0,
|
|
16274
|
+
addedCount: typeof row.addedCount === "number" ? row.addedCount : 0,
|
|
16275
|
+
scrollY: typeof row.scrollY === "number" ? row.scrollY : null,
|
|
16276
|
+
scrollHeight: typeof row.scrollHeight === "number" ? row.scrollHeight : null
|
|
16277
|
+
};
|
|
16278
|
+
})
|
|
16279
|
+
};
|
|
16280
|
+
}
|
|
16281
|
+
function formatInstagramProfileContent(raw, input) {
|
|
16282
|
+
const parsed = parseData(raw);
|
|
16283
|
+
if ("error" in parsed) return { content: [{ type: "text", text: parsed.error }], isError: true };
|
|
16284
|
+
const d = parsed.data;
|
|
16285
|
+
const items = Array.isArray(d.items) ? d.items : [];
|
|
16286
|
+
const typeCounts = d.typeCounts;
|
|
16287
|
+
const limitations = Array.isArray(d.limitations) ? d.limitations.map(String) : [];
|
|
16288
|
+
const browser = structuredInstagramBrowser(d.browser);
|
|
16289
|
+
const pagination = structuredInstagramPagination(d.pagination, input);
|
|
16290
|
+
const itemRows = items.slice(0, 100).map(
|
|
16291
|
+
(item, i) => `| ${i + 1} | ${item.type} | \`${item.shortcode}\` | ${item.url} | ${cell(item.firstSeenStage ?? "")} |`
|
|
16292
|
+
).join("\n");
|
|
16293
|
+
const browserLabel = browser.mode === "local" ? `local profile ${browser.profileName ? `\`${browser.profileName}\`` : "(unnamed)"}` : "hosted browser";
|
|
16294
|
+
const full = [
|
|
16295
|
+
`# Instagram Profile Content: ${d.handle ?? input.handle ?? input.url ?? "profile"}`,
|
|
16296
|
+
`**Collected:** ${items.length} items \xB7 posts ${typeCounts?.post ?? 0} \xB7 reels ${typeCounts?.reel ?? 0} \xB7 tv ${typeCounts?.tv ?? 0}`,
|
|
16297
|
+
`**Browser:** ${browserLabel}`,
|
|
16298
|
+
`**Pagination:** ${pagination.attemptedScrolls} scrolls \xB7 stopped: ${pagination.stoppedReason}`,
|
|
16299
|
+
d.reportedPostCountText ? `**Profile count:** ${d.reportedPostCountText}` : "",
|
|
16300
|
+
d.followerCountText ? `**Followers:** ${d.followerCountText}` : "",
|
|
16301
|
+
`
|
|
16302
|
+
## Content Links
|
|
16303
|
+
| # | Type | Shortcode | URL | First Seen |
|
|
16304
|
+
|---|------|-----------|-----|------------|
|
|
16305
|
+
${itemRows || "| \u2014 | \u2014 | \u2014 | \u2014 | \u2014 |"}`,
|
|
16306
|
+
limitations.length ? `
|
|
16307
|
+
## Limits
|
|
16308
|
+
${limitations.map((l) => `- ${l}`).join("\n")}` : "",
|
|
16309
|
+
`
|
|
16310
|
+
---
|
|
16311
|
+
\u{1F4A1} Use \`instagram_media_download\` with one of these URLs to download text, image, reel tracks, and optional transcript.`
|
|
16312
|
+
].filter(Boolean).join("\n");
|
|
16313
|
+
return {
|
|
16314
|
+
...oneBlock(full),
|
|
16315
|
+
structuredContent: {
|
|
16316
|
+
handle: String(d.handle ?? input.handle ?? ""),
|
|
16317
|
+
profileUrl: String(d.profileUrl ?? input.url ?? ""),
|
|
16318
|
+
pageUrl: String(d.pageUrl ?? d.profileUrl ?? input.url ?? ""),
|
|
16319
|
+
browser,
|
|
16320
|
+
profileName: typeof d.profileName === "string" ? d.profileName : null,
|
|
16321
|
+
reportedPostCount: typeof d.reportedPostCount === "number" ? d.reportedPostCount : null,
|
|
16322
|
+
reportedPostCountText: typeof d.reportedPostCountText === "string" ? d.reportedPostCountText : null,
|
|
16323
|
+
followerCountText: typeof d.followerCountText === "string" ? d.followerCountText : null,
|
|
16324
|
+
followingCountText: typeof d.followingCountText === "string" ? d.followingCountText : null,
|
|
16325
|
+
collectedContentCount: items.length,
|
|
16326
|
+
typeCounts: {
|
|
16327
|
+
post: Number(typeCounts?.post ?? 0),
|
|
16328
|
+
reel: Number(typeCounts?.reel ?? 0),
|
|
16329
|
+
tv: Number(typeCounts?.tv ?? 0)
|
|
16330
|
+
},
|
|
16331
|
+
pagination,
|
|
16332
|
+
limited: d.limited === true,
|
|
16333
|
+
limitations,
|
|
16334
|
+
items: items.map((item) => ({
|
|
16335
|
+
url: String(item.url ?? ""),
|
|
16336
|
+
type: item.type,
|
|
16337
|
+
shortcode: String(item.shortcode ?? ""),
|
|
16338
|
+
anchorText: item.anchorText ?? null,
|
|
16339
|
+
firstSeenStage: String(item.firstSeenStage ?? "")
|
|
16340
|
+
}))
|
|
16341
|
+
}
|
|
16342
|
+
};
|
|
16343
|
+
}
|
|
16344
|
+
function structuredInstagramTrack(track) {
|
|
16345
|
+
if (!track) return null;
|
|
16346
|
+
return {
|
|
16347
|
+
url: String(track.url ?? ""),
|
|
16348
|
+
streamType: track.streamType ?? "unknown",
|
|
16349
|
+
bitrate: typeof track.bitrate === "number" ? track.bitrate : null,
|
|
16350
|
+
durationSec: typeof track.durationSec === "number" ? track.durationSec : null,
|
|
16351
|
+
vencodeTag: track.vencodeTag ?? null,
|
|
16352
|
+
width: typeof track.width === "number" ? track.width : null,
|
|
16353
|
+
height: typeof track.height === "number" ? track.height : null
|
|
16354
|
+
};
|
|
16355
|
+
}
|
|
16356
|
+
function formatInstagramMediaDownload(raw, input) {
|
|
16357
|
+
const parsed = parseData(raw);
|
|
16358
|
+
if ("error" in parsed) return { content: [{ type: "text", text: parsed.error }], isError: true };
|
|
16359
|
+
const d = parsed.data;
|
|
16360
|
+
const tracks = Array.isArray(d.tracks) ? d.tracks : [];
|
|
16361
|
+
const downloads = Array.isArray(d.downloads) ? d.downloads : [];
|
|
16362
|
+
const warnings = Array.isArray(d.warnings) ? d.warnings.map(String) : [];
|
|
16363
|
+
const limitations = Array.isArray(d.limitations) ? d.limitations.map(String) : [];
|
|
16364
|
+
const transcript = d.transcript;
|
|
16365
|
+
const transcriptText = transcript?.text ?? "";
|
|
16366
|
+
const chunks = transcript?.chunks ?? [];
|
|
16367
|
+
const browser = structuredInstagramBrowser(d.browser);
|
|
16368
|
+
const browserLabel = browser.mode === "local" ? `local profile ${browser.profileName ? `\`${browser.profileName}\`` : "(unnamed)"}` : "hosted browser";
|
|
16369
|
+
const downloadRows = downloads.map((download, i) => {
|
|
16370
|
+
const status = download.error ? `error: ${cell(download.error)}` : `${download.sizeBytes ?? 0} bytes`;
|
|
16371
|
+
return `| ${i + 1} | ${download.kind} | ${download.savedPath ? `\`${download.savedPath}\`` : "\u2014"} | ${status} |`;
|
|
16372
|
+
}).join("\n");
|
|
16373
|
+
const trackRows = tracks.slice(0, 20).map(
|
|
16374
|
+
(track, i) => `| ${i + 1} | ${track.streamType} | ${track.bitrate ?? "\u2014"} | ${track.durationSec ?? "\u2014"} | ${cell(track.vencodeTag ?? "")} |`
|
|
16375
|
+
).join("\n");
|
|
16376
|
+
const full = [
|
|
16377
|
+
`# Instagram Media Download`,
|
|
16378
|
+
`**URL:** ${d.pageUrl ?? input.url}`,
|
|
16379
|
+
`**Browser:** ${browserLabel}`,
|
|
16380
|
+
d.ownerName ? `**Owner:** ${d.ownerName}` : "",
|
|
16381
|
+
d.shortcode ? `**Shortcode:** \`${d.shortcode}\`` : "",
|
|
16382
|
+
d.caption ? `
|
|
16383
|
+
## Caption
|
|
16384
|
+
${truncate(String(d.caption), 1200)}` : "",
|
|
16385
|
+
d.imageUrl ? `
|
|
16386
|
+
## Image
|
|
16387
|
+
${d.imageUrl}` : "",
|
|
16388
|
+
tracks.length ? `
|
|
16389
|
+
## Media Tracks
|
|
16390
|
+
| # | Type | Bitrate | Duration | Tag |
|
|
16391
|
+
|---|------|---------|----------|-----|
|
|
16392
|
+
${trackRows}` : "\n## Media Tracks\n*None captured.*",
|
|
16393
|
+
downloads.length ? `
|
|
16394
|
+
## Downloads
|
|
16395
|
+
| # | Kind | File | Status |
|
|
16396
|
+
|---|------|------|--------|
|
|
16397
|
+
${downloadRows}` : "",
|
|
16398
|
+
d.outputDir ? `
|
|
16399
|
+
**Output directory:** \`${d.outputDir}\`` : "",
|
|
16400
|
+
transcript ? `
|
|
16401
|
+
## Transcript
|
|
16402
|
+
**${wordCount(transcriptText)} words** \xB7 ${chunks.length} chunks
|
|
16403
|
+
|
|
16404
|
+
${transcriptText}` : "",
|
|
16405
|
+
warnings.length ? `
|
|
16406
|
+
## Warnings
|
|
16407
|
+
${warnings.map((w) => `- ${w}`).join("\n")}` : "",
|
|
16408
|
+
limitations.length ? `
|
|
16409
|
+
## Limits
|
|
16410
|
+
${limitations.map((l) => `- ${l}`).join("\n")}` : "",
|
|
16411
|
+
`
|
|
16412
|
+
---
|
|
16413
|
+
\u{1F4A1} Reels may expose separate video-only and audio-only MP4 tracks. Use the muxed file when present; otherwise use the selected video/audio track URLs or saved files.`
|
|
16414
|
+
].filter(Boolean).join("\n");
|
|
16415
|
+
return {
|
|
16416
|
+
...oneBlock(full),
|
|
16417
|
+
structuredContent: {
|
|
16418
|
+
sourceUrl: String(d.sourceUrl ?? input.url),
|
|
16419
|
+
pageUrl: String(d.pageUrl ?? input.url),
|
|
16420
|
+
browser,
|
|
16421
|
+
type: d.type === "post" || d.type === "reel" || d.type === "tv" ? d.type : null,
|
|
16422
|
+
shortcode: typeof d.shortcode === "string" ? d.shortcode : null,
|
|
16423
|
+
ownerName: typeof d.ownerName === "string" ? d.ownerName : null,
|
|
16424
|
+
caption: typeof d.caption === "string" ? d.caption : null,
|
|
16425
|
+
imageUrl: typeof d.imageUrl === "string" ? d.imageUrl : null,
|
|
16426
|
+
trackCount: tracks.length,
|
|
16427
|
+
selectedVideoTrack: structuredInstagramTrack(d.selectedVideoTrack),
|
|
16428
|
+
selectedAudioTrack: structuredInstagramTrack(d.selectedAudioTrack),
|
|
16429
|
+
downloads: downloads.map((download) => ({
|
|
16430
|
+
kind: download.kind,
|
|
16431
|
+
url: download.url ?? null,
|
|
16432
|
+
savedPath: download.savedPath ?? null,
|
|
16433
|
+
sizeBytes: typeof download.sizeBytes === "number" ? download.sizeBytes : null,
|
|
16434
|
+
mimeType: download.mimeType ?? null,
|
|
16435
|
+
error: download.error ?? null
|
|
16436
|
+
})),
|
|
16437
|
+
outputDir: typeof d.outputDir === "string" ? d.outputDir : null,
|
|
16438
|
+
warnings,
|
|
16439
|
+
limitations,
|
|
16440
|
+
transcript: transcript ? {
|
|
16441
|
+
wordCount: wordCount(transcriptText),
|
|
16442
|
+
chunkCount: chunks.length,
|
|
16443
|
+
durationMs: typeof transcript.durationMs === "number" ? transcript.durationMs : null,
|
|
16444
|
+
transcriptText,
|
|
16445
|
+
chunks: structuredTranscriptChunks(chunks)
|
|
16446
|
+
} : null
|
|
16447
|
+
}
|
|
16448
|
+
};
|
|
16449
|
+
}
|
|
15374
16450
|
function formatCaptureSerpSnapshot(raw, input) {
|
|
15375
16451
|
const parsed = parseData(raw);
|
|
15376
16452
|
if ("error" in parsed) return { content: [{ type: "text", text: parsed.error }], isError: true };
|
|
@@ -15461,13 +16537,13 @@ ${rows}` : ""
|
|
|
15461
16537
|
}
|
|
15462
16538
|
};
|
|
15463
16539
|
}
|
|
15464
|
-
var
|
|
16540
|
+
var import_node_fs5, import_node_os6, import_node_path8, reportSavingEnabled;
|
|
15465
16541
|
var init_mcp_response_formatter = __esm({
|
|
15466
16542
|
"src/mcp/mcp-response-formatter.ts"() {
|
|
15467
16543
|
"use strict";
|
|
15468
|
-
|
|
15469
|
-
|
|
15470
|
-
|
|
16544
|
+
import_node_fs5 = require("fs");
|
|
16545
|
+
import_node_os6 = require("os");
|
|
16546
|
+
import_node_path8 = require("path");
|
|
15471
16547
|
init_errors();
|
|
15472
16548
|
init_workflow_catalog();
|
|
15473
16549
|
reportSavingEnabled = true;
|
|
@@ -15574,7 +16650,7 @@ function localLocationFileAllowed() {
|
|
|
15574
16650
|
async function existingPath(value) {
|
|
15575
16651
|
const trimmed = value?.trim();
|
|
15576
16652
|
if (!trimmed) return null;
|
|
15577
|
-
await (0,
|
|
16653
|
+
await (0, import_promises7.access)(trimmed);
|
|
15578
16654
|
return trimmed;
|
|
15579
16655
|
}
|
|
15580
16656
|
async function resolveUsZipsPath(requestedPath) {
|
|
@@ -15592,7 +16668,7 @@ async function loadZipGroups(stateAbbr, requestedPath, warnings) {
|
|
|
15592
16668
|
}
|
|
15593
16669
|
const path6 = await resolveUsZipsPath(requestedPath);
|
|
15594
16670
|
if (!path6) return { path: null, groups: /* @__PURE__ */ new Map() };
|
|
15595
|
-
const records = csvRecords(await (0,
|
|
16671
|
+
const records = csvRecords(await (0, import_promises7.readFile)(path6, "utf8"));
|
|
15596
16672
|
const groups = /* @__PURE__ */ new Map();
|
|
15597
16673
|
for (const record of records) {
|
|
15598
16674
|
const state = (record.state_abbr ?? record.state ?? "").trim().toUpperCase();
|
|
@@ -15643,11 +16719,11 @@ async function resolveDirectoryMarkets(options) {
|
|
|
15643
16719
|
}
|
|
15644
16720
|
return { markets, censusSourceUrl: sourceUrl, usZipsSourcePath: zipData.path, warnings };
|
|
15645
16721
|
}
|
|
15646
|
-
var
|
|
16722
|
+
var import_promises7, POPULATION_YEARS, STATE_META, STATE_BY_NAME;
|
|
15647
16723
|
var init_location_db = __esm({
|
|
15648
16724
|
"src/directory/location-db.ts"() {
|
|
15649
16725
|
"use strict";
|
|
15650
|
-
|
|
16726
|
+
import_promises7 = require("fs/promises");
|
|
15651
16727
|
init_csv();
|
|
15652
16728
|
POPULATION_YEARS = [2020, 2021, 2022, 2023, 2024, 2025];
|
|
15653
16729
|
STATE_META = {
|
|
@@ -15863,11 +16939,11 @@ function csvRowsFor(result) {
|
|
|
15863
16939
|
return rows;
|
|
15864
16940
|
}
|
|
15865
16941
|
async function saveDirectoryCsv(result) {
|
|
15866
|
-
const outDir = (0,
|
|
15867
|
-
await (0,
|
|
16942
|
+
const outDir = (0, import_node_path9.join)(outputBaseDir2(), "directory-workflows");
|
|
16943
|
+
await (0, import_promises8.mkdir)(outDir, { recursive: true });
|
|
15868
16944
|
const stamp = result.extractedAt.replace(/[:.]/g, "-");
|
|
15869
16945
|
const slug = `${result.state}-${result.query}`.toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-+|-+$/g, "").slice(0, 80);
|
|
15870
|
-
const path6 = (0,
|
|
16946
|
+
const path6 = (0, import_node_path9.join)(outDir, `${stamp}-${slug}-directory-workflow.csv`);
|
|
15871
16947
|
const headers = [
|
|
15872
16948
|
"source_query",
|
|
15873
16949
|
"source_location",
|
|
@@ -15899,7 +16975,7 @@ async function saveDirectoryCsv(result) {
|
|
|
15899
16975
|
"extracted_at",
|
|
15900
16976
|
"duration_ms"
|
|
15901
16977
|
];
|
|
15902
|
-
await (0,
|
|
16978
|
+
await (0, import_promises8.writeFile)(path6, rowsToCsv(headers, csvRowsFor(result)), "utf8");
|
|
15903
16979
|
return path6;
|
|
15904
16980
|
}
|
|
15905
16981
|
async function runDirectoryWorkflowFromPlan(options, plan) {
|
|
@@ -15925,47 +17001,47 @@ async function runDirectoryWorkflowFromPlan(options, plan) {
|
|
|
15925
17001
|
const csvPath = options.saveCsv ? await saveDirectoryCsv(base) : null;
|
|
15926
17002
|
return { ...base, csvPath };
|
|
15927
17003
|
}
|
|
15928
|
-
var
|
|
17004
|
+
var import_promises8, import_node_path9, import_zod19, DirectoryWorkflowOptionsSchema;
|
|
15929
17005
|
var init_directory_workflow = __esm({
|
|
15930
17006
|
"src/directory/directory-workflow.ts"() {
|
|
15931
17007
|
"use strict";
|
|
15932
|
-
|
|
15933
|
-
|
|
15934
|
-
|
|
17008
|
+
import_promises8 = require("fs/promises");
|
|
17009
|
+
import_node_path9 = require("path");
|
|
17010
|
+
import_zod19 = require("zod");
|
|
15935
17011
|
init_mcp_response_formatter();
|
|
15936
17012
|
init_browser_service_env();
|
|
15937
17013
|
init_maps_search_rotation();
|
|
15938
17014
|
init_schemas3();
|
|
15939
17015
|
init_csv();
|
|
15940
17016
|
init_location_db();
|
|
15941
|
-
DirectoryWorkflowOptionsSchema =
|
|
15942
|
-
query:
|
|
15943
|
-
state:
|
|
15944
|
-
minPopulation:
|
|
15945
|
-
populationYear:
|
|
15946
|
-
maxCities:
|
|
15947
|
-
maxResultsPerCity:
|
|
15948
|
-
concurrency:
|
|
15949
|
-
includeZipGroups:
|
|
15950
|
-
usZipsCsvPath:
|
|
15951
|
-
saveCsv:
|
|
15952
|
-
gl:
|
|
15953
|
-
hl:
|
|
15954
|
-
proxyMode:
|
|
15955
|
-
proxyZip:
|
|
15956
|
-
debug:
|
|
15957
|
-
headless:
|
|
15958
|
-
kernelApiKey:
|
|
17017
|
+
DirectoryWorkflowOptionsSchema = import_zod19.z.object({
|
|
17018
|
+
query: import_zod19.z.string().min(1),
|
|
17019
|
+
state: import_zod19.z.string().min(2).default("TN"),
|
|
17020
|
+
minPopulation: import_zod19.z.number().int().min(0).default(1e5),
|
|
17021
|
+
populationYear: import_zod19.z.union(POPULATION_YEARS.map((year) => import_zod19.z.literal(year))).default(2025),
|
|
17022
|
+
maxCities: import_zod19.z.number().int().min(1).max(100).default(25),
|
|
17023
|
+
maxResultsPerCity: import_zod19.z.number().int().min(1).max(50).default(50),
|
|
17024
|
+
concurrency: import_zod19.z.number().int().min(1).max(5).default(5),
|
|
17025
|
+
includeZipGroups: import_zod19.z.boolean().default(true),
|
|
17026
|
+
usZipsCsvPath: import_zod19.z.string().optional(),
|
|
17027
|
+
saveCsv: import_zod19.z.boolean().default(true),
|
|
17028
|
+
gl: import_zod19.z.string().length(2).default("us"),
|
|
17029
|
+
hl: import_zod19.z.string().length(2).default("en"),
|
|
17030
|
+
proxyMode: import_zod19.z.enum(["location", "configured", "none"]).default(DEFAULT_MAPS_PROXY_MODE),
|
|
17031
|
+
proxyZip: import_zod19.z.string().regex(/^\d{5}$/).optional(),
|
|
17032
|
+
debug: import_zod19.z.boolean().default(false),
|
|
17033
|
+
headless: import_zod19.z.boolean().default(true),
|
|
17034
|
+
kernelApiKey: import_zod19.z.string().optional()
|
|
15959
17035
|
});
|
|
15960
17036
|
}
|
|
15961
17037
|
});
|
|
15962
17038
|
|
|
15963
17039
|
// src/api/directory-routes.ts
|
|
15964
|
-
var
|
|
17040
|
+
var import_hono7, directoryApp;
|
|
15965
17041
|
var init_directory_routes = __esm({
|
|
15966
17042
|
"src/api/directory-routes.ts"() {
|
|
15967
17043
|
"use strict";
|
|
15968
|
-
|
|
17044
|
+
import_hono7 = require("hono");
|
|
15969
17045
|
init_api_auth();
|
|
15970
17046
|
init_db();
|
|
15971
17047
|
init_rates();
|
|
@@ -15973,7 +17049,7 @@ var init_directory_routes = __esm({
|
|
|
15973
17049
|
init_location_db();
|
|
15974
17050
|
init_browser_service_env();
|
|
15975
17051
|
init_concurrency_gates();
|
|
15976
|
-
directoryApp = new
|
|
17052
|
+
directoryApp = new import_hono7.Hono();
|
|
15977
17053
|
directoryApp.post("/run", createApiKeyAuth(), async (c) => {
|
|
15978
17054
|
const user = c.get("user");
|
|
15979
17055
|
const body = await c.req.json().catch(() => ({}));
|
|
@@ -16057,7 +17133,7 @@ var init_slugify = __esm({
|
|
|
16057
17133
|
|
|
16058
17134
|
// src/workflows/artifact-writer.ts
|
|
16059
17135
|
function workflowOutputBaseDir(outputDir) {
|
|
16060
|
-
return outputDir?.trim() || process.env.MCP_SCRAPER_OUTPUT_DIR?.trim() || (0,
|
|
17136
|
+
return outputDir?.trim() || process.env.MCP_SCRAPER_OUTPUT_DIR?.trim() || (0, import_node_path10.join)((0, import_node_os7.homedir)(), "Downloads", "mcp-scraper");
|
|
16061
17137
|
}
|
|
16062
17138
|
function timestamp() {
|
|
16063
17139
|
return (/* @__PURE__ */ new Date()).toISOString().replace(/[:.]/g, "-");
|
|
@@ -16066,13 +17142,13 @@ function safeSlug(value) {
|
|
|
16066
17142
|
return slugify(value).replace(/^-+|-+$/g, "").slice(0, 80) || "run";
|
|
16067
17143
|
}
|
|
16068
17144
|
function indexPath(baseDir = workflowOutputBaseDir()) {
|
|
16069
|
-
return (0,
|
|
17145
|
+
return (0, import_node_path10.join)(baseDir, "workflows", "index.json");
|
|
16070
17146
|
}
|
|
16071
17147
|
async function readIndex(baseDir) {
|
|
16072
17148
|
const path6 = indexPath(baseDir);
|
|
16073
|
-
if (!(0,
|
|
17149
|
+
if (!(0, import_node_fs6.existsSync)(path6)) return { runs: [] };
|
|
16074
17150
|
try {
|
|
16075
|
-
return JSON.parse(await (0,
|
|
17151
|
+
return JSON.parse(await (0, import_promises9.readFile)(path6, "utf8"));
|
|
16076
17152
|
} catch {
|
|
16077
17153
|
return { runs: [] };
|
|
16078
17154
|
}
|
|
@@ -16092,18 +17168,18 @@ async function updateWorkflowIndex(manifest, manifestPath, baseDir) {
|
|
|
16092
17168
|
summary: `${manifest.title} (${manifest.status})`
|
|
16093
17169
|
};
|
|
16094
17170
|
index.runs = [entry, ...index.runs.filter((r) => r.runId !== manifest.runId)].slice(0, 200);
|
|
16095
|
-
await (0,
|
|
16096
|
-
await (0,
|
|
17171
|
+
await (0, import_promises9.mkdir)((0, import_node_path10.dirname)(path6), { recursive: true });
|
|
17172
|
+
await (0, import_promises9.writeFile)(path6, JSON.stringify(index, null, 2), "utf8");
|
|
16097
17173
|
}
|
|
16098
|
-
var
|
|
17174
|
+
var import_promises9, import_node_fs6, import_node_os7, import_node_path10, import_node_child_process3, ArtifactWriter;
|
|
16099
17175
|
var init_artifact_writer = __esm({
|
|
16100
17176
|
"src/workflows/artifact-writer.ts"() {
|
|
16101
17177
|
"use strict";
|
|
16102
|
-
|
|
16103
|
-
|
|
16104
|
-
|
|
16105
|
-
|
|
16106
|
-
|
|
17178
|
+
import_promises9 = require("fs/promises");
|
|
17179
|
+
import_node_fs6 = require("fs");
|
|
17180
|
+
import_node_os7 = require("os");
|
|
17181
|
+
import_node_path10 = require("path");
|
|
17182
|
+
import_node_child_process3 = require("child_process");
|
|
16107
17183
|
init_csv();
|
|
16108
17184
|
init_slugify();
|
|
16109
17185
|
ArtifactWriter = class _ArtifactWriter {
|
|
@@ -16129,31 +17205,31 @@ var init_artifact_writer = __esm({
|
|
|
16129
17205
|
const runId = forcedRunId?.trim() || `${timestamp()}-${safeSlug(workflowId)}-${Math.random().toString(36).slice(2, 8)}`;
|
|
16130
17206
|
const nameSource = String(input.keyword ?? input.query ?? input.domain ?? input.state ?? workflowId);
|
|
16131
17207
|
const baseDir = workflowOutputBaseDir(outputDir);
|
|
16132
|
-
const runDir = (0,
|
|
16133
|
-
await (0,
|
|
17208
|
+
const runDir = (0, import_node_path10.join)(baseDir, "workflows", workflowId, `${timestamp()}-${safeSlug(nameSource)}-${safeSlug(runId).slice(0, 20)}`);
|
|
17209
|
+
await (0, import_promises9.mkdir)(runDir, { recursive: true });
|
|
16134
17210
|
return new _ArtifactWriter(workflowId, title, runId, baseDir, runDir, startedAt, input);
|
|
16135
17211
|
}
|
|
16136
17212
|
async remember(kind, label, path6, rows) {
|
|
16137
|
-
const size = await (0,
|
|
17213
|
+
const size = await (0, import_promises9.stat)(path6).then((s) => s.size).catch(() => void 0);
|
|
16138
17214
|
this.artifacts.push({ kind, label, path: path6, bytes: size, rows });
|
|
16139
17215
|
return path6;
|
|
16140
17216
|
}
|
|
16141
17217
|
async writeJson(label, relativePath, data) {
|
|
16142
|
-
const path6 = (0,
|
|
16143
|
-
await (0,
|
|
16144
|
-
await (0,
|
|
17218
|
+
const path6 = (0, import_node_path10.join)(this.runDir, relativePath);
|
|
17219
|
+
await (0, import_promises9.mkdir)((0, import_node_path10.dirname)(path6), { recursive: true });
|
|
17220
|
+
await (0, import_promises9.writeFile)(path6, JSON.stringify(data, null, 2), "utf8");
|
|
16145
17221
|
return this.remember("json", label, path6);
|
|
16146
17222
|
}
|
|
16147
17223
|
async writeText(label, relativePath, text, kind = "markdown") {
|
|
16148
|
-
const path6 = (0,
|
|
16149
|
-
await (0,
|
|
16150
|
-
await (0,
|
|
17224
|
+
const path6 = (0, import_node_path10.join)(this.runDir, relativePath);
|
|
17225
|
+
await (0, import_promises9.mkdir)((0, import_node_path10.dirname)(path6), { recursive: true });
|
|
17226
|
+
await (0, import_promises9.writeFile)(path6, text, "utf8");
|
|
16151
17227
|
return this.remember(kind, label, path6);
|
|
16152
17228
|
}
|
|
16153
17229
|
async writeCsv(label, relativePath, headers, rows) {
|
|
16154
|
-
const path6 = (0,
|
|
16155
|
-
await (0,
|
|
16156
|
-
await (0,
|
|
17230
|
+
const path6 = (0, import_node_path10.join)(this.runDir, relativePath);
|
|
17231
|
+
await (0, import_promises9.mkdir)((0, import_node_path10.dirname)(path6), { recursive: true });
|
|
17232
|
+
await (0, import_promises9.writeFile)(path6, rowsToCsv(headers, rows), "utf8");
|
|
16157
17233
|
return this.remember("csv", label, path6, rows.length);
|
|
16158
17234
|
}
|
|
16159
17235
|
async writeHtml(label, relativePath, html) {
|
|
@@ -16173,8 +17249,8 @@ var init_artifact_writer = __esm({
|
|
|
16173
17249
|
errors,
|
|
16174
17250
|
counts
|
|
16175
17251
|
};
|
|
16176
|
-
const path6 = (0,
|
|
16177
|
-
await (0,
|
|
17252
|
+
const path6 = (0, import_node_path10.join)(this.runDir, "manifest.json");
|
|
17253
|
+
await (0, import_promises9.writeFile)(path6, JSON.stringify(manifest, null, 2), "utf8");
|
|
16178
17254
|
await updateWorkflowIndex(manifest, path6, this.baseDir);
|
|
16179
17255
|
return path6;
|
|
16180
17256
|
}
|
|
@@ -16354,21 +17430,21 @@ function competitorRows(rows, targetDomain) {
|
|
|
16354
17430
|
source_url_count: entry.urls.size
|
|
16355
17431
|
})).sort((a, b) => Number(a.organic_best_position || 999) - Number(b.organic_best_position || 999));
|
|
16356
17432
|
}
|
|
16357
|
-
var
|
|
17433
|
+
var import_zod20, AgentPacketInputSchema, agentPacketWorkflowDefinition;
|
|
16358
17434
|
var init_agent_packet = __esm({
|
|
16359
17435
|
"src/workflows/workflows/agent-packet.ts"() {
|
|
16360
17436
|
"use strict";
|
|
16361
|
-
|
|
17437
|
+
import_zod20 = require("zod");
|
|
16362
17438
|
init_report_renderer();
|
|
16363
|
-
AgentPacketInputSchema =
|
|
16364
|
-
keyword:
|
|
16365
|
-
domain:
|
|
16366
|
-
location:
|
|
16367
|
-
maxQuestions:
|
|
16368
|
-
includeSerp:
|
|
16369
|
-
includePaa:
|
|
16370
|
-
includeAiOverview:
|
|
16371
|
-
returnPartial:
|
|
17439
|
+
AgentPacketInputSchema = import_zod20.z.object({
|
|
17440
|
+
keyword: import_zod20.z.string().min(1),
|
|
17441
|
+
domain: import_zod20.z.string().optional(),
|
|
17442
|
+
location: import_zod20.z.string().optional(),
|
|
17443
|
+
maxQuestions: import_zod20.z.number().int().min(1).max(200).default(40),
|
|
17444
|
+
includeSerp: import_zod20.z.boolean().default(true),
|
|
17445
|
+
includePaa: import_zod20.z.boolean().default(true),
|
|
17446
|
+
includeAiOverview: import_zod20.z.boolean().default(true),
|
|
17447
|
+
returnPartial: import_zod20.z.boolean().default(true)
|
|
16372
17448
|
});
|
|
16373
17449
|
agentPacketWorkflowDefinition = {
|
|
16374
17450
|
id: "agent-packet",
|
|
@@ -16566,22 +17642,22 @@ function directoryRows(result) {
|
|
|
16566
17642
|
}
|
|
16567
17643
|
return rows;
|
|
16568
17644
|
}
|
|
16569
|
-
var
|
|
17645
|
+
var import_zod21, DirectoryWorkflowCliInputSchema, DIRECTORY_CSV_HEADERS, directoryWorkflowDefinition;
|
|
16570
17646
|
var init_directory = __esm({
|
|
16571
17647
|
"src/workflows/workflows/directory.ts"() {
|
|
16572
17648
|
"use strict";
|
|
16573
|
-
|
|
17649
|
+
import_zod21 = require("zod");
|
|
16574
17650
|
init_report_renderer();
|
|
16575
17651
|
init_schemas3();
|
|
16576
|
-
DirectoryWorkflowCliInputSchema =
|
|
16577
|
-
query:
|
|
16578
|
-
state:
|
|
16579
|
-
minPopulation:
|
|
16580
|
-
maxCities:
|
|
16581
|
-
maxResultsPerCity:
|
|
16582
|
-
concurrency:
|
|
16583
|
-
proxyMode:
|
|
16584
|
-
saveCsv:
|
|
17652
|
+
DirectoryWorkflowCliInputSchema = import_zod21.z.object({
|
|
17653
|
+
query: import_zod21.z.string().min(1),
|
|
17654
|
+
state: import_zod21.z.string().min(2).default("TN"),
|
|
17655
|
+
minPopulation: import_zod21.z.number().int().min(0).default(1e5),
|
|
17656
|
+
maxCities: import_zod21.z.number().int().min(1).max(100).default(25),
|
|
17657
|
+
maxResultsPerCity: import_zod21.z.number().int().min(1).max(50).default(20),
|
|
17658
|
+
concurrency: import_zod21.z.number().int().min(1).max(5).default(5),
|
|
17659
|
+
proxyMode: import_zod21.z.enum(["location", "configured", "none"]).default(DEFAULT_MAPS_PROXY_MODE),
|
|
17660
|
+
saveCsv: import_zod21.z.boolean().default(true)
|
|
16585
17661
|
});
|
|
16586
17662
|
DIRECTORY_CSV_HEADERS = [
|
|
16587
17663
|
"source_query",
|
|
@@ -16690,25 +17766,25 @@ function termsFrom(texts) {
|
|
|
16690
17766
|
}
|
|
16691
17767
|
return [...counts.entries()].sort((a, b) => b[1] - a[1]).slice(0, 8).map(([term, count]) => `${term} (${count})`).join("; ");
|
|
16692
17768
|
}
|
|
16693
|
-
var
|
|
17769
|
+
var import_zod22, LocalCompetitiveAuditInputSchema, localCompetitiveAuditWorkflowDefinition;
|
|
16694
17770
|
var init_local_competitive_audit = __esm({
|
|
16695
17771
|
"src/workflows/workflows/local-competitive-audit.ts"() {
|
|
16696
17772
|
"use strict";
|
|
16697
|
-
|
|
17773
|
+
import_zod22 = require("zod");
|
|
16698
17774
|
init_report_renderer();
|
|
16699
17775
|
init_directory();
|
|
16700
17776
|
init_schemas3();
|
|
16701
|
-
LocalCompetitiveAuditInputSchema =
|
|
16702
|
-
query:
|
|
16703
|
-
state:
|
|
16704
|
-
minPopulation:
|
|
16705
|
-
maxCities:
|
|
16706
|
-
maxResultsPerCity:
|
|
16707
|
-
hydrateTop:
|
|
16708
|
-
maxReviews:
|
|
16709
|
-
concurrency:
|
|
16710
|
-
proxyMode:
|
|
16711
|
-
returnPartial:
|
|
17777
|
+
LocalCompetitiveAuditInputSchema = import_zod22.z.object({
|
|
17778
|
+
query: import_zod22.z.string().min(1),
|
|
17779
|
+
state: import_zod22.z.string().min(2).default("TN"),
|
|
17780
|
+
minPopulation: import_zod22.z.number().int().min(0).default(1e5),
|
|
17781
|
+
maxCities: import_zod22.z.number().int().min(1).max(100).default(25),
|
|
17782
|
+
maxResultsPerCity: import_zod22.z.number().int().min(1).max(50).default(20),
|
|
17783
|
+
hydrateTop: import_zod22.z.number().int().min(0).max(10).default(5),
|
|
17784
|
+
maxReviews: import_zod22.z.number().int().min(0).max(500).default(50),
|
|
17785
|
+
concurrency: import_zod22.z.number().int().min(1).max(5).default(5),
|
|
17786
|
+
proxyMode: import_zod22.z.enum(["location", "configured", "none"]).default(DEFAULT_MAPS_PROXY_MODE),
|
|
17787
|
+
returnPartial: import_zod22.z.boolean().default(true)
|
|
16712
17788
|
});
|
|
16713
17789
|
localCompetitiveAuditWorkflowDefinition = {
|
|
16714
17790
|
id: "local-competitive-audit",
|
|
@@ -17151,58 +18227,58 @@ async function extractPages(ctx, sources, warnings, labelPrefix) {
|
|
|
17151
18227
|
}
|
|
17152
18228
|
}).then((items) => items.filter((item) => item !== null));
|
|
17153
18229
|
}
|
|
17154
|
-
var
|
|
18230
|
+
var import_zod23, ProxyModeSchema, MapComparisonInputSchema, SerpComparisonInputSchema, PaaExpansionBriefInputSchema, AiOverviewLanguageInputSchema, mapComparisonWorkflowDefinition, serpComparisonWorkflowDefinition, paaExpansionBriefWorkflowDefinition, aiOverviewLanguageWorkflowDefinition;
|
|
17155
18231
|
var init_comparison_briefs = __esm({
|
|
17156
18232
|
"src/workflows/workflows/comparison-briefs.ts"() {
|
|
17157
18233
|
"use strict";
|
|
17158
|
-
|
|
18234
|
+
import_zod23 = require("zod");
|
|
17159
18235
|
init_report_renderer();
|
|
17160
18236
|
init_directory();
|
|
17161
18237
|
init_seo_workflow_utils();
|
|
17162
18238
|
init_schemas3();
|
|
17163
|
-
ProxyModeSchema =
|
|
17164
|
-
MapComparisonInputSchema =
|
|
17165
|
-
query:
|
|
17166
|
-
location:
|
|
17167
|
-
state:
|
|
17168
|
-
minPopulation:
|
|
17169
|
-
maxCities:
|
|
17170
|
-
maxResultsPerCity:
|
|
17171
|
-
hydrateTop:
|
|
17172
|
-
maxReviews:
|
|
17173
|
-
concurrency:
|
|
18239
|
+
ProxyModeSchema = import_zod23.z.enum(["location", "configured", "none"]);
|
|
18240
|
+
MapComparisonInputSchema = import_zod23.z.object({
|
|
18241
|
+
query: import_zod23.z.string().min(1),
|
|
18242
|
+
location: import_zod23.z.string().optional(),
|
|
18243
|
+
state: import_zod23.z.string().optional(),
|
|
18244
|
+
minPopulation: import_zod23.z.number().int().min(0).default(1e5),
|
|
18245
|
+
maxCities: import_zod23.z.number().int().min(1).max(100).default(5),
|
|
18246
|
+
maxResultsPerCity: import_zod23.z.number().int().min(1).max(50).default(20),
|
|
18247
|
+
hydrateTop: import_zod23.z.number().int().min(0).max(10).default(5),
|
|
18248
|
+
maxReviews: import_zod23.z.number().int().min(0).max(500).default(25),
|
|
18249
|
+
concurrency: import_zod23.z.number().int().min(1).max(5).default(5),
|
|
17174
18250
|
proxyMode: ProxyModeSchema.default(DEFAULT_MAPS_PROXY_MODE),
|
|
17175
|
-
returnPartial:
|
|
18251
|
+
returnPartial: import_zod23.z.boolean().default(true)
|
|
17176
18252
|
}).refine((input) => input.location || input.state, {
|
|
17177
18253
|
message: "Either location or state is required for map-comparison"
|
|
17178
18254
|
});
|
|
17179
|
-
SerpComparisonInputSchema =
|
|
17180
|
-
keyword:
|
|
17181
|
-
domain:
|
|
17182
|
-
url:
|
|
17183
|
-
location:
|
|
17184
|
-
maxResults:
|
|
17185
|
-
maxQuestions:
|
|
17186
|
-
extractTop:
|
|
17187
|
-
includePaa:
|
|
17188
|
-
includeAiOverview:
|
|
17189
|
-
returnPartial:
|
|
18255
|
+
SerpComparisonInputSchema = import_zod23.z.object({
|
|
18256
|
+
keyword: import_zod23.z.string().min(1),
|
|
18257
|
+
domain: import_zod23.z.string().optional(),
|
|
18258
|
+
url: import_zod23.z.string().url().optional(),
|
|
18259
|
+
location: import_zod23.z.string().optional(),
|
|
18260
|
+
maxResults: import_zod23.z.number().int().min(1).max(20).default(10),
|
|
18261
|
+
maxQuestions: import_zod23.z.number().int().min(1).max(200).default(40),
|
|
18262
|
+
extractTop: import_zod23.z.number().int().min(0).max(10).default(5),
|
|
18263
|
+
includePaa: import_zod23.z.boolean().default(true),
|
|
18264
|
+
includeAiOverview: import_zod23.z.boolean().default(true),
|
|
18265
|
+
returnPartial: import_zod23.z.boolean().default(true)
|
|
17190
18266
|
});
|
|
17191
|
-
PaaExpansionBriefInputSchema =
|
|
17192
|
-
keyword:
|
|
17193
|
-
location:
|
|
17194
|
-
maxQuestions:
|
|
17195
|
-
depth:
|
|
17196
|
-
returnPartial:
|
|
18267
|
+
PaaExpansionBriefInputSchema = import_zod23.z.object({
|
|
18268
|
+
keyword: import_zod23.z.string().min(1),
|
|
18269
|
+
location: import_zod23.z.string().optional(),
|
|
18270
|
+
maxQuestions: import_zod23.z.number().int().min(1).max(300).default(80),
|
|
18271
|
+
depth: import_zod23.z.number().int().min(1).max(6).default(3),
|
|
18272
|
+
returnPartial: import_zod23.z.boolean().default(true)
|
|
17197
18273
|
});
|
|
17198
|
-
AiOverviewLanguageInputSchema =
|
|
17199
|
-
keyword:
|
|
17200
|
-
domain:
|
|
17201
|
-
url:
|
|
17202
|
-
location:
|
|
17203
|
-
maxQuestions:
|
|
17204
|
-
extractTop:
|
|
17205
|
-
returnPartial:
|
|
18274
|
+
AiOverviewLanguageInputSchema = import_zod23.z.object({
|
|
18275
|
+
keyword: import_zod23.z.string().min(1),
|
|
18276
|
+
domain: import_zod23.z.string().optional(),
|
|
18277
|
+
url: import_zod23.z.string().url().optional(),
|
|
18278
|
+
location: import_zod23.z.string().optional(),
|
|
18279
|
+
maxQuestions: import_zod23.z.number().int().min(1).max(200).default(40),
|
|
18280
|
+
extractTop: import_zod23.z.number().int().min(0).max(8).default(3),
|
|
18281
|
+
returnPartial: import_zod23.z.boolean().default(true)
|
|
17206
18282
|
});
|
|
17207
18283
|
mapComparisonWorkflowDefinition = {
|
|
17208
18284
|
id: "map-comparison",
|
|
@@ -17653,7 +18729,7 @@ async function runWorkflowStep(id, rawInput, opts) {
|
|
|
17653
18729
|
const writtenArtifacts = await Promise.all(
|
|
17654
18730
|
artifacts.artifacts.map(async (artifact) => ({
|
|
17655
18731
|
...artifact,
|
|
17656
|
-
content: await (0,
|
|
18732
|
+
content: await (0, import_promises10.readFile)(artifact.path, "utf8").catch(() => "")
|
|
17657
18733
|
}))
|
|
17658
18734
|
);
|
|
17659
18735
|
return {
|
|
@@ -17672,12 +18748,12 @@ async function runWorkflowStep(id, rawInput, opts) {
|
|
|
17672
18748
|
artifacts: writtenArtifacts
|
|
17673
18749
|
};
|
|
17674
18750
|
}
|
|
17675
|
-
var
|
|
18751
|
+
var import_promises10, import_zod24, DEFINITIONS;
|
|
17676
18752
|
var init_registry2 = __esm({
|
|
17677
18753
|
"src/workflows/registry.ts"() {
|
|
17678
18754
|
"use strict";
|
|
17679
|
-
|
|
17680
|
-
|
|
18755
|
+
import_promises10 = require("fs/promises");
|
|
18756
|
+
import_zod24 = require("zod");
|
|
17681
18757
|
init_artifact_writer();
|
|
17682
18758
|
init_http_client2();
|
|
17683
18759
|
init_agent_packet();
|
|
@@ -17745,7 +18821,7 @@ async function readManifestFromSummary(summary) {
|
|
|
17745
18821
|
if (!summary.reportPath) return null;
|
|
17746
18822
|
const manifestPath = summary.reportPath.replace(/report\.html$/, "manifest.json");
|
|
17747
18823
|
try {
|
|
17748
|
-
return JSON.parse(await (0,
|
|
18824
|
+
return JSON.parse(await (0, import_promises11.readFile)(manifestPath, "utf8"));
|
|
17749
18825
|
} catch {
|
|
17750
18826
|
return null;
|
|
17751
18827
|
}
|
|
@@ -17960,47 +19036,47 @@ async function dispatchDueWorkflowSchedules(apiUrl, limit = 3) {
|
|
|
17960
19036
|
}
|
|
17961
19037
|
return { dispatched: results.length, results };
|
|
17962
19038
|
}
|
|
17963
|
-
var import_node_crypto3,
|
|
19039
|
+
var import_node_crypto3, import_promises11, import_hono8, import_zod25, workflowApp, WorkflowInputSchema, WorkflowIdSchema, CadenceSchema, ScheduleStatusSchema, RunBodySchema, ScheduleCreateSchema, SchedulePatchSchema, TERMINAL_RUN_STATUSES;
|
|
17964
19040
|
var init_workflow_routes = __esm({
|
|
17965
19041
|
"src/api/workflow-routes.ts"() {
|
|
17966
19042
|
"use strict";
|
|
17967
19043
|
import_node_crypto3 = require("crypto");
|
|
17968
|
-
|
|
17969
|
-
|
|
17970
|
-
|
|
19044
|
+
import_promises11 = require("fs/promises");
|
|
19045
|
+
import_hono8 = require("hono");
|
|
19046
|
+
import_zod25 = require("zod");
|
|
17971
19047
|
init_artifact_writer();
|
|
17972
19048
|
init_registry2();
|
|
17973
19049
|
init_api_auth();
|
|
17974
19050
|
init_db();
|
|
17975
19051
|
init_url_utils();
|
|
17976
19052
|
init_concurrency_gates();
|
|
17977
|
-
workflowApp = new
|
|
17978
|
-
WorkflowInputSchema =
|
|
17979
|
-
WorkflowIdSchema =
|
|
17980
|
-
CadenceSchema =
|
|
17981
|
-
ScheduleStatusSchema =
|
|
17982
|
-
RunBodySchema =
|
|
19053
|
+
workflowApp = new import_hono8.Hono();
|
|
19054
|
+
WorkflowInputSchema = import_zod25.z.record(import_zod25.z.unknown()).default({});
|
|
19055
|
+
WorkflowIdSchema = import_zod25.z.string().min(1);
|
|
19056
|
+
CadenceSchema = import_zod25.z.enum(["daily", "weekly", "monthly"]);
|
|
19057
|
+
ScheduleStatusSchema = import_zod25.z.enum(["active", "paused"]);
|
|
19058
|
+
RunBodySchema = import_zod25.z.object({
|
|
17983
19059
|
workflowId: WorkflowIdSchema,
|
|
17984
19060
|
input: WorkflowInputSchema,
|
|
17985
|
-
webhookUrl:
|
|
19061
|
+
webhookUrl: import_zod25.z.string().url().optional()
|
|
17986
19062
|
});
|
|
17987
|
-
ScheduleCreateSchema =
|
|
19063
|
+
ScheduleCreateSchema = import_zod25.z.object({
|
|
17988
19064
|
workflowId: WorkflowIdSchema,
|
|
17989
|
-
name:
|
|
19065
|
+
name: import_zod25.z.string().min(1).max(120).optional(),
|
|
17990
19066
|
input: WorkflowInputSchema,
|
|
17991
19067
|
cadence: CadenceSchema.default("weekly"),
|
|
17992
|
-
timezone:
|
|
17993
|
-
webhookUrl:
|
|
17994
|
-
nextRunAt:
|
|
19068
|
+
timezone: import_zod25.z.string().min(1).max(64).default("UTC"),
|
|
19069
|
+
webhookUrl: import_zod25.z.string().url().optional(),
|
|
19070
|
+
nextRunAt: import_zod25.z.string().datetime().optional()
|
|
17995
19071
|
});
|
|
17996
|
-
SchedulePatchSchema =
|
|
17997
|
-
name:
|
|
19072
|
+
SchedulePatchSchema = import_zod25.z.object({
|
|
19073
|
+
name: import_zod25.z.string().min(1).max(120).optional(),
|
|
17998
19074
|
status: ScheduleStatusSchema.optional(),
|
|
17999
19075
|
input: WorkflowInputSchema.optional(),
|
|
18000
19076
|
cadence: CadenceSchema.optional(),
|
|
18001
|
-
timezone:
|
|
18002
|
-
webhookUrl:
|
|
18003
|
-
nextRunAt:
|
|
19077
|
+
timezone: import_zod25.z.string().min(1).max(64).optional(),
|
|
19078
|
+
webhookUrl: import_zod25.z.string().url().nullable().optional(),
|
|
19079
|
+
nextRunAt: import_zod25.z.string().datetime().nullable().optional()
|
|
18004
19080
|
});
|
|
18005
19081
|
workflowApp.get("/definitions", createApiKeyAuth(), (c) => {
|
|
18006
19082
|
return c.json({ workflows: listWorkflowDefinitions() });
|
|
@@ -18135,7 +19211,7 @@ var init_workflow_routes = __esm({
|
|
|
18135
19211
|
return new Response(artifact.content, { headers });
|
|
18136
19212
|
}
|
|
18137
19213
|
try {
|
|
18138
|
-
const content = await (0,
|
|
19214
|
+
const content = await (0, import_promises11.readFile)(artifact.path);
|
|
18139
19215
|
return new Response(content, { headers });
|
|
18140
19216
|
} catch {
|
|
18141
19217
|
return c.json({ error: "Artifact file is no longer available" }, 410);
|
|
@@ -19794,52 +20870,52 @@ var init_PAAExtractor = __esm({
|
|
|
19794
20870
|
});
|
|
19795
20871
|
|
|
19796
20872
|
// src/output/OutputSerializer.ts
|
|
19797
|
-
var
|
|
20873
|
+
var import_node_fs7, import_node_path11, import_papaparse2, OutputSerializer;
|
|
19798
20874
|
var init_OutputSerializer = __esm({
|
|
19799
20875
|
"src/output/OutputSerializer.ts"() {
|
|
19800
20876
|
"use strict";
|
|
19801
|
-
|
|
19802
|
-
|
|
20877
|
+
import_node_fs7 = require("fs");
|
|
20878
|
+
import_node_path11 = __toESM(require("path"), 1);
|
|
19803
20879
|
import_papaparse2 = __toESM(require("papaparse"), 1);
|
|
19804
20880
|
OutputSerializer = class {
|
|
19805
20881
|
async writeJSON(result, outputDir) {
|
|
19806
|
-
await
|
|
20882
|
+
await import_node_fs7.promises.mkdir(outputDir, { recursive: true });
|
|
19807
20883
|
const slug = result.seed.toLowerCase().replace(/\W+/g, "-").slice(0, 40);
|
|
19808
20884
|
const filename = `${slug}-${Date.now()}.json`;
|
|
19809
|
-
const fullPath =
|
|
19810
|
-
await
|
|
20885
|
+
const fullPath = import_node_path11.default.join(outputDir, filename);
|
|
20886
|
+
await import_node_fs7.promises.writeFile(fullPath, JSON.stringify(result, null, 2), "utf8");
|
|
19811
20887
|
return fullPath;
|
|
19812
20888
|
}
|
|
19813
20889
|
async writeCSV(rows, outputDir) {
|
|
19814
|
-
await
|
|
20890
|
+
await import_node_fs7.promises.mkdir(outputDir, { recursive: true });
|
|
19815
20891
|
const seedRaw = rows[0]?.seed_query ?? "paa";
|
|
19816
20892
|
const slug = seedRaw.toLowerCase().replace(/\W+/g, "-").slice(0, 40);
|
|
19817
20893
|
const csv = import_papaparse2.default.unparse(rows, { header: true });
|
|
19818
20894
|
const filename = `${slug}-${Date.now()}.csv`;
|
|
19819
|
-
const fullPath =
|
|
19820
|
-
await
|
|
20895
|
+
const fullPath = import_node_path11.default.join(outputDir, filename);
|
|
20896
|
+
await import_node_fs7.promises.writeFile(fullPath, csv, "utf8");
|
|
19821
20897
|
return fullPath;
|
|
19822
20898
|
}
|
|
19823
20899
|
async writeVideoCSV(videos, seed, outputDir) {
|
|
19824
|
-
await
|
|
20900
|
+
await import_node_fs7.promises.mkdir(outputDir, { recursive: true });
|
|
19825
20901
|
const slug = seed.toLowerCase().replace(/\W+/g, "-").slice(0, 40);
|
|
19826
20902
|
const csv = import_papaparse2.default.unparse(videos, { header: true });
|
|
19827
20903
|
const filename = `${slug}-videos-${Date.now()}.csv`;
|
|
19828
|
-
const fullPath =
|
|
19829
|
-
await
|
|
20904
|
+
const fullPath = import_node_path11.default.join(outputDir, filename);
|
|
20905
|
+
await import_node_fs7.promises.writeFile(fullPath, csv, "utf8");
|
|
19830
20906
|
return fullPath;
|
|
19831
20907
|
}
|
|
19832
20908
|
async writeForumCSV(forums, seed, outputDir) {
|
|
19833
|
-
await
|
|
20909
|
+
await import_node_fs7.promises.mkdir(outputDir, { recursive: true });
|
|
19834
20910
|
const slug = seed.toLowerCase().replace(/\W+/g, "-").slice(0, 40);
|
|
19835
20911
|
const csv = import_papaparse2.default.unparse(forums, { header: true });
|
|
19836
20912
|
const filename = `${slug}-forums-${Date.now()}.csv`;
|
|
19837
|
-
const fullPath =
|
|
19838
|
-
await
|
|
20913
|
+
const fullPath = import_node_path11.default.join(outputDir, filename);
|
|
20914
|
+
await import_node_fs7.promises.writeFile(fullPath, csv, "utf8");
|
|
19839
20915
|
return fullPath;
|
|
19840
20916
|
}
|
|
19841
20917
|
async writeAIOverviewCSV(citations, text, seed, outputDir) {
|
|
19842
|
-
await
|
|
20918
|
+
await import_node_fs7.promises.mkdir(outputDir, { recursive: true });
|
|
19843
20919
|
const slug = seed.toLowerCase().replace(/\W+/g, "-").slice(0, 40);
|
|
19844
20920
|
const rows = citations.map((c, i) => ({
|
|
19845
20921
|
seed_query: seed,
|
|
@@ -19849,12 +20925,12 @@ var init_OutputSerializer = __esm({
|
|
|
19849
20925
|
}));
|
|
19850
20926
|
const csv = import_papaparse2.default.unparse(rows, { header: true });
|
|
19851
20927
|
const filename = `${slug}-ai-overview-${Date.now()}.csv`;
|
|
19852
|
-
const fullPath =
|
|
19853
|
-
await
|
|
20928
|
+
const fullPath = import_node_path11.default.join(outputDir, filename);
|
|
20929
|
+
await import_node_fs7.promises.writeFile(fullPath, csv, "utf8");
|
|
19854
20930
|
return fullPath;
|
|
19855
20931
|
}
|
|
19856
20932
|
async writeAIModeCSV(citations, text, seed, outputDir) {
|
|
19857
|
-
await
|
|
20933
|
+
await import_node_fs7.promises.mkdir(outputDir, { recursive: true });
|
|
19858
20934
|
const slug = seed.toLowerCase().replace(/\W+/g, "-").slice(0, 40);
|
|
19859
20935
|
const rows = citations.map((c, i) => ({
|
|
19860
20936
|
seed_query: seed,
|
|
@@ -19864,18 +20940,18 @@ var init_OutputSerializer = __esm({
|
|
|
19864
20940
|
}));
|
|
19865
20941
|
const csv = import_papaparse2.default.unparse(rows, { header: true });
|
|
19866
20942
|
const filename = `${slug}-ai-mode-${Date.now()}.csv`;
|
|
19867
|
-
const fullPath =
|
|
19868
|
-
await
|
|
20943
|
+
const fullPath = import_node_path11.default.join(outputDir, filename);
|
|
20944
|
+
await import_node_fs7.promises.writeFile(fullPath, csv, "utf8");
|
|
19869
20945
|
return fullPath;
|
|
19870
20946
|
}
|
|
19871
20947
|
async writeWhatPeopleSayingCSV(cards, seed, outputDir) {
|
|
19872
|
-
await
|
|
20948
|
+
await import_node_fs7.promises.mkdir(outputDir, { recursive: true });
|
|
19873
20949
|
const slug = seed.toLowerCase().replace(/\W+/g, "-").slice(0, 40);
|
|
19874
20950
|
const rows = cards.map((c) => ({ seed_query: seed, ...c }));
|
|
19875
20951
|
const csv = import_papaparse2.default.unparse(rows, { header: true });
|
|
19876
20952
|
const filename = `${slug}-what-people-saying-${Date.now()}.csv`;
|
|
19877
|
-
const fullPath =
|
|
19878
|
-
await
|
|
20953
|
+
const fullPath = import_node_path11.default.join(outputDir, filename);
|
|
20954
|
+
await import_node_fs7.promises.writeFile(fullPath, csv, "utf8");
|
|
19879
20955
|
return fullPath;
|
|
19880
20956
|
}
|
|
19881
20957
|
};
|
|
@@ -20426,11 +21502,11 @@ function isPublicHttpUrl(value) {
|
|
|
20426
21502
|
return false;
|
|
20427
21503
|
}
|
|
20428
21504
|
}
|
|
20429
|
-
var
|
|
21505
|
+
var import_zod26, SerpIntelligenceDeviceValues, SerpIntelligenceProxyModeValues, SerpIntelligenceAttemptOutcomeValues, SerpIntelligenceLocalizationStatusValues, SerpPageSnapshotSourceKindValues, SerpPageFetchStatusValues, SerpPageFetchedViaValues, HostnameSuffixPattern, Ipv4Pattern, SerpIntelligencePublicHttpUrlSchema, SerpIntelligenceCaptureBodySchema, SerpIntelligencePageSnapshotRequestSchema, SerpIntelligencePageSnapshotsBodySchema, SerpIntelligenceAICitationSchema, SerpIntelligenceOrganicResultSchema, SerpIntelligenceLocationEvidenceSchema, SerpIntelligenceHarvestResultSchema, SerpIntelligenceCaptureAttemptSchema, SerpPageSnapshotCaptureSchema, SerpIntelligenceCaptureResponseSchema, SerpIntelligencePageSnapshotsResponseSchema;
|
|
20430
21506
|
var init_schemas4 = __esm({
|
|
20431
21507
|
"src/serp-intelligence/schemas.ts"() {
|
|
20432
21508
|
"use strict";
|
|
20433
|
-
|
|
21509
|
+
import_zod26 = require("zod");
|
|
20434
21510
|
init_schemas3();
|
|
20435
21511
|
SerpIntelligenceDeviceValues = ["desktop", "mobile"];
|
|
20436
21512
|
SerpIntelligenceProxyModeValues = ["location", "configured", "none"];
|
|
@@ -20465,171 +21541,171 @@ var init_schemas4 = __esm({
|
|
|
20465
21541
|
SerpPageFetchedViaValues = ["fetch", "headless", "browser", "mcp"];
|
|
20466
21542
|
HostnameSuffixPattern = /(^|\.)localhost$/i;
|
|
20467
21543
|
Ipv4Pattern = /^\d{1,3}(?:\.\d{1,3}){3}$/;
|
|
20468
|
-
SerpIntelligencePublicHttpUrlSchema =
|
|
20469
|
-
SerpIntelligenceCaptureBodySchema =
|
|
20470
|
-
query:
|
|
20471
|
-
location:
|
|
20472
|
-
gl:
|
|
20473
|
-
hl:
|
|
20474
|
-
device:
|
|
20475
|
-
proxyMode:
|
|
20476
|
-
proxyZip:
|
|
20477
|
-
pages:
|
|
20478
|
-
debug:
|
|
20479
|
-
includePageSnapshots:
|
|
20480
|
-
pageSnapshotLimit:
|
|
21544
|
+
SerpIntelligencePublicHttpUrlSchema = import_zod26.z.string().url().refine(isPublicHttpUrl, "url must be a public HTTP or HTTPS URL");
|
|
21545
|
+
SerpIntelligenceCaptureBodySchema = import_zod26.z.object({
|
|
21546
|
+
query: import_zod26.z.string().trim().min(1, "query is required"),
|
|
21547
|
+
location: import_zod26.z.string().trim().min(1).optional(),
|
|
21548
|
+
gl: import_zod26.z.string().trim().length(2).default("us"),
|
|
21549
|
+
hl: import_zod26.z.string().trim().length(2).default("en"),
|
|
21550
|
+
device: import_zod26.z.enum(SerpIntelligenceDeviceValues).default("desktop"),
|
|
21551
|
+
proxyMode: import_zod26.z.enum(SerpIntelligenceProxyModeValues).default(DEFAULT_PROXY_MODE),
|
|
21552
|
+
proxyZip: import_zod26.z.string().regex(/^\d{5}$/).optional(),
|
|
21553
|
+
pages: import_zod26.z.number().int().min(1).max(2).default(1),
|
|
21554
|
+
debug: import_zod26.z.boolean().default(false),
|
|
21555
|
+
includePageSnapshots: import_zod26.z.boolean().default(false),
|
|
21556
|
+
pageSnapshotLimit: import_zod26.z.number().int().min(0).max(10).default(0)
|
|
20481
21557
|
}).strict();
|
|
20482
|
-
SerpIntelligencePageSnapshotRequestSchema =
|
|
21558
|
+
SerpIntelligencePageSnapshotRequestSchema = import_zod26.z.object({
|
|
20483
21559
|
url: SerpIntelligencePublicHttpUrlSchema,
|
|
20484
|
-
sourceKind:
|
|
20485
|
-
sourcePosition:
|
|
21560
|
+
sourceKind: import_zod26.z.enum(SerpPageSnapshotSourceKindValues).default("configured_target"),
|
|
21561
|
+
sourcePosition: import_zod26.z.number().int().min(1).optional()
|
|
20486
21562
|
}).strict();
|
|
20487
|
-
SerpIntelligencePageSnapshotsBodySchema =
|
|
20488
|
-
urls:
|
|
20489
|
-
targets:
|
|
20490
|
-
maxConcurrency:
|
|
20491
|
-
timeoutMs:
|
|
20492
|
-
debug:
|
|
21563
|
+
SerpIntelligencePageSnapshotsBodySchema = import_zod26.z.object({
|
|
21564
|
+
urls: import_zod26.z.array(SerpIntelligencePublicHttpUrlSchema).min(1).max(25),
|
|
21565
|
+
targets: import_zod26.z.array(SerpIntelligencePageSnapshotRequestSchema).min(1).max(25).optional(),
|
|
21566
|
+
maxConcurrency: import_zod26.z.number().int().min(1).max(5).default(2),
|
|
21567
|
+
timeoutMs: import_zod26.z.number().int().min(1e3).max(6e4).default(15e3),
|
|
21568
|
+
debug: import_zod26.z.boolean().default(false)
|
|
20493
21569
|
}).strict();
|
|
20494
|
-
SerpIntelligenceAICitationSchema =
|
|
20495
|
-
text:
|
|
20496
|
-
href:
|
|
21570
|
+
SerpIntelligenceAICitationSchema = import_zod26.z.object({
|
|
21571
|
+
text: import_zod26.z.string(),
|
|
21572
|
+
href: import_zod26.z.string()
|
|
20497
21573
|
}).strict();
|
|
20498
|
-
SerpIntelligenceOrganicResultSchema =
|
|
20499
|
-
position:
|
|
20500
|
-
title:
|
|
20501
|
-
url:
|
|
20502
|
-
domain:
|
|
20503
|
-
cite:
|
|
20504
|
-
snippet:
|
|
20505
|
-
isRedditStyle:
|
|
20506
|
-
inlineRating:
|
|
20507
|
-
value:
|
|
20508
|
-
count:
|
|
21574
|
+
SerpIntelligenceOrganicResultSchema = import_zod26.z.object({
|
|
21575
|
+
position: import_zod26.z.number().int().min(1),
|
|
21576
|
+
title: import_zod26.z.string(),
|
|
21577
|
+
url: import_zod26.z.string(),
|
|
21578
|
+
domain: import_zod26.z.string(),
|
|
21579
|
+
cite: import_zod26.z.string().nullable(),
|
|
21580
|
+
snippet: import_zod26.z.string().nullable(),
|
|
21581
|
+
isRedditStyle: import_zod26.z.boolean(),
|
|
21582
|
+
inlineRating: import_zod26.z.object({
|
|
21583
|
+
value: import_zod26.z.string(),
|
|
21584
|
+
count: import_zod26.z.string()
|
|
20509
21585
|
}).strict().nullable()
|
|
20510
21586
|
}).strict();
|
|
20511
|
-
SerpIntelligenceLocationEvidenceSchema =
|
|
20512
|
-
status:
|
|
20513
|
-
expected:
|
|
20514
|
-
city:
|
|
20515
|
-
regionCode:
|
|
20516
|
-
canonicalLocation:
|
|
21587
|
+
SerpIntelligenceLocationEvidenceSchema = import_zod26.z.object({
|
|
21588
|
+
status: import_zod26.z.enum(SerpIntelligenceLocalizationStatusValues),
|
|
21589
|
+
expected: import_zod26.z.object({
|
|
21590
|
+
city: import_zod26.z.string(),
|
|
21591
|
+
regionCode: import_zod26.z.string().nullable(),
|
|
21592
|
+
canonicalLocation: import_zod26.z.string()
|
|
20517
21593
|
}).strict().nullable(),
|
|
20518
|
-
candidates:
|
|
20519
|
-
city:
|
|
20520
|
-
regionCode:
|
|
20521
|
-
count:
|
|
20522
|
-
examples:
|
|
21594
|
+
candidates: import_zod26.z.array(import_zod26.z.object({
|
|
21595
|
+
city: import_zod26.z.string(),
|
|
21596
|
+
regionCode: import_zod26.z.string(),
|
|
21597
|
+
count: import_zod26.z.number().int().min(0),
|
|
21598
|
+
examples: import_zod26.z.array(import_zod26.z.string())
|
|
20523
21599
|
}).strict())
|
|
20524
21600
|
}).strict();
|
|
20525
|
-
SerpIntelligenceHarvestResultSchema =
|
|
20526
|
-
seed:
|
|
20527
|
-
location:
|
|
20528
|
-
extractedAt:
|
|
20529
|
-
totalQuestions:
|
|
20530
|
-
surface:
|
|
20531
|
-
aiOverview:
|
|
20532
|
-
detected:
|
|
20533
|
-
text:
|
|
20534
|
-
citations:
|
|
20535
|
-
expanded:
|
|
20536
|
-
fullyExpanded:
|
|
20537
|
-
sections:
|
|
21601
|
+
SerpIntelligenceHarvestResultSchema = import_zod26.z.object({
|
|
21602
|
+
seed: import_zod26.z.string(),
|
|
21603
|
+
location: import_zod26.z.string().nullable(),
|
|
21604
|
+
extractedAt: import_zod26.z.string(),
|
|
21605
|
+
totalQuestions: import_zod26.z.number().int().min(0),
|
|
21606
|
+
surface: import_zod26.z.enum(["web", "aim", "unknown"]),
|
|
21607
|
+
aiOverview: import_zod26.z.object({
|
|
21608
|
+
detected: import_zod26.z.boolean(),
|
|
21609
|
+
text: import_zod26.z.string().nullable(),
|
|
21610
|
+
citations: import_zod26.z.array(SerpIntelligenceAICitationSchema),
|
|
21611
|
+
expanded: import_zod26.z.boolean().optional(),
|
|
21612
|
+
fullyExpanded: import_zod26.z.boolean().optional(),
|
|
21613
|
+
sections: import_zod26.z.array(import_zod26.z.string()).optional()
|
|
20538
21614
|
}).strict(),
|
|
20539
|
-
aiMode:
|
|
20540
|
-
detected:
|
|
20541
|
-
text:
|
|
20542
|
-
citations:
|
|
21615
|
+
aiMode: import_zod26.z.object({
|
|
21616
|
+
detected: import_zod26.z.boolean(),
|
|
21617
|
+
text: import_zod26.z.string().nullable(),
|
|
21618
|
+
citations: import_zod26.z.array(SerpIntelligenceAICitationSchema)
|
|
20543
21619
|
}).strict(),
|
|
20544
|
-
tree:
|
|
20545
|
-
flat:
|
|
20546
|
-
videos:
|
|
20547
|
-
forums:
|
|
20548
|
-
organicResults:
|
|
20549
|
-
localPack:
|
|
20550
|
-
entityIds:
|
|
20551
|
-
entities:
|
|
20552
|
-
name:
|
|
20553
|
-
kgId:
|
|
20554
|
-
cid:
|
|
20555
|
-
gcid:
|
|
21620
|
+
tree: import_zod26.z.array(import_zod26.z.unknown()),
|
|
21621
|
+
flat: import_zod26.z.array(import_zod26.z.unknown()),
|
|
21622
|
+
videos: import_zod26.z.array(import_zod26.z.unknown()),
|
|
21623
|
+
forums: import_zod26.z.array(import_zod26.z.unknown()),
|
|
21624
|
+
organicResults: import_zod26.z.array(SerpIntelligenceOrganicResultSchema),
|
|
21625
|
+
localPack: import_zod26.z.array(import_zod26.z.unknown()),
|
|
21626
|
+
entityIds: import_zod26.z.object({
|
|
21627
|
+
entities: import_zod26.z.array(import_zod26.z.object({
|
|
21628
|
+
name: import_zod26.z.string(),
|
|
21629
|
+
kgId: import_zod26.z.string().nullable(),
|
|
21630
|
+
cid: import_zod26.z.string().nullable(),
|
|
21631
|
+
gcid: import_zod26.z.string().nullable()
|
|
20556
21632
|
}).strict()),
|
|
20557
|
-
kgIds:
|
|
20558
|
-
cids:
|
|
20559
|
-
gcids:
|
|
21633
|
+
kgIds: import_zod26.z.array(import_zod26.z.string()),
|
|
21634
|
+
cids: import_zod26.z.array(import_zod26.z.string()),
|
|
21635
|
+
gcids: import_zod26.z.array(import_zod26.z.string())
|
|
20560
21636
|
}).strict(),
|
|
20561
|
-
stats:
|
|
20562
|
-
seed:
|
|
20563
|
-
totalQuestions:
|
|
20564
|
-
maxDepthReached:
|
|
20565
|
-
durationMs:
|
|
20566
|
-
errorCount:
|
|
21637
|
+
stats: import_zod26.z.object({
|
|
21638
|
+
seed: import_zod26.z.string(),
|
|
21639
|
+
totalQuestions: import_zod26.z.number().int().min(0),
|
|
21640
|
+
maxDepthReached: import_zod26.z.number().int().min(0),
|
|
21641
|
+
durationMs: import_zod26.z.number().min(0),
|
|
21642
|
+
errorCount: import_zod26.z.number().int().min(0)
|
|
20567
21643
|
}).strict(),
|
|
20568
|
-
diagnostics:
|
|
20569
|
-
completionStatus:
|
|
20570
|
-
problem:
|
|
20571
|
-
warnings:
|
|
20572
|
-
debug:
|
|
21644
|
+
diagnostics: import_zod26.z.object({
|
|
21645
|
+
completionStatus: import_zod26.z.enum(["paa_found", "no_paa", "serp_only"]),
|
|
21646
|
+
problem: import_zod26.z.null(),
|
|
21647
|
+
warnings: import_zod26.z.array(import_zod26.z.unknown()).optional(),
|
|
21648
|
+
debug: import_zod26.z.object({
|
|
20573
21649
|
locationEvidence: SerpIntelligenceLocationEvidenceSchema.optional()
|
|
20574
21650
|
}).passthrough().optional()
|
|
20575
21651
|
}).passthrough(),
|
|
20576
|
-
whatPeopleSaying:
|
|
21652
|
+
whatPeopleSaying: import_zod26.z.array(import_zod26.z.unknown())
|
|
20577
21653
|
}).strict();
|
|
20578
|
-
SerpIntelligenceCaptureAttemptSchema =
|
|
20579
|
-
attemptNumber:
|
|
20580
|
-
outcome:
|
|
20581
|
-
startedAt:
|
|
20582
|
-
completedAt:
|
|
20583
|
-
durationMs:
|
|
20584
|
-
problemCode:
|
|
20585
|
-
message:
|
|
20586
|
-
kernelSessionId:
|
|
20587
|
-
cleanupSucceeded:
|
|
21654
|
+
SerpIntelligenceCaptureAttemptSchema = import_zod26.z.object({
|
|
21655
|
+
attemptNumber: import_zod26.z.number().int().min(1),
|
|
21656
|
+
outcome: import_zod26.z.enum(SerpIntelligenceAttemptOutcomeValues),
|
|
21657
|
+
startedAt: import_zod26.z.string().optional(),
|
|
21658
|
+
completedAt: import_zod26.z.string().optional(),
|
|
21659
|
+
durationMs: import_zod26.z.number().min(0).optional(),
|
|
21660
|
+
problemCode: import_zod26.z.string().optional(),
|
|
21661
|
+
message: import_zod26.z.string().optional(),
|
|
21662
|
+
kernelSessionId: import_zod26.z.string().nullable().optional(),
|
|
21663
|
+
cleanupSucceeded: import_zod26.z.boolean().nullable().optional()
|
|
20588
21664
|
}).strict();
|
|
20589
|
-
SerpPageSnapshotCaptureSchema =
|
|
21665
|
+
SerpPageSnapshotCaptureSchema = import_zod26.z.object({
|
|
20590
21666
|
url: SerpIntelligencePublicHttpUrlSchema,
|
|
20591
21667
|
requestedUrl: SerpIntelligencePublicHttpUrlSchema,
|
|
20592
21668
|
finalUrl: SerpIntelligencePublicHttpUrlSchema.nullable(),
|
|
20593
|
-
sourceKind:
|
|
20594
|
-
sourcePosition:
|
|
20595
|
-
status:
|
|
20596
|
-
fetchedVia:
|
|
20597
|
-
httpStatus:
|
|
20598
|
-
contentType:
|
|
20599
|
-
title:
|
|
21669
|
+
sourceKind: import_zod26.z.enum(SerpPageSnapshotSourceKindValues),
|
|
21670
|
+
sourcePosition: import_zod26.z.number().int().min(1).nullable(),
|
|
21671
|
+
status: import_zod26.z.enum(SerpPageFetchStatusValues),
|
|
21672
|
+
fetchedVia: import_zod26.z.enum(SerpPageFetchedViaValues).nullable(),
|
|
21673
|
+
httpStatus: import_zod26.z.number().int().min(100).max(599).nullable(),
|
|
21674
|
+
contentType: import_zod26.z.string().nullable(),
|
|
21675
|
+
title: import_zod26.z.string().nullable(),
|
|
20600
21676
|
canonicalUrl: SerpIntelligencePublicHttpUrlSchema.nullable(),
|
|
20601
|
-
metaDescription:
|
|
20602
|
-
headings:
|
|
20603
|
-
level:
|
|
20604
|
-
text:
|
|
21677
|
+
metaDescription: import_zod26.z.string().nullable(),
|
|
21678
|
+
headings: import_zod26.z.array(import_zod26.z.object({
|
|
21679
|
+
level: import_zod26.z.number().int().min(1).max(6),
|
|
21680
|
+
text: import_zod26.z.string()
|
|
20605
21681
|
}).strict()).default([]),
|
|
20606
|
-
artifact:
|
|
20607
|
-
htmlBlobUrl:
|
|
20608
|
-
textBlobUrl:
|
|
20609
|
-
markdownBlobUrl:
|
|
20610
|
-
screenshotBlobUrl:
|
|
20611
|
-
contentSha256:
|
|
20612
|
-
capturedAt:
|
|
21682
|
+
artifact: import_zod26.z.object({
|
|
21683
|
+
htmlBlobUrl: import_zod26.z.string().url().nullable(),
|
|
21684
|
+
textBlobUrl: import_zod26.z.string().url().nullable(),
|
|
21685
|
+
markdownBlobUrl: import_zod26.z.string().url().nullable(),
|
|
21686
|
+
screenshotBlobUrl: import_zod26.z.string().url().nullable(),
|
|
21687
|
+
contentSha256: import_zod26.z.string().nullable(),
|
|
21688
|
+
capturedAt: import_zod26.z.string().nullable()
|
|
20613
21689
|
}).strict(),
|
|
20614
|
-
error:
|
|
20615
|
-
code:
|
|
20616
|
-
message:
|
|
21690
|
+
error: import_zod26.z.object({
|
|
21691
|
+
code: import_zod26.z.string(),
|
|
21692
|
+
message: import_zod26.z.string()
|
|
20617
21693
|
}).strict().nullable()
|
|
20618
21694
|
}).strict();
|
|
20619
|
-
SerpIntelligenceCaptureResponseSchema =
|
|
21695
|
+
SerpIntelligenceCaptureResponseSchema = import_zod26.z.object({
|
|
20620
21696
|
harvestResult: SerpIntelligenceHarvestResultSchema,
|
|
20621
|
-
attempts:
|
|
21697
|
+
attempts: import_zod26.z.array(SerpIntelligenceCaptureAttemptSchema),
|
|
20622
21698
|
locationEvidence: SerpIntelligenceLocationEvidenceSchema.nullable(),
|
|
20623
|
-
pageSnapshotArtifacts:
|
|
20624
|
-
billing:
|
|
20625
|
-
creditsUsed:
|
|
20626
|
-
requestId:
|
|
20627
|
-
jobId:
|
|
21699
|
+
pageSnapshotArtifacts: import_zod26.z.array(SerpPageSnapshotCaptureSchema),
|
|
21700
|
+
billing: import_zod26.z.object({
|
|
21701
|
+
creditsUsed: import_zod26.z.number().min(0).optional(),
|
|
21702
|
+
requestId: import_zod26.z.string().optional(),
|
|
21703
|
+
jobId: import_zod26.z.string().optional()
|
|
20628
21704
|
}).strict().optional()
|
|
20629
21705
|
}).strict();
|
|
20630
|
-
SerpIntelligencePageSnapshotsResponseSchema =
|
|
20631
|
-
pageSnapshotArtifacts:
|
|
20632
|
-
attempts:
|
|
21706
|
+
SerpIntelligencePageSnapshotsResponseSchema = import_zod26.z.object({
|
|
21707
|
+
pageSnapshotArtifacts: import_zod26.z.array(SerpPageSnapshotCaptureSchema),
|
|
21708
|
+
attempts: import_zod26.z.array(SerpIntelligenceCaptureAttemptSchema).default([])
|
|
20633
21709
|
}).strict();
|
|
20634
21710
|
}
|
|
20635
21711
|
});
|
|
@@ -20853,11 +21929,11 @@ function pageSnapshotTargetsFromBody(body) {
|
|
|
20853
21929
|
sourcePosition: null
|
|
20854
21930
|
}));
|
|
20855
21931
|
}
|
|
20856
|
-
var
|
|
21932
|
+
var import_hono9, SERP_INTELLIGENCE_RATE_LIMIT, SERP_INTELLIGENCE_RATE_WINDOW_SECONDS, POST_CAPTURE_ROUTE_LABEL, POST_PAGE_SNAPSHOTS_ROUTE_LABEL, serpIntelligenceApp;
|
|
20857
21933
|
var init_serp_intelligence_routes = __esm({
|
|
20858
21934
|
"src/api/serp-intelligence-routes.ts"() {
|
|
20859
21935
|
"use strict";
|
|
20860
|
-
|
|
21936
|
+
import_hono9 = require("hono");
|
|
20861
21937
|
init_browser_service_env();
|
|
20862
21938
|
init_page_snapshot_extractor();
|
|
20863
21939
|
init_serp_capture_service();
|
|
@@ -20870,7 +21946,7 @@ var init_serp_intelligence_routes = __esm({
|
|
|
20870
21946
|
SERP_INTELLIGENCE_RATE_WINDOW_SECONDS = 60;
|
|
20871
21947
|
POST_CAPTURE_ROUTE_LABEL = "POST /capture";
|
|
20872
21948
|
POST_PAGE_SNAPSHOTS_ROUTE_LABEL = "POST /page-snapshots";
|
|
20873
|
-
serpIntelligenceApp = new
|
|
21949
|
+
serpIntelligenceApp = new import_hono9.Hono();
|
|
20874
21950
|
serpIntelligenceApp.use("*", createApiKeyAuth());
|
|
20875
21951
|
serpIntelligenceApp.post("/capture", async (c) => {
|
|
20876
21952
|
void POST_CAPTURE_ROUTE_LABEL;
|
|
@@ -21002,139 +22078,159 @@ var PACKAGE_VERSION;
|
|
|
21002
22078
|
var init_version = __esm({
|
|
21003
22079
|
"src/version.ts"() {
|
|
21004
22080
|
"use strict";
|
|
21005
|
-
PACKAGE_VERSION = "0.3.
|
|
22081
|
+
PACKAGE_VERSION = "0.3.2";
|
|
21006
22082
|
}
|
|
21007
22083
|
});
|
|
21008
22084
|
|
|
21009
22085
|
// src/mcp/mcp-tool-schemas.ts
|
|
21010
|
-
var
|
|
22086
|
+
var import_zod27, HarvestPaaInputSchema, ExtractUrlInputSchema, MapSiteUrlsInputSchema, ExtractSiteInputSchema, YoutubeHarvestInputSchema, YoutubeTranscribeInputSchema, FacebookPageIntelInputSchema, FacebookAdSearchInputSchema, FacebookAdTranscribeInputSchema, FacebookVideoTranscribeInputSchema, InstagramProfileContentInputSchema, InstagramMediaDownloadInputSchema, MapsPlaceIntelInputSchema, MapsSearchInputSchema, DirectoryWorkflowInputSchema, RankTrackerModeSchema, RankTrackerBlueprintInputSchema, NullableString, MapsSearchAttemptOutput, MapsSearchOutputSchema, DirectoryMapsBusinessOutput, DirectoryWorkflowOutputSchema, RankTrackerToolPlanOutput, RankTrackerTableOutput, RankTrackerCronJobOutput, RankTrackerBlueprintOutputSchema, OrganicResultOutput, AiOverviewOutput, EntityIdsOutput, HarvestPaaOutputSchema, SearchSerpOutputSchema, ExtractUrlOutputSchema, ExtractSiteOutputSchema, MapsPlaceIntelOutputSchema, CreditsInfoOutputSchema, MapSiteUrlsOutputSchema, YoutubeHarvestOutputSchema, FacebookAdSearchOutputSchema, FacebookPageIntelOutputSchema, FacebookVideoTranscribeOutputSchema, TranscriptChunkOutput, InstagramBrowserOutput, InstagramPaginationOutput, InstagramProfileContentOutputSchema, InstagramMediaTrackOutput, InstagramDownloadOutput, InstagramMediaDownloadOutputSchema, YoutubeTranscribeOutputSchema, FacebookAdTranscribeOutputSchema, CaptureSerpSnapshotOutputSchema, CaptureSerpPageSnapshotsOutputSchema, CreditsInfoInputSchema, WorkflowIdSchema2, WorkflowListInputSchema, WorkflowSuggestInputSchema, WorkflowRunInputSchema, WorkflowStepInputSchema, WorkflowStatusInputSchema, WorkflowArtifactReadInputSchema, WorkflowRecipeOutput, WorkflowDefinitionOutput, WorkflowArtifactOutput, WorkflowListOutputSchema, WorkflowSuggestOutputSchema, WorkflowRunOutputSchema, WorkflowStepOutputSchema, WorkflowStatusOutputSchema, WorkflowArtifactReadOutputSchema, SearchSerpInputSchema, CaptureSerpSnapshotInputSchema, ScreenshotInputSchema, CaptureSerpPageSnapshotsInputSchema;
|
|
21011
22087
|
var init_mcp_tool_schemas = __esm({
|
|
21012
22088
|
"src/mcp/mcp-tool-schemas.ts"() {
|
|
21013
22089
|
"use strict";
|
|
21014
|
-
|
|
22090
|
+
import_zod27 = require("zod");
|
|
21015
22091
|
init_schemas3();
|
|
21016
22092
|
HarvestPaaInputSchema = {
|
|
21017
|
-
query:
|
|
21018
|
-
location:
|
|
21019
|
-
maxQuestions:
|
|
21020
|
-
gl:
|
|
21021
|
-
hl:
|
|
21022
|
-
device:
|
|
21023
|
-
proxyMode:
|
|
21024
|
-
proxyZip:
|
|
21025
|
-
debug:
|
|
22093
|
+
query: import_zod27.z.string().min(1).describe('Core search topic only. If the user says "best hvac company in Denver CO", use query="best hvac company" and location="Denver, CO". Do not include the location in query when it can be separated.'),
|
|
22094
|
+
location: import_zod27.z.string().optional().describe('City, region, or country for geo-targeted results, inferred from the user request when present, e.g. "Denver, CO", "Tokyo, Japan", "London, UK".'),
|
|
22095
|
+
maxQuestions: import_zod27.z.number().int().min(1).max(200).default(30).describe("Number of PAA questions to extract. Default 30. Maximum 200. Use 10 for quick probes, 30 for normal research, 100-200 when the user asks for everything/full/deep research. Larger harvests get a longer server time budget (151-200 questions \u2192 up to 280s). Credits are charged by extracted question; unused request hold is refunded."),
|
|
22096
|
+
gl: import_zod27.z.string().length(2).default("us").describe("Google country code inferred from location or user language. Examples: United States us, United Kingdom gb, Japan jp, Canada ca, Australia au."),
|
|
22097
|
+
hl: import_zod27.z.string().default("en").describe("Google interface/content language inferred from the user request. Use en unless the user asks for another language or locale."),
|
|
22098
|
+
device: import_zod27.z.enum(["desktop", "mobile"]).default("desktop").describe("SERP device context. Use desktop by default; use mobile only when the user asks for mobile rankings."),
|
|
22099
|
+
proxyMode: import_zod27.z.enum(["location", "configured", "none"]).default(DEFAULT_PROXY_MODE).describe("Proxy targeting mode. Default configured uses the service proxy without city/ZIP targeting for the highest general success rate. Use location only when the user explicitly needs city/ZIP-targeted residential proxy evidence. Use none only for direct-network debugging."),
|
|
22100
|
+
proxyZip: import_zod27.z.string().regex(/^\d{5}$/).optional().describe("Optional US ZIP override for residential location proxy targeting. Use only with proxyMode location when the user gives a specific ZIP or city-center targeting needs to be forced."),
|
|
22101
|
+
debug: import_zod27.z.boolean().default(false).describe("Include sanitized browser/session/location diagnostics in the response. Use true when debugging localization, CAPTCHA, or proxy behavior.")
|
|
21026
22102
|
};
|
|
21027
22103
|
ExtractUrlInputSchema = {
|
|
21028
|
-
url:
|
|
21029
|
-
screenshot:
|
|
21030
|
-
screenshotDevice:
|
|
21031
|
-
extractBranding:
|
|
21032
|
-
downloadMedia:
|
|
21033
|
-
mediaTypes:
|
|
21034
|
-
allowLocal:
|
|
22104
|
+
url: import_zod27.z.string().url().describe("Public http/https URL to extract. Use this when the user provides one specific page URL."),
|
|
22105
|
+
screenshot: import_zod27.z.boolean().default(false).describe("Also capture a full-page screenshot of the URL. Saved to ~/Downloads/mcp-scraper/screenshots/ and returned inline. Use when the user asks to see or capture the page visually."),
|
|
22106
|
+
screenshotDevice: import_zod27.z.enum(["desktop", "mobile"]).default("desktop").describe("Viewport for screenshot. desktop = 1440\xD7900. mobile = 390\xD7844. Default desktop."),
|
|
22107
|
+
extractBranding: import_zod27.z.boolean().default(false).describe("Extract brand colors, fonts, logo, and favicon using a rendered browser session. Returns colorScheme (light/dark), colors (primary/accent/background/text/heading as hex), fonts (heading/body family names), and assets (logo URL, favicon URL). Use when the user asks about brand colors, site theme, or brand assets."),
|
|
22108
|
+
downloadMedia: import_zod27.z.boolean().default(false).describe("Extract and download all page media (images, video, audio) to ~/Downloads/mcp-scraper/media/. Ad networks, tracking pixels, and noise URLs are filtered automatically. Use when the user asks to download or harvest assets from a page."),
|
|
22109
|
+
mediaTypes: import_zod27.z.array(import_zod27.z.enum(["image", "video", "audio"])).default(["image", "video", "audio"]).describe("Which media types to download. Default all three."),
|
|
22110
|
+
allowLocal: import_zod27.z.boolean().default(false).describe("Allow localhost and private-network URLs. For local development only.")
|
|
21035
22111
|
};
|
|
21036
22112
|
MapSiteUrlsInputSchema = {
|
|
21037
|
-
url:
|
|
21038
|
-
maxUrls:
|
|
22113
|
+
url: import_zod27.z.string().url().describe("Public website URL or domain to crawl for internal URLs. Use before extract_site when the user asks to audit/map/crawl a site."),
|
|
22114
|
+
maxUrls: import_zod27.z.number().int().min(1).max(500).optional().describe("Maximum URLs to discover. Use 100 for normal maps, higher when the user asks for a full inventory.")
|
|
21039
22115
|
};
|
|
21040
22116
|
ExtractSiteInputSchema = {
|
|
21041
|
-
url:
|
|
21042
|
-
maxPages:
|
|
22117
|
+
url: import_zod27.z.string().url().describe("Public website URL or domain to extract across multiple pages. Use when the user asks for a site audit, website crawl, or full-site content/schema extraction."),
|
|
22118
|
+
maxPages: import_zod27.z.number().int().min(1).max(50).optional().describe("Maximum pages to extract. Use 50 when the user asks for full results or a complete crawl within MCP limits.")
|
|
21043
22119
|
};
|
|
21044
22120
|
YoutubeHarvestInputSchema = {
|
|
21045
|
-
mode:
|
|
21046
|
-
query:
|
|
21047
|
-
channelHandle:
|
|
21048
|
-
maxVideos:
|
|
22121
|
+
mode: import_zod27.z.enum(["search", "channel"]).describe("Use search for topic/keyword requests. Use channel when the user provides @handle, channel ID, or channel URL."),
|
|
22122
|
+
query: import_zod27.z.string().optional().describe("Required when mode is search. The YouTube search topic in the user\u2019s words."),
|
|
22123
|
+
channelHandle: import_zod27.z.string().optional().describe("YouTube channel handle, channel ID, or URL. Examples: @mkbhd, UC..., https://youtube.com/@mkbhd."),
|
|
22124
|
+
maxVideos: import_zod27.z.number().int().min(1).max(500).default(50).describe("Number of videos to return. Default 50, maximum 500. Use 10-25 for quick topic discovery, 50 for normal channel/search harvests, and larger values only when the user asks for full channel/history because large responses consume more context.")
|
|
21049
22125
|
};
|
|
21050
22126
|
YoutubeTranscribeInputSchema = {
|
|
21051
|
-
videoId:
|
|
21052
|
-
url:
|
|
22127
|
+
videoId: import_zod27.z.string().min(1).optional().describe("YouTube video ID, e.g. dQw4w9WgXcQ. Use only an ID returned by youtube_harvest or visible in a YouTube URL; do not invent one."),
|
|
22128
|
+
url: import_zod27.z.string().url().optional().describe("Full YouTube URL, e.g. https://www.youtube.com/watch?v=dQw4w9WgXcQ or https://youtu.be/dQw4w9WgXcQ. Use this when the user pasted a URL instead of an ID. Provide videoId or url.")
|
|
21053
22129
|
};
|
|
21054
22130
|
FacebookPageIntelInputSchema = {
|
|
21055
|
-
pageId:
|
|
21056
|
-
libraryId:
|
|
21057
|
-
query:
|
|
21058
|
-
maxAds:
|
|
21059
|
-
country:
|
|
22131
|
+
pageId: import_zod27.z.string().optional().describe("Facebook advertiser/page ID. Use only a pageId returned by facebook_ad_search/facebook_page_intel or copied from Facebook Ad Library; do not construct one yourself."),
|
|
22132
|
+
libraryId: import_zod27.z.string().optional().describe("Facebook Ad Library archive ID for a known ad or advertiser sample. Use a libraryId returned by facebook_ad_search, or a libraryId/adArchiveId visible in Ad Library."),
|
|
22133
|
+
query: import_zod27.z.string().optional().describe("Advertiser or brand name when pageId/libraryId is not known. One of pageId, libraryId, or query is required."),
|
|
22134
|
+
maxAds: import_zod27.z.number().int().min(1).max(200).default(50).describe("Maximum ads to inspect. Default 50, maximum 200. Prefer 25-50 for focused advertiser scans; use 100-200 only when the user asks for a broad ad archive sweep."),
|
|
22135
|
+
country: import_zod27.z.string().length(2).default("US").describe("Two-letter Ad Library country code. Default US. Examples: US, CA, GB, AU. Infer from the user request when they name a country.")
|
|
21060
22136
|
};
|
|
21061
22137
|
FacebookAdSearchInputSchema = {
|
|
21062
|
-
query:
|
|
21063
|
-
country:
|
|
21064
|
-
maxResults:
|
|
22138
|
+
query: import_zod27.z.string().min(1).describe("Advertiser, brand, competitor, niche, or keyword to search in Facebook Ad Library."),
|
|
22139
|
+
country: import_zod27.z.string().length(2).default("US").describe("Two-letter Ad Library country code. Default US. Examples: US, CA, GB, AU."),
|
|
22140
|
+
maxResults: import_zod27.z.number().int().min(1).max(20).default(10).describe("Maximum advertisers to return. Default 10, maximum 20. Prefer tighter search terms over maxing this out.")
|
|
21065
22141
|
};
|
|
21066
22142
|
FacebookAdTranscribeInputSchema = {
|
|
21067
|
-
videoUrl:
|
|
22143
|
+
videoUrl: import_zod27.z.string().url().describe("Direct Facebook CDN video URL from a facebook_page_intel ad result. Do not pass a public Facebook reel/post/share URL here; use facebook_video_transcribe for organic Facebook URLs.")
|
|
21068
22144
|
};
|
|
21069
22145
|
FacebookVideoTranscribeInputSchema = {
|
|
21070
|
-
url:
|
|
21071
|
-
quality:
|
|
22146
|
+
url: import_zod27.z.string().url().describe("Organic Facebook reel, video, watch, post, or share URL from facebook.com, m.facebook.com, or fb.watch. The tool renders the page, extracts the best matching public Facebook CDN MP4 URL, then transcribes it. Use this when the user pastes a normal Facebook video page URL and asks for the transcript or downloadable MP4."),
|
|
22147
|
+
quality: import_zod27.z.enum(["best", "hd", "sd"]).default("best").describe("Preferred progressive MP4 quality. Use best by default; hd prefers the highest HD progressive URL; sd forces the SD URL.")
|
|
22148
|
+
};
|
|
22149
|
+
InstagramProfileContentInputSchema = {
|
|
22150
|
+
handle: import_zod27.z.string().min(1).optional().describe("Instagram handle, with or without @. Provide handle or url. Use only public profiles unless a configured browser profile is available server-side."),
|
|
22151
|
+
url: import_zod27.z.string().url().optional().describe("Instagram profile URL, e.g. https://www.instagram.com/nasaartemis/. Provide handle or url."),
|
|
22152
|
+
maxItems: import_zod27.z.number().int().min(1).max(500).default(50).describe("Maximum profile grid post/reel/tv URLs to collect. Default 50, maximum 500. Use higher values with an authenticated browser profile when the user asks for a fuller archive."),
|
|
22153
|
+
maxScrolls: import_zod27.z.number().int().min(0).max(100).default(10).describe("Maximum pagination scroll attempts. Default 10, maximum 100. Increase when using authenticated Chromium and the profile has a long history."),
|
|
22154
|
+
scrollDelayMs: import_zod27.z.number().int().min(250).max(5e3).default(1200).describe("Delay after each pagination scroll before collecting newly loaded links. Default 1200ms. Increase to 2000-3000ms when Instagram loads slowly."),
|
|
22155
|
+
stableScrollLimit: import_zod27.z.number().int().min(1).max(10).default(4).describe("Stop after this many consecutive scrolls with no new links or scroll progress. Default 4."),
|
|
22156
|
+
browserMode: import_zod27.z.enum(["auto", "hosted", "local"]).default("auto").describe("Browser backend. auto uses the hosted browser unless local mode is configured or browserProfile is passed. local uses an imported/authenticated Chrome profile from browser_profile_import/sync. hosted forces the remote browser path."),
|
|
22157
|
+
browserProfile: import_zod27.z.string().min(1).optional().describe("Managed local browser profile name created by browser_profile_import/sync. Passing this opts the Instagram tool into local authenticated Chromium for this call.")
|
|
22158
|
+
};
|
|
22159
|
+
InstagramMediaDownloadInputSchema = {
|
|
22160
|
+
url: import_zod27.z.string().url().describe("Instagram post, reel, or tv URL, e.g. https://www.instagram.com/reel/SHORTCODE/. The tool renders the page, extracts text, image metadata, and Instagram CDN media tracks."),
|
|
22161
|
+
mediaTypes: import_zod27.z.array(import_zod27.z.enum(["image", "video", "audio"])).default(["image", "video", "audio"]).describe("Which media types to download when downloadMedia is true. Reels commonly expose separate video-only and audio-only MP4 tracks."),
|
|
22162
|
+
downloadMedia: import_zod27.z.boolean().default(true).describe("Download extracted text/media files to the MCP Scraper output directory when the API server can write files. Always returns extracted media URLs even when false."),
|
|
22163
|
+
downloadAllTracks: import_zod27.z.boolean().default(false).describe("Download every captured Instagram MP4 track instead of only the selected best video and audio tracks. Use false by default to avoid duplicate bitrates."),
|
|
22164
|
+
includeTranscript: import_zod27.z.boolean().default(false).describe("Transcribe the selected audio track when available. This adds transcription cost and may take longer."),
|
|
22165
|
+
mux: import_zod27.z.boolean().default(true).describe("When video and audio tracks are downloaded separately, try to mux them into a single MP4 if ffmpeg is available. Returns separate tracks when muxing is unavailable."),
|
|
22166
|
+
browserMode: import_zod27.z.enum(["auto", "hosted", "local"]).default("auto").describe("Browser backend. auto uses the hosted browser unless local mode is configured or browserProfile is passed. local uses an imported/authenticated Chrome profile from browser_profile_import/sync. hosted forces the remote browser path."),
|
|
22167
|
+
browserProfile: import_zod27.z.string().min(1).optional().describe("Managed local browser profile name created by browser_profile_import/sync. Passing this opts the Instagram tool into local authenticated Chromium for this call.")
|
|
21072
22168
|
};
|
|
21073
22169
|
MapsPlaceIntelInputSchema = {
|
|
21074
|
-
businessName:
|
|
21075
|
-
location:
|
|
21076
|
-
gl:
|
|
21077
|
-
hl:
|
|
21078
|
-
includeReviews:
|
|
21079
|
-
maxReviews:
|
|
22170
|
+
businessName: import_zod27.z.string().min(1).describe('Business name only. If user says "Elite Roofing Denver CO", use businessName="Elite Roofing" and location="Denver, CO".'),
|
|
22171
|
+
location: import_zod27.z.string().min(1).describe('City/region/country where the business should be searched, e.g. "Denver, CO". Infer from the user request when possible.'),
|
|
22172
|
+
gl: import_zod27.z.string().length(2).default("us").describe("Google country code inferred from location."),
|
|
22173
|
+
hl: import_zod27.z.string().length(2).default("en").describe("Language inferred from user request."),
|
|
22174
|
+
includeReviews: import_zod27.z.boolean().default(false).describe("Whether to fetch individual review cards. Use true when the user asks for reviews, customer pain, complaints, praise, themes, or review evidence."),
|
|
22175
|
+
maxReviews: import_zod27.z.number().int().min(1).max(500).default(50).describe("Max review cards to return when includeReviews is true. Default 50, maximum 500. Use 50 for normal review analysis and larger values only for deep review mining.")
|
|
21080
22176
|
};
|
|
21081
22177
|
MapsSearchInputSchema = {
|
|
21082
|
-
query:
|
|
21083
|
-
location:
|
|
21084
|
-
gl:
|
|
21085
|
-
hl:
|
|
21086
|
-
maxResults:
|
|
21087
|
-
proxyMode:
|
|
21088
|
-
proxyZip:
|
|
21089
|
-
debug:
|
|
22178
|
+
query: import_zod27.z.string().min(1).describe('Business category, niche, keyword, or search term. If the user says "roofers in Denver CO", use query="roofers" and location="Denver, CO". Do not put the location here when it can be separated.'),
|
|
22179
|
+
location: import_zod27.z.string().optional().describe('City, region, country, or service area for the Maps search, e.g. "Denver, CO". Infer from the user request when present.'),
|
|
22180
|
+
gl: import_zod27.z.string().length(2).default("us").describe("Google country code inferred from location."),
|
|
22181
|
+
hl: import_zod27.z.string().length(2).default("en").describe("Language inferred from user request."),
|
|
22182
|
+
maxResults: import_zod27.z.number().int().min(1).max(50).default(10).describe("Number of Google Maps business/profile candidates to return. Default 10. Maximum 50. Use 10 unless the user asks for more."),
|
|
22183
|
+
proxyMode: import_zod27.z.enum(["location", "configured", "none"]).default(DEFAULT_MAPS_PROXY_MODE).describe("Proxy targeting mode. Maps defaults to location so local market searches get city/state residential proxy targeting and rotation. Use configured to force the service proxy without city/ZIP targeting, and none only for local direct-network debugging."),
|
|
22184
|
+
proxyZip: import_zod27.z.string().regex(/^\d{5}$/).optional().describe("Optional US ZIP override for residential location proxy targeting. Use when the user gives a specific ZIP or city-center ZIP."),
|
|
22185
|
+
debug: import_zod27.z.boolean().default(false).describe("Include sanitized browser/proxy diagnostics when debugging Maps localization, CAPTCHA, or proxy behavior.")
|
|
21090
22186
|
};
|
|
21091
22187
|
DirectoryWorkflowInputSchema = {
|
|
21092
|
-
query:
|
|
21093
|
-
state:
|
|
21094
|
-
minPopulation:
|
|
21095
|
-
populationYear:
|
|
21096
|
-
maxCities:
|
|
21097
|
-
maxResultsPerCity:
|
|
21098
|
-
concurrency:
|
|
21099
|
-
includeZipGroups:
|
|
21100
|
-
usZipsCsvPath:
|
|
21101
|
-
saveCsv:
|
|
21102
|
-
proxyMode:
|
|
21103
|
-
proxyZip:
|
|
21104
|
-
debug:
|
|
22188
|
+
query: import_zod27.z.string().min(1).describe("Business category, niche, or keyword to search on Google Maps for every selected market, e.g. roofers, dentists, med spas. Do not include the city here."),
|
|
22189
|
+
state: import_zod27.z.string().min(2).default("TN").describe("US state abbreviation or state name used to select Census places, e.g. TN or Tennessee."),
|
|
22190
|
+
minPopulation: import_zod27.z.number().int().min(0).default(1e5).describe('Minimum Census place population for market selection. Use 100000 for "cities above 100k population".'),
|
|
22191
|
+
populationYear: import_zod27.z.number().int().min(2020).max(2025).default(2025).describe("Census population estimate year from the 2020-2025 Population Estimates Program city/place dataset."),
|
|
22192
|
+
maxCities: import_zod27.z.number().int().min(1).max(100).default(25).describe("Maximum number of markets to process after sorting by population descending."),
|
|
22193
|
+
maxResultsPerCity: import_zod27.z.number().int().min(1).max(50).default(50).describe("Google Maps business/profile candidates to collect for each city. Maximum 50."),
|
|
22194
|
+
concurrency: import_zod27.z.number().int().min(1).max(5).default(5).describe("How many city Maps searches to run in parallel. Use 5 for broad directory batches unless debugging."),
|
|
22195
|
+
includeZipGroups: import_zod27.z.boolean().default(true).describe("Attach ZIP groups from a configured US ZIPS CSV when available. Set MCP_SCRAPER_USZIPS_CSV_PATH on the API server or pass usZipsCsvPath in local/test mode."),
|
|
22196
|
+
usZipsCsvPath: import_zod27.z.string().optional().describe("Local/test-only path to a US ZIPS CSV with state_abbr, zipcode, county, city columns, such as Lead Magician tools/analytics/data/uszips.csv. Deployed APIs should use MCP_SCRAPER_USZIPS_CSV_PATH instead."),
|
|
22197
|
+
saveCsv: import_zod27.z.boolean().default(true).describe("Save a directory-ready CSV to the MCP Scraper output directory and return its path. CSV rows include source_location, result_position, business_name, review_stars, review_count, category, address, phone, hours_status, website_url, directions_url, place_url, CID fields, population, and ZIP groups."),
|
|
22198
|
+
proxyMode: import_zod27.z.enum(["location", "configured", "none"]).default(DEFAULT_MAPS_PROXY_MODE).describe("Proxy targeting mode for every city Maps search. Maps workflows default to location so each city can use city/state or ZIP-group residential proxy targeting. Use configured to force the service proxy without city/ZIP targeting, and none only for local direct-network debugging."),
|
|
22199
|
+
proxyZip: import_zod27.z.string().regex(/^\d{5}$/).optional().describe("Optional ZIP override for proxy targeting. Normally omit it so each city can use its ZIP group or city/state location."),
|
|
22200
|
+
debug: import_zod27.z.boolean().default(false).describe("Include sanitized browser/proxy diagnostics in each Maps browser session when supported.")
|
|
21105
22201
|
};
|
|
21106
|
-
RankTrackerModeSchema =
|
|
22202
|
+
RankTrackerModeSchema = import_zod27.z.enum(["maps", "organic", "ai_overview", "paa"]);
|
|
21107
22203
|
RankTrackerBlueprintInputSchema = {
|
|
21108
|
-
projectName:
|
|
21109
|
-
targetDomain:
|
|
21110
|
-
targetBusinessName:
|
|
21111
|
-
trackingModes:
|
|
21112
|
-
keywords:
|
|
21113
|
-
locations:
|
|
21114
|
-
competitors:
|
|
21115
|
-
database:
|
|
21116
|
-
scheduleCadence:
|
|
21117
|
-
customCron:
|
|
21118
|
-
timezone:
|
|
21119
|
-
includeCron:
|
|
21120
|
-
includeDashboard:
|
|
21121
|
-
includeAlerts:
|
|
21122
|
-
notes:
|
|
22204
|
+
projectName: import_zod27.z.string().min(1).optional().describe("Optional name for the rank tracker project, client, or campaign."),
|
|
22205
|
+
targetDomain: import_zod27.z.string().min(1).optional().describe("Primary domain to track in organic results, AI Overview citations, and PAA sources, e.g. example.com."),
|
|
22206
|
+
targetBusinessName: import_zod27.z.string().min(1).optional().describe("Primary Google Business Profile or brand/business name to match in Maps results. Required for reliable Maps rank tracking."),
|
|
22207
|
+
trackingModes: import_zod27.z.array(RankTrackerModeSchema).min(1).max(4).default(["maps", "organic", "ai_overview", "paa"]).describe("Rank tracker surfaces to build. maps uses directory_workflow/maps_search. organic uses search_serp. ai_overview uses search_serp/harvest_paa. paa uses harvest_paa source presence."),
|
|
22208
|
+
keywords: import_zod27.z.array(import_zod27.z.string().min(1)).max(200).default([]).describe("Seed keywords or service queries to track. Leave empty when the downstream AI should create the keyword table from user input."),
|
|
22209
|
+
locations: import_zod27.z.array(import_zod27.z.string().min(1)).max(100).default([]).describe("Markets, cities, ZIPs, or service areas to track. Use city/state strings like Denver, CO for localized SERP and Maps checks."),
|
|
22210
|
+
competitors: import_zod27.z.array(import_zod27.z.string().min(1)).max(100).default([]).describe("Optional competitor domains or business names to persist as comparison targets."),
|
|
22211
|
+
database: import_zod27.z.enum(["postgres", "neon", "supabase", "sqlite", "mysql"]).default("postgres").describe("Database family the downstream AI should target when generating migrations."),
|
|
22212
|
+
scheduleCadence: import_zod27.z.enum(["daily", "weekly", "monthly", "custom"]).default("weekly").describe("Default recurring rank check cadence for the generated cron/heartbeat plan."),
|
|
22213
|
+
customCron: import_zod27.z.string().min(1).optional().describe("Cron expression to use when scheduleCadence is custom."),
|
|
22214
|
+
timezone: import_zod27.z.string().min(1).default("UTC").describe("IANA timezone for scheduled rank checks, e.g. America/Denver."),
|
|
22215
|
+
includeCron: import_zod27.z.boolean().default(true).describe("Include a cron or heartbeat worker plan. Keep true for production rank trackers."),
|
|
22216
|
+
includeDashboard: import_zod27.z.boolean().default(true).describe("Include dashboard/reporting requirements in the generated prompt."),
|
|
22217
|
+
includeAlerts: import_zod27.z.boolean().default(true).describe("Include alert rules for rank movement and SERP feature gains/losses."),
|
|
22218
|
+
notes: import_zod27.z.string().max(4e3).optional().describe("Extra product, client, stack, or hosting requirements to include in the implementation prompt.")
|
|
21123
22219
|
};
|
|
21124
|
-
NullableString =
|
|
21125
|
-
MapsSearchAttemptOutput =
|
|
21126
|
-
attemptNumber:
|
|
21127
|
-
maxAttempts:
|
|
21128
|
-
status:
|
|
21129
|
-
outcome:
|
|
21130
|
-
willRetry:
|
|
21131
|
-
durationMs:
|
|
21132
|
-
resultCount:
|
|
22220
|
+
NullableString = import_zod27.z.string().nullable();
|
|
22221
|
+
MapsSearchAttemptOutput = import_zod27.z.object({
|
|
22222
|
+
attemptNumber: import_zod27.z.number().int().min(1),
|
|
22223
|
+
maxAttempts: import_zod27.z.number().int().min(1),
|
|
22224
|
+
status: import_zod27.z.enum(["ok", "failed"]),
|
|
22225
|
+
outcome: import_zod27.z.string(),
|
|
22226
|
+
willRetry: import_zod27.z.boolean(),
|
|
22227
|
+
durationMs: import_zod27.z.number().int().min(0),
|
|
22228
|
+
resultCount: import_zod27.z.number().int().min(0),
|
|
21133
22229
|
error: NullableString,
|
|
21134
|
-
proxyMode:
|
|
21135
|
-
proxyResolutionSource:
|
|
22230
|
+
proxyMode: import_zod27.z.enum(["location", "configured", "none"]),
|
|
22231
|
+
proxyResolutionSource: import_zod27.z.enum(["disabled", "location_reused", "location_created", "configured_fallback", "unavailable"]).nullable(),
|
|
21136
22232
|
proxyIdSuffix: NullableString,
|
|
21137
|
-
proxyTargetLevel:
|
|
22233
|
+
proxyTargetLevel: import_zod27.z.enum(["zip", "city", "state"]).nullable(),
|
|
21138
22234
|
proxyTargetLocation: NullableString,
|
|
21139
22235
|
proxyTargetZip: NullableString,
|
|
21140
22236
|
browserSessionIdSuffix: NullableString,
|
|
@@ -21143,17 +22239,17 @@ var init_mcp_tool_schemas = __esm({
|
|
|
21143
22239
|
observedRegion: NullableString
|
|
21144
22240
|
});
|
|
21145
22241
|
MapsSearchOutputSchema = {
|
|
21146
|
-
query:
|
|
21147
|
-
location:
|
|
21148
|
-
searchQuery:
|
|
21149
|
-
searchUrl:
|
|
21150
|
-
extractedAt:
|
|
21151
|
-
requestedMaxResults:
|
|
21152
|
-
resultCount:
|
|
21153
|
-
results:
|
|
21154
|
-
position:
|
|
21155
|
-
name:
|
|
21156
|
-
placeUrl:
|
|
22242
|
+
query: import_zod27.z.string(),
|
|
22243
|
+
location: import_zod27.z.string().nullable(),
|
|
22244
|
+
searchQuery: import_zod27.z.string(),
|
|
22245
|
+
searchUrl: import_zod27.z.string().url(),
|
|
22246
|
+
extractedAt: import_zod27.z.string(),
|
|
22247
|
+
requestedMaxResults: import_zod27.z.number().int().min(1).max(50),
|
|
22248
|
+
resultCount: import_zod27.z.number().int().min(0).max(50),
|
|
22249
|
+
results: import_zod27.z.array(import_zod27.z.object({
|
|
22250
|
+
position: import_zod27.z.number().int().min(1),
|
|
22251
|
+
name: import_zod27.z.string(),
|
|
22252
|
+
placeUrl: import_zod27.z.string().url(),
|
|
21157
22253
|
cid: NullableString,
|
|
21158
22254
|
cidDecimal: NullableString,
|
|
21159
22255
|
rating: NullableString,
|
|
@@ -21164,15 +22260,15 @@ var init_mcp_tool_schemas = __esm({
|
|
|
21164
22260
|
hoursStatus: NullableString,
|
|
21165
22261
|
websiteUrl: NullableString,
|
|
21166
22262
|
directionsUrl: NullableString,
|
|
21167
|
-
metadata:
|
|
22263
|
+
metadata: import_zod27.z.array(import_zod27.z.string())
|
|
21168
22264
|
})),
|
|
21169
|
-
attempts:
|
|
21170
|
-
durationMs:
|
|
22265
|
+
attempts: import_zod27.z.array(MapsSearchAttemptOutput),
|
|
22266
|
+
durationMs: import_zod27.z.number().int().min(0)
|
|
21171
22267
|
};
|
|
21172
|
-
DirectoryMapsBusinessOutput =
|
|
21173
|
-
position:
|
|
21174
|
-
name:
|
|
21175
|
-
placeUrl:
|
|
22268
|
+
DirectoryMapsBusinessOutput = import_zod27.z.object({
|
|
22269
|
+
position: import_zod27.z.number().int().min(1),
|
|
22270
|
+
name: import_zod27.z.string(),
|
|
22271
|
+
placeUrl: import_zod27.z.string().url(),
|
|
21176
22272
|
cid: NullableString,
|
|
21177
22273
|
cidDecimal: NullableString,
|
|
21178
22274
|
rating: NullableString,
|
|
@@ -21183,113 +22279,113 @@ var init_mcp_tool_schemas = __esm({
|
|
|
21183
22279
|
hoursStatus: NullableString,
|
|
21184
22280
|
websiteUrl: NullableString,
|
|
21185
22281
|
directionsUrl: NullableString,
|
|
21186
|
-
metadata:
|
|
22282
|
+
metadata: import_zod27.z.array(import_zod27.z.string())
|
|
21187
22283
|
});
|
|
21188
22284
|
DirectoryWorkflowOutputSchema = {
|
|
21189
|
-
query:
|
|
21190
|
-
state:
|
|
21191
|
-
minPopulation:
|
|
21192
|
-
populationYear:
|
|
21193
|
-
maxResultsPerCity:
|
|
21194
|
-
concurrency:
|
|
21195
|
-
censusSourceUrl:
|
|
22285
|
+
query: import_zod27.z.string(),
|
|
22286
|
+
state: import_zod27.z.string(),
|
|
22287
|
+
minPopulation: import_zod27.z.number().int().min(0),
|
|
22288
|
+
populationYear: import_zod27.z.number().int().min(2020).max(2025),
|
|
22289
|
+
maxResultsPerCity: import_zod27.z.number().int().min(1).max(50),
|
|
22290
|
+
concurrency: import_zod27.z.number().int().min(1).max(5),
|
|
22291
|
+
censusSourceUrl: import_zod27.z.string().url(),
|
|
21196
22292
|
usZipsSourcePath: NullableString,
|
|
21197
|
-
warnings:
|
|
21198
|
-
extractedAt:
|
|
21199
|
-
selectedCityCount:
|
|
21200
|
-
totalResultCount:
|
|
22293
|
+
warnings: import_zod27.z.array(import_zod27.z.string()),
|
|
22294
|
+
extractedAt: import_zod27.z.string(),
|
|
22295
|
+
selectedCityCount: import_zod27.z.number().int().min(0),
|
|
22296
|
+
totalResultCount: import_zod27.z.number().int().min(0),
|
|
21201
22297
|
csvPath: NullableString,
|
|
21202
|
-
cities:
|
|
21203
|
-
city:
|
|
21204
|
-
state:
|
|
21205
|
-
location:
|
|
21206
|
-
cityKey:
|
|
21207
|
-
censusName:
|
|
21208
|
-
population:
|
|
21209
|
-
populationYear:
|
|
21210
|
-
zips:
|
|
21211
|
-
counties:
|
|
21212
|
-
status:
|
|
22298
|
+
cities: import_zod27.z.array(import_zod27.z.object({
|
|
22299
|
+
city: import_zod27.z.string(),
|
|
22300
|
+
state: import_zod27.z.string(),
|
|
22301
|
+
location: import_zod27.z.string(),
|
|
22302
|
+
cityKey: import_zod27.z.string(),
|
|
22303
|
+
censusName: import_zod27.z.string(),
|
|
22304
|
+
population: import_zod27.z.number().int().min(0),
|
|
22305
|
+
populationYear: import_zod27.z.number().int().min(2020).max(2025),
|
|
22306
|
+
zips: import_zod27.z.array(import_zod27.z.string()),
|
|
22307
|
+
counties: import_zod27.z.array(import_zod27.z.string()),
|
|
22308
|
+
status: import_zod27.z.enum(["ok", "empty", "failed"]),
|
|
21213
22309
|
error: NullableString,
|
|
21214
|
-
resultCount:
|
|
21215
|
-
durationMs:
|
|
21216
|
-
attempts:
|
|
21217
|
-
results:
|
|
22310
|
+
resultCount: import_zod27.z.number().int().min(0),
|
|
22311
|
+
durationMs: import_zod27.z.number().int().min(0),
|
|
22312
|
+
attempts: import_zod27.z.array(MapsSearchAttemptOutput),
|
|
22313
|
+
results: import_zod27.z.array(DirectoryMapsBusinessOutput)
|
|
21218
22314
|
})),
|
|
21219
|
-
durationMs:
|
|
22315
|
+
durationMs: import_zod27.z.number().int().min(0)
|
|
21220
22316
|
};
|
|
21221
|
-
RankTrackerToolPlanOutput =
|
|
21222
|
-
tool:
|
|
21223
|
-
purpose:
|
|
22317
|
+
RankTrackerToolPlanOutput = import_zod27.z.object({
|
|
22318
|
+
tool: import_zod27.z.string(),
|
|
22319
|
+
purpose: import_zod27.z.string()
|
|
21224
22320
|
});
|
|
21225
|
-
RankTrackerTableOutput =
|
|
21226
|
-
name:
|
|
21227
|
-
purpose:
|
|
21228
|
-
keyColumns:
|
|
22321
|
+
RankTrackerTableOutput = import_zod27.z.object({
|
|
22322
|
+
name: import_zod27.z.string(),
|
|
22323
|
+
purpose: import_zod27.z.string(),
|
|
22324
|
+
keyColumns: import_zod27.z.array(import_zod27.z.string())
|
|
21229
22325
|
});
|
|
21230
|
-
RankTrackerCronJobOutput =
|
|
21231
|
-
name:
|
|
21232
|
-
purpose:
|
|
21233
|
-
modes:
|
|
21234
|
-
recommendedTools:
|
|
22326
|
+
RankTrackerCronJobOutput = import_zod27.z.object({
|
|
22327
|
+
name: import_zod27.z.string(),
|
|
22328
|
+
purpose: import_zod27.z.string(),
|
|
22329
|
+
modes: import_zod27.z.array(RankTrackerModeSchema),
|
|
22330
|
+
recommendedTools: import_zod27.z.array(import_zod27.z.string())
|
|
21235
22331
|
});
|
|
21236
22332
|
RankTrackerBlueprintOutputSchema = {
|
|
21237
|
-
projectName:
|
|
22333
|
+
projectName: import_zod27.z.string(),
|
|
21238
22334
|
targetDomain: NullableString,
|
|
21239
22335
|
targetBusinessName: NullableString,
|
|
21240
|
-
trackingModes:
|
|
21241
|
-
database:
|
|
21242
|
-
recommendedTools:
|
|
21243
|
-
tables:
|
|
21244
|
-
cron:
|
|
21245
|
-
enabled:
|
|
21246
|
-
cadence:
|
|
21247
|
-
expression:
|
|
21248
|
-
timezone:
|
|
21249
|
-
jobs:
|
|
22336
|
+
trackingModes: import_zod27.z.array(RankTrackerModeSchema),
|
|
22337
|
+
database: import_zod27.z.string(),
|
|
22338
|
+
recommendedTools: import_zod27.z.array(RankTrackerToolPlanOutput),
|
|
22339
|
+
tables: import_zod27.z.array(RankTrackerTableOutput),
|
|
22340
|
+
cron: import_zod27.z.object({
|
|
22341
|
+
enabled: import_zod27.z.boolean(),
|
|
22342
|
+
cadence: import_zod27.z.string(),
|
|
22343
|
+
expression: import_zod27.z.string(),
|
|
22344
|
+
timezone: import_zod27.z.string(),
|
|
22345
|
+
jobs: import_zod27.z.array(RankTrackerCronJobOutput)
|
|
21250
22346
|
}),
|
|
21251
|
-
metrics:
|
|
21252
|
-
implementationPrompt:
|
|
22347
|
+
metrics: import_zod27.z.array(import_zod27.z.string()),
|
|
22348
|
+
implementationPrompt: import_zod27.z.string()
|
|
21253
22349
|
};
|
|
21254
|
-
OrganicResultOutput =
|
|
21255
|
-
position:
|
|
21256
|
-
title:
|
|
21257
|
-
url:
|
|
21258
|
-
domain:
|
|
22350
|
+
OrganicResultOutput = import_zod27.z.object({
|
|
22351
|
+
position: import_zod27.z.number().int(),
|
|
22352
|
+
title: import_zod27.z.string(),
|
|
22353
|
+
url: import_zod27.z.string(),
|
|
22354
|
+
domain: import_zod27.z.string(),
|
|
21259
22355
|
snippet: NullableString
|
|
21260
22356
|
});
|
|
21261
|
-
AiOverviewOutput =
|
|
21262
|
-
detected:
|
|
22357
|
+
AiOverviewOutput = import_zod27.z.object({
|
|
22358
|
+
detected: import_zod27.z.boolean(),
|
|
21263
22359
|
text: NullableString
|
|
21264
22360
|
}).nullable();
|
|
21265
|
-
EntityIdsOutput =
|
|
21266
|
-
kgIds:
|
|
21267
|
-
cids:
|
|
21268
|
-
gcids:
|
|
22361
|
+
EntityIdsOutput = import_zod27.z.object({
|
|
22362
|
+
kgIds: import_zod27.z.array(import_zod27.z.string()),
|
|
22363
|
+
cids: import_zod27.z.array(import_zod27.z.string()),
|
|
22364
|
+
gcids: import_zod27.z.array(import_zod27.z.string())
|
|
21269
22365
|
}).nullable();
|
|
21270
22366
|
HarvestPaaOutputSchema = {
|
|
21271
|
-
query:
|
|
22367
|
+
query: import_zod27.z.string(),
|
|
21272
22368
|
location: NullableString,
|
|
21273
|
-
questionCount:
|
|
22369
|
+
questionCount: import_zod27.z.number().int().min(0),
|
|
21274
22370
|
completionStatus: NullableString,
|
|
21275
|
-
questions:
|
|
21276
|
-
question:
|
|
22371
|
+
questions: import_zod27.z.array(import_zod27.z.object({
|
|
22372
|
+
question: import_zod27.z.string(),
|
|
21277
22373
|
answer: NullableString,
|
|
21278
22374
|
sourceTitle: NullableString,
|
|
21279
22375
|
sourceSite: NullableString
|
|
21280
22376
|
})),
|
|
21281
|
-
organicResults:
|
|
22377
|
+
organicResults: import_zod27.z.array(OrganicResultOutput),
|
|
21282
22378
|
aiOverview: AiOverviewOutput,
|
|
21283
22379
|
entityIds: EntityIdsOutput,
|
|
21284
|
-
durationMs:
|
|
22380
|
+
durationMs: import_zod27.z.number().min(0).nullable()
|
|
21285
22381
|
};
|
|
21286
22382
|
SearchSerpOutputSchema = {
|
|
21287
|
-
query:
|
|
22383
|
+
query: import_zod27.z.string(),
|
|
21288
22384
|
location: NullableString,
|
|
21289
|
-
organicResults:
|
|
21290
|
-
localPack:
|
|
21291
|
-
position:
|
|
21292
|
-
name:
|
|
22385
|
+
organicResults: import_zod27.z.array(OrganicResultOutput),
|
|
22386
|
+
localPack: import_zod27.z.array(import_zod27.z.object({
|
|
22387
|
+
position: import_zod27.z.number().int(),
|
|
22388
|
+
name: import_zod27.z.string(),
|
|
21293
22389
|
rating: NullableString,
|
|
21294
22390
|
reviewCount: NullableString,
|
|
21295
22391
|
websiteUrl: NullableString
|
|
@@ -21298,31 +22394,31 @@ var init_mcp_tool_schemas = __esm({
|
|
|
21298
22394
|
entityIds: EntityIdsOutput
|
|
21299
22395
|
};
|
|
21300
22396
|
ExtractUrlOutputSchema = {
|
|
21301
|
-
url:
|
|
22397
|
+
url: import_zod27.z.string(),
|
|
21302
22398
|
title: NullableString,
|
|
21303
|
-
headings:
|
|
21304
|
-
level:
|
|
21305
|
-
text:
|
|
22399
|
+
headings: import_zod27.z.array(import_zod27.z.object({
|
|
22400
|
+
level: import_zod27.z.number().int(),
|
|
22401
|
+
text: import_zod27.z.string()
|
|
21306
22402
|
})),
|
|
21307
|
-
schemaBlockCount:
|
|
22403
|
+
schemaBlockCount: import_zod27.z.number().int().min(0),
|
|
21308
22404
|
entityName: NullableString,
|
|
21309
|
-
entityTypes:
|
|
21310
|
-
napScore:
|
|
21311
|
-
missingSchemaFields:
|
|
22405
|
+
entityTypes: import_zod27.z.array(import_zod27.z.string()),
|
|
22406
|
+
napScore: import_zod27.z.number().nullable(),
|
|
22407
|
+
missingSchemaFields: import_zod27.z.array(import_zod27.z.string()),
|
|
21312
22408
|
screenshotSaved: NullableString
|
|
21313
22409
|
};
|
|
21314
22410
|
ExtractSiteOutputSchema = {
|
|
21315
|
-
url:
|
|
21316
|
-
pageCount:
|
|
21317
|
-
pages:
|
|
21318
|
-
url:
|
|
22411
|
+
url: import_zod27.z.string(),
|
|
22412
|
+
pageCount: import_zod27.z.number().int().min(0),
|
|
22413
|
+
pages: import_zod27.z.array(import_zod27.z.object({
|
|
22414
|
+
url: import_zod27.z.string(),
|
|
21319
22415
|
title: NullableString,
|
|
21320
|
-
schemaTypes:
|
|
22416
|
+
schemaTypes: import_zod27.z.array(import_zod27.z.string())
|
|
21321
22417
|
})),
|
|
21322
|
-
durationMs:
|
|
22418
|
+
durationMs: import_zod27.z.number().min(0)
|
|
21323
22419
|
};
|
|
21324
22420
|
MapsPlaceIntelOutputSchema = {
|
|
21325
|
-
name:
|
|
22421
|
+
name: import_zod27.z.string(),
|
|
21326
22422
|
rating: NullableString,
|
|
21327
22423
|
reviewCount: NullableString,
|
|
21328
22424
|
category: NullableString,
|
|
@@ -21334,75 +22430,75 @@ var init_mcp_tool_schemas = __esm({
|
|
|
21334
22430
|
kgmid: NullableString,
|
|
21335
22431
|
cidDecimal: NullableString,
|
|
21336
22432
|
cidUrl: NullableString,
|
|
21337
|
-
lat:
|
|
21338
|
-
lng:
|
|
21339
|
-
reviewsStatus:
|
|
21340
|
-
reviewsCollected:
|
|
21341
|
-
reviewTopics:
|
|
21342
|
-
label:
|
|
21343
|
-
count:
|
|
22433
|
+
lat: import_zod27.z.number().nullable(),
|
|
22434
|
+
lng: import_zod27.z.number().nullable(),
|
|
22435
|
+
reviewsStatus: import_zod27.z.string(),
|
|
22436
|
+
reviewsCollected: import_zod27.z.number().int().min(0),
|
|
22437
|
+
reviewTopics: import_zod27.z.array(import_zod27.z.object({
|
|
22438
|
+
label: import_zod27.z.string(),
|
|
22439
|
+
count: import_zod27.z.string()
|
|
21344
22440
|
}))
|
|
21345
22441
|
};
|
|
21346
22442
|
CreditsInfoOutputSchema = {
|
|
21347
|
-
balanceCredits:
|
|
21348
|
-
matchedCost:
|
|
21349
|
-
label:
|
|
21350
|
-
credits:
|
|
21351
|
-
unit:
|
|
22443
|
+
balanceCredits: import_zod27.z.number().nullable(),
|
|
22444
|
+
matchedCost: import_zod27.z.object({
|
|
22445
|
+
label: import_zod27.z.string(),
|
|
22446
|
+
credits: import_zod27.z.number(),
|
|
22447
|
+
unit: import_zod27.z.string(),
|
|
21352
22448
|
notes: NullableString
|
|
21353
22449
|
}).nullable(),
|
|
21354
|
-
costs:
|
|
21355
|
-
key:
|
|
21356
|
-
label:
|
|
21357
|
-
credits:
|
|
21358
|
-
unit:
|
|
22450
|
+
costs: import_zod27.z.array(import_zod27.z.object({
|
|
22451
|
+
key: import_zod27.z.string(),
|
|
22452
|
+
label: import_zod27.z.string(),
|
|
22453
|
+
credits: import_zod27.z.number(),
|
|
22454
|
+
unit: import_zod27.z.string(),
|
|
21359
22455
|
notes: NullableString
|
|
21360
22456
|
})),
|
|
21361
|
-
ledger:
|
|
21362
|
-
createdAt:
|
|
21363
|
-
operation:
|
|
21364
|
-
credits:
|
|
22457
|
+
ledger: import_zod27.z.array(import_zod27.z.object({
|
|
22458
|
+
createdAt: import_zod27.z.string(),
|
|
22459
|
+
operation: import_zod27.z.string(),
|
|
22460
|
+
credits: import_zod27.z.number(),
|
|
21365
22461
|
description: NullableString
|
|
21366
22462
|
})),
|
|
21367
|
-
concurrency:
|
|
21368
|
-
currentExtraSlots:
|
|
21369
|
-
currentLimit:
|
|
21370
|
-
hasSubscription:
|
|
21371
|
-
upgrade:
|
|
21372
|
-
product:
|
|
21373
|
-
priceLabel:
|
|
21374
|
-
unitAmountUsd:
|
|
21375
|
-
currency:
|
|
21376
|
-
interval:
|
|
21377
|
-
billingUrl:
|
|
21378
|
-
terminalCommand:
|
|
21379
|
-
terminalCommandWithApiKeyEnv:
|
|
22463
|
+
concurrency: import_zod27.z.object({
|
|
22464
|
+
currentExtraSlots: import_zod27.z.number().int().min(0),
|
|
22465
|
+
currentLimit: import_zod27.z.number().int().min(1),
|
|
22466
|
+
hasSubscription: import_zod27.z.boolean(),
|
|
22467
|
+
upgrade: import_zod27.z.object({
|
|
22468
|
+
product: import_zod27.z.string(),
|
|
22469
|
+
priceLabel: import_zod27.z.string(),
|
|
22470
|
+
unitAmountUsd: import_zod27.z.number(),
|
|
22471
|
+
currency: import_zod27.z.string(),
|
|
22472
|
+
interval: import_zod27.z.string(),
|
|
22473
|
+
billingUrl: import_zod27.z.string().url(),
|
|
22474
|
+
terminalCommand: import_zod27.z.string(),
|
|
22475
|
+
terminalCommandWithApiKeyEnv: import_zod27.z.string()
|
|
21380
22476
|
})
|
|
21381
22477
|
}).nullable()
|
|
21382
22478
|
};
|
|
21383
22479
|
MapSiteUrlsOutputSchema = {
|
|
21384
|
-
startUrl:
|
|
21385
|
-
totalFound:
|
|
21386
|
-
truncated:
|
|
21387
|
-
okCount:
|
|
21388
|
-
redirectCount:
|
|
21389
|
-
brokenCount:
|
|
21390
|
-
urls:
|
|
21391
|
-
url:
|
|
21392
|
-
status:
|
|
22480
|
+
startUrl: import_zod27.z.string(),
|
|
22481
|
+
totalFound: import_zod27.z.number().int().min(0),
|
|
22482
|
+
truncated: import_zod27.z.boolean(),
|
|
22483
|
+
okCount: import_zod27.z.number().int().min(0),
|
|
22484
|
+
redirectCount: import_zod27.z.number().int().min(0),
|
|
22485
|
+
brokenCount: import_zod27.z.number().int().min(0),
|
|
22486
|
+
urls: import_zod27.z.array(import_zod27.z.object({
|
|
22487
|
+
url: import_zod27.z.string(),
|
|
22488
|
+
status: import_zod27.z.number().int().nullable()
|
|
21393
22489
|
})),
|
|
21394
|
-
durationMs:
|
|
22490
|
+
durationMs: import_zod27.z.number().min(0)
|
|
21395
22491
|
};
|
|
21396
22492
|
YoutubeHarvestOutputSchema = {
|
|
21397
|
-
mode:
|
|
21398
|
-
videoCount:
|
|
21399
|
-
channel:
|
|
22493
|
+
mode: import_zod27.z.string(),
|
|
22494
|
+
videoCount: import_zod27.z.number().int().min(0),
|
|
22495
|
+
channel: import_zod27.z.object({
|
|
21400
22496
|
title: NullableString,
|
|
21401
22497
|
subscriberCount: NullableString
|
|
21402
22498
|
}).nullable(),
|
|
21403
|
-
videos:
|
|
21404
|
-
videoId:
|
|
21405
|
-
title:
|
|
22499
|
+
videos: import_zod27.z.array(import_zod27.z.object({
|
|
22500
|
+
videoId: import_zod27.z.string(),
|
|
22501
|
+
title: import_zod27.z.string(),
|
|
21406
22502
|
channelName: NullableString,
|
|
21407
22503
|
views: NullableString,
|
|
21408
22504
|
duration: NullableString,
|
|
@@ -21410,24 +22506,24 @@ var init_mcp_tool_schemas = __esm({
|
|
|
21410
22506
|
}))
|
|
21411
22507
|
};
|
|
21412
22508
|
FacebookAdSearchOutputSchema = {
|
|
21413
|
-
query:
|
|
21414
|
-
advertiserCount:
|
|
21415
|
-
advertisers:
|
|
22509
|
+
query: import_zod27.z.string(),
|
|
22510
|
+
advertiserCount: import_zod27.z.number().int().min(0),
|
|
22511
|
+
advertisers: import_zod27.z.array(import_zod27.z.object({
|
|
21416
22512
|
name: NullableString,
|
|
21417
22513
|
pageId: NullableString,
|
|
21418
22514
|
pageUrl: NullableString,
|
|
21419
|
-
adCount:
|
|
22515
|
+
adCount: import_zod27.z.number().int().nullable(),
|
|
21420
22516
|
libraryId: NullableString,
|
|
21421
22517
|
sampleLibraryId: NullableString
|
|
21422
22518
|
}))
|
|
21423
22519
|
};
|
|
21424
22520
|
FacebookPageIntelOutputSchema = {
|
|
21425
22521
|
advertiserName: NullableString,
|
|
21426
|
-
totalAds:
|
|
21427
|
-
activeCount:
|
|
21428
|
-
videoCount:
|
|
21429
|
-
imageCount:
|
|
21430
|
-
ads:
|
|
22522
|
+
totalAds: import_zod27.z.number().int().min(0),
|
|
22523
|
+
activeCount: import_zod27.z.number().int().min(0),
|
|
22524
|
+
videoCount: import_zod27.z.number().int().min(0),
|
|
22525
|
+
imageCount: import_zod27.z.number().int().min(0),
|
|
22526
|
+
ads: import_zod27.z.array(import_zod27.z.object({
|
|
21431
22527
|
libraryId: NullableString,
|
|
21432
22528
|
status: NullableString,
|
|
21433
22529
|
creativeType: NullableString,
|
|
@@ -21440,84 +22536,179 @@ var init_mcp_tool_schemas = __esm({
|
|
|
21440
22536
|
videoUrl: NullableString,
|
|
21441
22537
|
imageUrl: NullableString,
|
|
21442
22538
|
videoPoster: NullableString,
|
|
21443
|
-
variations:
|
|
22539
|
+
variations: import_zod27.z.number().int().nullable()
|
|
21444
22540
|
}))
|
|
21445
22541
|
};
|
|
21446
22542
|
FacebookVideoTranscribeOutputSchema = {
|
|
21447
|
-
sourceUrl:
|
|
21448
|
-
pageUrl:
|
|
22543
|
+
sourceUrl: import_zod27.z.string().url(),
|
|
22544
|
+
pageUrl: import_zod27.z.string().url(),
|
|
21449
22545
|
videoId: NullableString,
|
|
21450
22546
|
ownerName: NullableString,
|
|
21451
|
-
selectedQuality:
|
|
21452
|
-
bitrate:
|
|
21453
|
-
videoDurationSec:
|
|
21454
|
-
videoUrl:
|
|
21455
|
-
wordCount:
|
|
21456
|
-
chunkCount:
|
|
21457
|
-
transcriptText:
|
|
21458
|
-
chunks:
|
|
21459
|
-
startSec:
|
|
21460
|
-
endSec:
|
|
21461
|
-
text:
|
|
22547
|
+
selectedQuality: import_zod27.z.string(),
|
|
22548
|
+
bitrate: import_zod27.z.number().int().nullable(),
|
|
22549
|
+
videoDurationSec: import_zod27.z.number().nullable(),
|
|
22550
|
+
videoUrl: import_zod27.z.string().url(),
|
|
22551
|
+
wordCount: import_zod27.z.number().int().min(0),
|
|
22552
|
+
chunkCount: import_zod27.z.number().int().min(0),
|
|
22553
|
+
transcriptText: import_zod27.z.string(),
|
|
22554
|
+
chunks: import_zod27.z.array(import_zod27.z.object({
|
|
22555
|
+
startSec: import_zod27.z.number(),
|
|
22556
|
+
endSec: import_zod27.z.number(),
|
|
22557
|
+
text: import_zod27.z.string()
|
|
22558
|
+
}))
|
|
22559
|
+
};
|
|
22560
|
+
TranscriptChunkOutput = import_zod27.z.object({
|
|
22561
|
+
startSec: import_zod27.z.number(),
|
|
22562
|
+
endSec: import_zod27.z.number(),
|
|
22563
|
+
text: import_zod27.z.string()
|
|
22564
|
+
});
|
|
22565
|
+
InstagramBrowserOutput = import_zod27.z.object({
|
|
22566
|
+
mode: import_zod27.z.enum(["hosted", "local"]),
|
|
22567
|
+
requestedMode: import_zod27.z.enum(["auto", "hosted", "local"]),
|
|
22568
|
+
profileName: NullableString,
|
|
22569
|
+
profileSource: import_zod27.z.enum(["hosted", "managed_profile", "direct_profile_dir"]),
|
|
22570
|
+
profileDirConfigured: import_zod27.z.boolean(),
|
|
22571
|
+
executablePathConfigured: import_zod27.z.boolean()
|
|
22572
|
+
});
|
|
22573
|
+
InstagramPaginationOutput = import_zod27.z.object({
|
|
22574
|
+
maxItems: import_zod27.z.number().int().min(1).max(500),
|
|
22575
|
+
maxScrolls: import_zod27.z.number().int().min(0).max(100),
|
|
22576
|
+
attemptedScrolls: import_zod27.z.number().int().min(0),
|
|
22577
|
+
stableScrolls: import_zod27.z.number().int().min(0),
|
|
22578
|
+
stableScrollLimit: import_zod27.z.number().int().min(1).max(10),
|
|
22579
|
+
scrollDelayMs: import_zod27.z.number().int().min(250).max(5e3),
|
|
22580
|
+
reachedMaxItems: import_zod27.z.boolean(),
|
|
22581
|
+
reachedReportedPostCount: import_zod27.z.boolean(),
|
|
22582
|
+
finalScrollHeight: import_zod27.z.number().int().nullable(),
|
|
22583
|
+
stoppedReason: import_zod27.z.enum(["max_items", "reported_post_count", "stable_scrolls", "max_scrolls", "no_scrolls"]),
|
|
22584
|
+
stages: import_zod27.z.array(import_zod27.z.object({
|
|
22585
|
+
stage: import_zod27.z.string(),
|
|
22586
|
+
itemCount: import_zod27.z.number().int().min(0),
|
|
22587
|
+
addedCount: import_zod27.z.number().int().min(0),
|
|
22588
|
+
scrollY: import_zod27.z.number().nullable(),
|
|
22589
|
+
scrollHeight: import_zod27.z.number().nullable()
|
|
22590
|
+
}))
|
|
22591
|
+
});
|
|
22592
|
+
InstagramProfileContentOutputSchema = {
|
|
22593
|
+
handle: import_zod27.z.string(),
|
|
22594
|
+
profileUrl: import_zod27.z.string().url(),
|
|
22595
|
+
pageUrl: import_zod27.z.string().url(),
|
|
22596
|
+
browser: InstagramBrowserOutput,
|
|
22597
|
+
profileName: NullableString,
|
|
22598
|
+
reportedPostCount: import_zod27.z.number().int().nullable(),
|
|
22599
|
+
reportedPostCountText: NullableString,
|
|
22600
|
+
followerCountText: NullableString,
|
|
22601
|
+
followingCountText: NullableString,
|
|
22602
|
+
collectedContentCount: import_zod27.z.number().int().min(0),
|
|
22603
|
+
typeCounts: import_zod27.z.object({
|
|
22604
|
+
post: import_zod27.z.number().int().min(0),
|
|
22605
|
+
reel: import_zod27.z.number().int().min(0),
|
|
22606
|
+
tv: import_zod27.z.number().int().min(0)
|
|
22607
|
+
}),
|
|
22608
|
+
pagination: InstagramPaginationOutput,
|
|
22609
|
+
limited: import_zod27.z.boolean(),
|
|
22610
|
+
limitations: import_zod27.z.array(import_zod27.z.string()),
|
|
22611
|
+
items: import_zod27.z.array(import_zod27.z.object({
|
|
22612
|
+
url: import_zod27.z.string().url(),
|
|
22613
|
+
type: import_zod27.z.enum(["post", "reel", "tv"]),
|
|
22614
|
+
shortcode: import_zod27.z.string(),
|
|
22615
|
+
anchorText: NullableString,
|
|
22616
|
+
firstSeenStage: import_zod27.z.string()
|
|
21462
22617
|
}))
|
|
21463
22618
|
};
|
|
21464
|
-
|
|
21465
|
-
|
|
21466
|
-
|
|
21467
|
-
|
|
22619
|
+
InstagramMediaTrackOutput = import_zod27.z.object({
|
|
22620
|
+
url: import_zod27.z.string().url(),
|
|
22621
|
+
streamType: import_zod27.z.enum(["video", "audio", "unknown"]),
|
|
22622
|
+
bitrate: import_zod27.z.number().int().nullable(),
|
|
22623
|
+
durationSec: import_zod27.z.number().nullable(),
|
|
22624
|
+
vencodeTag: NullableString,
|
|
22625
|
+
width: import_zod27.z.number().int().nullable(),
|
|
22626
|
+
height: import_zod27.z.number().int().nullable()
|
|
21468
22627
|
});
|
|
22628
|
+
InstagramDownloadOutput = import_zod27.z.object({
|
|
22629
|
+
kind: import_zod27.z.enum(["text", "image", "video", "audio", "muxed_video"]),
|
|
22630
|
+
url: import_zod27.z.string().url().nullable(),
|
|
22631
|
+
savedPath: NullableString,
|
|
22632
|
+
sizeBytes: import_zod27.z.number().int().nullable(),
|
|
22633
|
+
mimeType: NullableString,
|
|
22634
|
+
error: NullableString
|
|
22635
|
+
});
|
|
22636
|
+
InstagramMediaDownloadOutputSchema = {
|
|
22637
|
+
sourceUrl: import_zod27.z.string().url(),
|
|
22638
|
+
pageUrl: import_zod27.z.string().url(),
|
|
22639
|
+
browser: InstagramBrowserOutput,
|
|
22640
|
+
type: import_zod27.z.enum(["post", "reel", "tv"]).nullable(),
|
|
22641
|
+
shortcode: NullableString,
|
|
22642
|
+
ownerName: NullableString,
|
|
22643
|
+
caption: NullableString,
|
|
22644
|
+
imageUrl: import_zod27.z.string().url().nullable(),
|
|
22645
|
+
trackCount: import_zod27.z.number().int().min(0),
|
|
22646
|
+
selectedVideoTrack: InstagramMediaTrackOutput.nullable(),
|
|
22647
|
+
selectedAudioTrack: InstagramMediaTrackOutput.nullable(),
|
|
22648
|
+
downloads: import_zod27.z.array(InstagramDownloadOutput),
|
|
22649
|
+
outputDir: NullableString,
|
|
22650
|
+
warnings: import_zod27.z.array(import_zod27.z.string()),
|
|
22651
|
+
limitations: import_zod27.z.array(import_zod27.z.string()),
|
|
22652
|
+
transcript: import_zod27.z.object({
|
|
22653
|
+
wordCount: import_zod27.z.number().int().min(0),
|
|
22654
|
+
chunkCount: import_zod27.z.number().int().min(0),
|
|
22655
|
+
durationMs: import_zod27.z.number().nullable(),
|
|
22656
|
+
transcriptText: import_zod27.z.string(),
|
|
22657
|
+
chunks: import_zod27.z.array(TranscriptChunkOutput)
|
|
22658
|
+
}).nullable()
|
|
22659
|
+
};
|
|
21469
22660
|
YoutubeTranscribeOutputSchema = {
|
|
21470
22661
|
videoId: NullableString,
|
|
21471
22662
|
url: NullableString,
|
|
21472
|
-
wordCount:
|
|
21473
|
-
chunkCount:
|
|
21474
|
-
durationMs:
|
|
21475
|
-
transcriptText:
|
|
21476
|
-
chunks:
|
|
21477
|
-
resolvedInputs:
|
|
22663
|
+
wordCount: import_zod27.z.number().int().min(0),
|
|
22664
|
+
chunkCount: import_zod27.z.number().int().min(0),
|
|
22665
|
+
durationMs: import_zod27.z.number().nullable(),
|
|
22666
|
+
transcriptText: import_zod27.z.string(),
|
|
22667
|
+
chunks: import_zod27.z.array(TranscriptChunkOutput),
|
|
22668
|
+
resolvedInputs: import_zod27.z.object({
|
|
21478
22669
|
videoId: NullableString,
|
|
21479
22670
|
url: NullableString
|
|
21480
22671
|
})
|
|
21481
22672
|
};
|
|
21482
22673
|
FacebookAdTranscribeOutputSchema = {
|
|
21483
|
-
videoUrl:
|
|
21484
|
-
wordCount:
|
|
21485
|
-
chunkCount:
|
|
21486
|
-
durationMs:
|
|
21487
|
-
transcriptText:
|
|
21488
|
-
chunks:
|
|
21489
|
-
resolvedInputs:
|
|
21490
|
-
videoUrl:
|
|
22674
|
+
videoUrl: import_zod27.z.string().url(),
|
|
22675
|
+
wordCount: import_zod27.z.number().int().min(0),
|
|
22676
|
+
chunkCount: import_zod27.z.number().int().min(0),
|
|
22677
|
+
durationMs: import_zod27.z.number().nullable(),
|
|
22678
|
+
transcriptText: import_zod27.z.string(),
|
|
22679
|
+
chunks: import_zod27.z.array(TranscriptChunkOutput),
|
|
22680
|
+
resolvedInputs: import_zod27.z.object({
|
|
22681
|
+
videoUrl: import_zod27.z.string().url()
|
|
21491
22682
|
})
|
|
21492
22683
|
};
|
|
21493
22684
|
CaptureSerpSnapshotOutputSchema = {
|
|
21494
|
-
schemaVersion:
|
|
21495
|
-
status:
|
|
22685
|
+
schemaVersion: import_zod27.z.literal("serp-intelligence.capture.v1"),
|
|
22686
|
+
status: import_zod27.z.string(),
|
|
21496
22687
|
query: NullableString,
|
|
21497
22688
|
location: NullableString,
|
|
21498
22689
|
capturedAt: NullableString,
|
|
21499
|
-
resultCount:
|
|
22690
|
+
resultCount: import_zod27.z.number().int().min(0).nullable(),
|
|
21500
22691
|
snapshotId: NullableString,
|
|
21501
|
-
resolvedInputs:
|
|
21502
|
-
artifacts:
|
|
21503
|
-
diagnostics:
|
|
21504
|
-
providerPayload:
|
|
22692
|
+
resolvedInputs: import_zod27.z.record(import_zod27.z.unknown()),
|
|
22693
|
+
artifacts: import_zod27.z.array(import_zod27.z.record(import_zod27.z.unknown())),
|
|
22694
|
+
diagnostics: import_zod27.z.record(import_zod27.z.unknown()).nullable(),
|
|
22695
|
+
providerPayload: import_zod27.z.record(import_zod27.z.unknown())
|
|
21505
22696
|
};
|
|
21506
22697
|
CaptureSerpPageSnapshotsOutputSchema = {
|
|
21507
|
-
schemaVersion:
|
|
21508
|
-
status:
|
|
21509
|
-
count:
|
|
21510
|
-
failedCount:
|
|
21511
|
-
captures:
|
|
21512
|
-
resolvedInputs:
|
|
21513
|
-
diagnostics:
|
|
21514
|
-
providerPayload:
|
|
22698
|
+
schemaVersion: import_zod27.z.literal("serp-intelligence.page-snapshots.v1"),
|
|
22699
|
+
status: import_zod27.z.string(),
|
|
22700
|
+
count: import_zod27.z.number().int().min(0),
|
|
22701
|
+
failedCount: import_zod27.z.number().int().min(0),
|
|
22702
|
+
captures: import_zod27.z.array(import_zod27.z.record(import_zod27.z.unknown())),
|
|
22703
|
+
resolvedInputs: import_zod27.z.record(import_zod27.z.unknown()),
|
|
22704
|
+
diagnostics: import_zod27.z.record(import_zod27.z.unknown()).nullable(),
|
|
22705
|
+
providerPayload: import_zod27.z.record(import_zod27.z.unknown())
|
|
21515
22706
|
};
|
|
21516
22707
|
CreditsInfoInputSchema = {
|
|
21517
|
-
item:
|
|
21518
|
-
includeLedger:
|
|
22708
|
+
item: import_zod27.z.string().optional().describe('Optional tool, action, or feature to look up, e.g. "maps reviews", "extract_url", "YouTube transcription", or "concurrency"'),
|
|
22709
|
+
includeLedger: import_zod27.z.boolean().default(false).describe("Whether to include recent credit ledger entries")
|
|
21519
22710
|
};
|
|
21520
|
-
WorkflowIdSchema2 =
|
|
22711
|
+
WorkflowIdSchema2 = import_zod27.z.enum([
|
|
21521
22712
|
"directory",
|
|
21522
22713
|
"agent-packet",
|
|
21523
22714
|
"local-competitive-audit",
|
|
@@ -21527,129 +22718,129 @@ var init_mcp_tool_schemas = __esm({
|
|
|
21527
22718
|
"ai-overview-language"
|
|
21528
22719
|
]);
|
|
21529
22720
|
WorkflowListInputSchema = {
|
|
21530
|
-
includeRecipes:
|
|
22721
|
+
includeRecipes: import_zod27.z.boolean().default(true).describe("Include high-level AI-facing recipes such as market analysis, ICP research, forum/review research, brand design briefings, CRO audits, positioning briefs, content gaps, and AI search visibility audits.")
|
|
21531
22722
|
};
|
|
21532
22723
|
WorkflowSuggestInputSchema = {
|
|
21533
|
-
goal:
|
|
21534
|
-
query:
|
|
21535
|
-
keyword:
|
|
21536
|
-
domain:
|
|
21537
|
-
url:
|
|
21538
|
-
location:
|
|
21539
|
-
state:
|
|
21540
|
-
maxSuggestions:
|
|
22724
|
+
goal: import_zod27.z.string().min(1).describe('The user goal or job to route, e.g. "market analysis for roofers in Tennessee", "ICP research for med spas", "CRO audit for this URL", or "brand design briefing".'),
|
|
22725
|
+
query: import_zod27.z.string().optional().describe("Business category, niche, or Maps query when known."),
|
|
22726
|
+
keyword: import_zod27.z.string().optional().describe("Search keyword, audience problem, or content topic when known."),
|
|
22727
|
+
domain: import_zod27.z.string().optional().describe("Target domain or brand domain when known."),
|
|
22728
|
+
url: import_zod27.z.string().url().optional().describe("Target URL when the workflow should inspect a specific page."),
|
|
22729
|
+
location: import_zod27.z.string().optional().describe("City/region/country for localized research, e.g. Denver, CO."),
|
|
22730
|
+
state: import_zod27.z.string().optional().describe("US state abbreviation or name for state-wide market research."),
|
|
22731
|
+
maxSuggestions: import_zod27.z.number().int().min(1).max(8).default(3).describe("Number of matching workflow recipes to return.")
|
|
21541
22732
|
};
|
|
21542
22733
|
WorkflowRunInputSchema = {
|
|
21543
22734
|
workflowId: WorkflowIdSchema2.describe("Workflow to run: directory, agent-packet, local-competitive-audit, map-comparison, serp-comparison, paa-expansion-brief, ai-overview-language. Use only these values; call workflow_list or workflow_suggest first when unsure."),
|
|
21544
|
-
input:
|
|
21545
|
-
webhookUrl:
|
|
22735
|
+
input: import_zod27.z.record(import_zod27.z.unknown()).default({}).describe("Workflow-specific input object. Examples: agent-packet uses {keyword, domain?, location?, maxQuestions?}; local-competitive-audit uses {query, state, minPopulation?, maxCities?, maxResultsPerCity?, hydrateTop?, maxReviews?}; serp-comparison uses {keyword, domain?, url?, location?, extractTop?}."),
|
|
22736
|
+
webhookUrl: import_zod27.z.string().url().optional().describe("Optional HTTPS webhook to receive the completed hosted workflow run event.")
|
|
21546
22737
|
};
|
|
21547
22738
|
WorkflowStepInputSchema = {
|
|
21548
|
-
runId:
|
|
22739
|
+
runId: import_zod27.z.string().min(1).describe("Workflow run id returned by workflow_run, workflow_step, or workflow_status. Use only a returned runId; do not construct one yourself. Advances the run by exactly one step (one logical leg, e.g. one live harvest).")
|
|
21549
22740
|
};
|
|
21550
22741
|
WorkflowStatusInputSchema = {
|
|
21551
|
-
runId:
|
|
22742
|
+
runId: import_zod27.z.string().min(1).describe("Workflow run id returned by workflow_run, workflow_step, or workflow_status. Use only a returned runId; do not construct one yourself.")
|
|
21552
22743
|
};
|
|
21553
22744
|
WorkflowArtifactReadInputSchema = {
|
|
21554
|
-
runId:
|
|
21555
|
-
artifactId:
|
|
21556
|
-
maxBytes:
|
|
22745
|
+
runId: import_zod27.z.string().min(1).describe("Workflow run id returned by workflow_run, workflow_step, or workflow_status. Use only a returned runId; do not construct one yourself."),
|
|
22746
|
+
artifactId: import_zod27.z.string().min(1).describe("Artifact id from the run artifact list returned by workflow_run, workflow_step, or workflow_status. Use only a returned artifactId; do not construct one yourself."),
|
|
22747
|
+
maxBytes: import_zod27.z.number().int().min(1e3).max(1e6).default(2e5).describe("Maximum bytes of artifact text to return inline. Use lower values for large CSV/JSON artifacts; call again with the downloadUrl if needed outside MCP.")
|
|
21557
22748
|
};
|
|
21558
|
-
WorkflowRecipeOutput =
|
|
21559
|
-
id:
|
|
21560
|
-
title:
|
|
21561
|
-
description:
|
|
21562
|
-
primaryWorkflowId:
|
|
21563
|
-
recommendedTools:
|
|
21564
|
-
requiredInputs:
|
|
21565
|
-
optionalInputs:
|
|
21566
|
-
produces:
|
|
21567
|
-
runHint:
|
|
22749
|
+
WorkflowRecipeOutput = import_zod27.z.object({
|
|
22750
|
+
id: import_zod27.z.string(),
|
|
22751
|
+
title: import_zod27.z.string(),
|
|
22752
|
+
description: import_zod27.z.string(),
|
|
22753
|
+
primaryWorkflowId: import_zod27.z.string().nullable(),
|
|
22754
|
+
recommendedTools: import_zod27.z.array(import_zod27.z.string()),
|
|
22755
|
+
requiredInputs: import_zod27.z.array(import_zod27.z.string()),
|
|
22756
|
+
optionalInputs: import_zod27.z.array(import_zod27.z.string()),
|
|
22757
|
+
produces: import_zod27.z.array(import_zod27.z.string()),
|
|
22758
|
+
runHint: import_zod27.z.string()
|
|
21568
22759
|
});
|
|
21569
|
-
WorkflowDefinitionOutput =
|
|
21570
|
-
id:
|
|
21571
|
-
title:
|
|
21572
|
-
description:
|
|
22760
|
+
WorkflowDefinitionOutput = import_zod27.z.object({
|
|
22761
|
+
id: import_zod27.z.string(),
|
|
22762
|
+
title: import_zod27.z.string(),
|
|
22763
|
+
description: import_zod27.z.string()
|
|
21573
22764
|
});
|
|
21574
|
-
WorkflowArtifactOutput =
|
|
22765
|
+
WorkflowArtifactOutput = import_zod27.z.record(import_zod27.z.unknown());
|
|
21575
22766
|
WorkflowListOutputSchema = {
|
|
21576
|
-
workflows:
|
|
21577
|
-
recipes:
|
|
22767
|
+
workflows: import_zod27.z.array(WorkflowDefinitionOutput),
|
|
22768
|
+
recipes: import_zod27.z.array(WorkflowRecipeOutput)
|
|
21578
22769
|
};
|
|
21579
22770
|
WorkflowSuggestOutputSchema = {
|
|
21580
|
-
goal:
|
|
21581
|
-
suggestions:
|
|
22771
|
+
goal: import_zod27.z.string(),
|
|
22772
|
+
suggestions: import_zod27.z.array(WorkflowRecipeOutput)
|
|
21582
22773
|
};
|
|
21583
22774
|
WorkflowRunOutputSchema = {
|
|
21584
|
-
workflowId:
|
|
21585
|
-
input:
|
|
21586
|
-
run:
|
|
21587
|
-
summary:
|
|
21588
|
-
step:
|
|
21589
|
-
nextStep:
|
|
21590
|
-
done:
|
|
21591
|
-
artifacts:
|
|
22775
|
+
workflowId: import_zod27.z.string(),
|
|
22776
|
+
input: import_zod27.z.record(import_zod27.z.unknown()),
|
|
22777
|
+
run: import_zod27.z.record(import_zod27.z.unknown()).optional(),
|
|
22778
|
+
summary: import_zod27.z.record(import_zod27.z.unknown()).optional(),
|
|
22779
|
+
step: import_zod27.z.record(import_zod27.z.unknown()).optional(),
|
|
22780
|
+
nextStep: import_zod27.z.record(import_zod27.z.unknown()).nullable().optional(),
|
|
22781
|
+
done: import_zod27.z.boolean().optional(),
|
|
22782
|
+
artifacts: import_zod27.z.array(WorkflowArtifactOutput)
|
|
21592
22783
|
};
|
|
21593
22784
|
WorkflowStepOutputSchema = {
|
|
21594
|
-
runId:
|
|
21595
|
-
run:
|
|
21596
|
-
summary:
|
|
21597
|
-
step:
|
|
21598
|
-
nextStep:
|
|
21599
|
-
done:
|
|
21600
|
-
artifacts:
|
|
22785
|
+
runId: import_zod27.z.string(),
|
|
22786
|
+
run: import_zod27.z.record(import_zod27.z.unknown()).optional(),
|
|
22787
|
+
summary: import_zod27.z.record(import_zod27.z.unknown()).nullable().optional(),
|
|
22788
|
+
step: import_zod27.z.record(import_zod27.z.unknown()).optional(),
|
|
22789
|
+
nextStep: import_zod27.z.record(import_zod27.z.unknown()).nullable().optional(),
|
|
22790
|
+
done: import_zod27.z.boolean(),
|
|
22791
|
+
artifacts: import_zod27.z.array(WorkflowArtifactOutput)
|
|
21601
22792
|
};
|
|
21602
22793
|
WorkflowStatusOutputSchema = {
|
|
21603
|
-
run:
|
|
21604
|
-
artifacts:
|
|
22794
|
+
run: import_zod27.z.record(import_zod27.z.unknown()).optional(),
|
|
22795
|
+
artifacts: import_zod27.z.array(WorkflowArtifactOutput)
|
|
21605
22796
|
};
|
|
21606
22797
|
WorkflowArtifactReadOutputSchema = {
|
|
21607
|
-
runId:
|
|
21608
|
-
artifactId:
|
|
21609
|
-
contentType:
|
|
21610
|
-
bytes:
|
|
21611
|
-
truncated:
|
|
21612
|
-
text:
|
|
22798
|
+
runId: import_zod27.z.string(),
|
|
22799
|
+
artifactId: import_zod27.z.string(),
|
|
22800
|
+
contentType: import_zod27.z.string(),
|
|
22801
|
+
bytes: import_zod27.z.number().int().min(0),
|
|
22802
|
+
truncated: import_zod27.z.boolean(),
|
|
22803
|
+
text: import_zod27.z.string()
|
|
21613
22804
|
};
|
|
21614
22805
|
SearchSerpInputSchema = {
|
|
21615
|
-
query:
|
|
21616
|
-
location:
|
|
21617
|
-
gl:
|
|
21618
|
-
hl:
|
|
21619
|
-
device:
|
|
21620
|
-
proxyMode:
|
|
21621
|
-
proxyZip:
|
|
21622
|
-
debug:
|
|
21623
|
-
pages:
|
|
22806
|
+
query: import_zod27.z.string().min(1).describe('Core search topic only. Separate location when possible. If user says "best dentist in Brooklyn NY serp", use query="best dentist" and location="Brooklyn, NY".'),
|
|
22807
|
+
location: import_zod27.z.string().optional().describe("City, region, or country for geo-targeted results, inferred from user request when present."),
|
|
22808
|
+
gl: import_zod27.z.string().length(2).default("us").describe("Google country code inferred from location or user language."),
|
|
22809
|
+
hl: import_zod27.z.string().default("en").describe("Google interface/content language inferred from user request."),
|
|
22810
|
+
device: import_zod27.z.enum(["desktop", "mobile"]).default("desktop").describe("SERP device context. Use desktop by default; use mobile only when the user asks for mobile rankings."),
|
|
22811
|
+
proxyMode: import_zod27.z.enum(["location", "configured", "none"]).default(DEFAULT_PROXY_MODE).describe("Proxy targeting mode. Default configured uses the service proxy without city/ZIP targeting for the highest general SERP success rate. Use location only when the user explicitly needs city/ZIP-targeted residential proxy evidence. Use none only for direct-network debugging."),
|
|
22812
|
+
proxyZip: import_zod27.z.string().regex(/^\d{5}$/).optional().describe("Optional US ZIP override for residential location proxy targeting. Use only with proxyMode location when the user gives a specific ZIP or city-center targeting needs to be forced."),
|
|
22813
|
+
debug: import_zod27.z.boolean().default(false).describe("Include sanitized browser/session/location diagnostics in the response. Use true when debugging localization, CAPTCHA, or proxy behavior."),
|
|
22814
|
+
pages: import_zod27.z.number().int().min(1).max(2).default(1).describe("Number of result pages to fetch (1\u20132)")
|
|
21624
22815
|
};
|
|
21625
22816
|
CaptureSerpSnapshotInputSchema = {
|
|
21626
|
-
query:
|
|
21627
|
-
location:
|
|
21628
|
-
gl:
|
|
21629
|
-
hl:
|
|
21630
|
-
device:
|
|
21631
|
-
proxyMode:
|
|
21632
|
-
proxyZip:
|
|
21633
|
-
pages:
|
|
21634
|
-
debug:
|
|
21635
|
-
includePageSnapshots:
|
|
21636
|
-
pageSnapshotLimit:
|
|
22817
|
+
query: import_zod27.z.string().min(1).describe("Core search query to capture as a structured SERP Intelligence snapshot. Separate the place into location when the user gives a city, region, country, or ZIP."),
|
|
22818
|
+
location: import_zod27.z.string().optional().describe("City, region, country, or service area used for localized Google results. MCP Scraper records location evidence; UULE alone is not proof of localization."),
|
|
22819
|
+
gl: import_zod27.z.string().length(2).default("us").describe("Google country code inferred from the requested market, e.g. us, gb, ca, au."),
|
|
22820
|
+
hl: import_zod27.z.string().default("en").describe("Google interface/content language inferred from the user request."),
|
|
22821
|
+
device: import_zod27.z.enum(["desktop", "mobile"]).default("desktop").describe("SERP device context. Use mobile only when the user asks for mobile rankings or mobile SERP evidence."),
|
|
22822
|
+
proxyMode: import_zod27.z.enum(["location", "configured", "none"]).default(DEFAULT_PROXY_MODE).describe("Proxy behavior for capture. Default configured uses the service proxy without city/ZIP targeting for the highest general capture success rate. Use location only when the user explicitly needs city/ZIP-targeted residential proxy evidence. Use none only for direct-network debugging."),
|
|
22823
|
+
proxyZip: import_zod27.z.string().regex(/^\d{5}$/).optional().describe("Optional US ZIP override for residential location proxy targeting. Use only with proxyMode location when a precise city-center or ZIP proxy is needed."),
|
|
22824
|
+
pages: import_zod27.z.number().int().min(1).max(2).default(1).describe("Number of Google result pages to capture. Use 1 normally and 2 only when the user needs deeper ranking evidence."),
|
|
22825
|
+
debug: import_zod27.z.boolean().default(false).describe("Include sanitized browser, proxy, and location diagnostics. Use true when debugging localization, CAPTCHA, proxy selection, or capture reliability."),
|
|
22826
|
+
includePageSnapshots: import_zod27.z.boolean().default(false).describe("Also capture ranking-page snapshots for selected SERP URLs through the same product capture path."),
|
|
22827
|
+
pageSnapshotLimit: import_zod27.z.number().int().min(0).max(10).default(0).describe("Maximum ranking-page snapshots to capture when includePageSnapshots is true. Use 0 when only SERP evidence is needed.")
|
|
21637
22828
|
};
|
|
21638
22829
|
ScreenshotInputSchema = {
|
|
21639
|
-
url:
|
|
21640
|
-
device:
|
|
21641
|
-
allowLocal:
|
|
22830
|
+
url: import_zod27.z.string().url().describe("URL to capture as a full-page screenshot. Use http or https. Pass allowLocal: true to capture localhost or private-network URLs during development."),
|
|
22831
|
+
device: import_zod27.z.enum(["desktop", "mobile"]).default("desktop").describe("Viewport profile. desktop = 1440\xD7900. mobile = 390\xD7844. Use desktop by default; use mobile when the user asks for a mobile view."),
|
|
22832
|
+
allowLocal: import_zod27.z.boolean().default(false).describe("Allow localhost and private-network URLs (127.x, 192.168.x, 10.x, etc.). For local development only \u2014 not for production use.")
|
|
21642
22833
|
};
|
|
21643
22834
|
CaptureSerpPageSnapshotsInputSchema = {
|
|
21644
|
-
urls:
|
|
21645
|
-
targets:
|
|
21646
|
-
url:
|
|
21647
|
-
sourceKind:
|
|
21648
|
-
sourcePosition:
|
|
22835
|
+
urls: import_zod27.z.array(import_zod27.z.string().url()).min(1).max(25).describe("Public HTTP/HTTPS URLs to capture as SERP Intelligence page snapshots. Do not pass localhost, private IPs, file URLs, or internal admin URLs."),
|
|
22836
|
+
targets: import_zod27.z.array(import_zod27.z.object({
|
|
22837
|
+
url: import_zod27.z.string().url().describe("Public HTTP/HTTPS URL to capture."),
|
|
22838
|
+
sourceKind: import_zod27.z.enum(["organic", "ai_citation", "local_pack_website", "configured_target", "site_subject"]).default("configured_target").describe("Why this page is being captured for SERP Intelligence evidence."),
|
|
22839
|
+
sourcePosition: import_zod27.z.number().int().min(1).optional().describe("Ranking or citation position when the page came from SERP evidence.")
|
|
21649
22840
|
}).strict()).min(1).max(25).optional().describe("Structured page snapshot targets. Use this instead of urls when source kind or position should be preserved."),
|
|
21650
|
-
maxConcurrency:
|
|
21651
|
-
timeoutMs:
|
|
21652
|
-
debug:
|
|
22841
|
+
maxConcurrency: import_zod27.z.number().int().min(1).max(5).default(2).describe("Parallel page captures. Use 2 normally; higher values can increase proxy/browser pressure."),
|
|
22842
|
+
timeoutMs: import_zod27.z.number().int().min(1e3).max(6e4).default(15e3).describe("Per-page capture timeout in milliseconds. Increase for slow pages; timeout artifacts are returned as structured capture failures."),
|
|
22843
|
+
debug: import_zod27.z.boolean().default(false).describe("Include sanitized browser/proxy diagnostics for page snapshot debugging. Use true for capture, network, or proxy troubleshooting.")
|
|
21653
22844
|
};
|
|
21654
22845
|
}
|
|
21655
22846
|
});
|
|
@@ -22004,8 +23195,8 @@ function localPlanningToolAnnotations(title) {
|
|
|
22004
23195
|
}
|
|
22005
23196
|
function listSavedReports() {
|
|
22006
23197
|
try {
|
|
22007
|
-
const dir =
|
|
22008
|
-
return (0,
|
|
23198
|
+
const dir = outputBaseDir2();
|
|
23199
|
+
return (0, import_node_fs8.readdirSync)(dir).filter((f) => f.endsWith(".md")).map((f) => ({ filename: f, mtimeMs: (0, import_node_fs8.statSync)((0, import_node_path12.join)(dir, f)).mtimeMs })).sort((a, b) => b.mtimeMs - a.mtimeMs).slice(0, 100);
|
|
22009
23200
|
} catch {
|
|
22010
23201
|
return [];
|
|
22011
23202
|
}
|
|
@@ -22029,9 +23220,9 @@ function registerSavedReportResources(server) {
|
|
|
22029
23220
|
},
|
|
22030
23221
|
async (uri, variables) => {
|
|
22031
23222
|
const requested = Array.isArray(variables.filename) ? variables.filename[0] : variables.filename;
|
|
22032
|
-
const filename = (0,
|
|
23223
|
+
const filename = (0, import_node_path12.basename)(decodeURIComponent(String(requested ?? "")));
|
|
22033
23224
|
if (!filename.endsWith(".md")) throw new Error("Only saved .md reports can be read");
|
|
22034
|
-
const text = (0,
|
|
23225
|
+
const text = (0, import_node_fs8.readFileSync)((0, import_node_path12.join)(outputBaseDir2(), filename), "utf8");
|
|
22035
23226
|
return { contents: [{ uri: uri.href, mimeType: "text/markdown", text }] };
|
|
22036
23227
|
}
|
|
22037
23228
|
);
|
|
@@ -22123,6 +23314,20 @@ function registerPaaExtractorMcpTools(server, executor, options = {}) {
|
|
|
22123
23314
|
outputSchema: FacebookVideoTranscribeOutputSchema,
|
|
22124
23315
|
annotations: liveWebToolAnnotations("Facebook Organic Video Transcription")
|
|
22125
23316
|
}, async (input) => formatFacebookVideoTranscribe(await executor.facebookVideoTranscribe(input), input));
|
|
23317
|
+
server.registerTool("instagram_profile_content", {
|
|
23318
|
+
title: "Instagram Profile Content Discovery",
|
|
23319
|
+
description: withReportNote("Discover Instagram profile grid content links for a handle or profile URL. Use this when the user wants a person or brand account content inventory before selecting posts/reels to download. Returns profile stats, collected post/reel/tv URLs, shortcodes, type counts, browser mode, pagination attempts, stop reason, and limitations. For authenticated collection, first use browser_profile_import/sync, then pass browserProfile and raise maxItems/maxScrolls for longer profile pagination."),
|
|
23320
|
+
inputSchema: InstagramProfileContentInputSchema,
|
|
23321
|
+
outputSchema: InstagramProfileContentOutputSchema,
|
|
23322
|
+
annotations: liveWebToolAnnotations("Instagram Profile Content Discovery")
|
|
23323
|
+
}, async (input) => formatInstagramProfileContent(await executor.instagramProfileContent(input), input));
|
|
23324
|
+
server.registerTool("instagram_media_download", {
|
|
23325
|
+
title: "Instagram Post/Reel Media Download",
|
|
23326
|
+
description: withReportNote("Extract and download media from one Instagram post, reel, or tv URL. Use after instagram_profile_content or when the user gives a specific Instagram URL and wants the image, caption/text, reel audio/video tracks, optional muxed MP4, or optional transcript. Reels commonly expose separate video-only and audio-only MP4 tracks; this tool selects the best video and audio tracks and attempts muxing when ffmpeg is available. Pass browserProfile from browser_profile_import/sync when the media is login-gated."),
|
|
23327
|
+
inputSchema: InstagramMediaDownloadInputSchema,
|
|
23328
|
+
outputSchema: InstagramMediaDownloadOutputSchema,
|
|
23329
|
+
annotations: liveWebToolAnnotations("Instagram Post/Reel Media Download")
|
|
23330
|
+
}, async (input) => formatInstagramMediaDownload(await executor.instagramMediaDownload(input), input));
|
|
22126
23331
|
server.registerTool("maps_place_intel", {
|
|
22127
23332
|
title: "Google Maps Business Profile Details",
|
|
22128
23333
|
description: withReportNote('Extract Google Maps business intelligence for one known/named business: rating, review count, category, address, phone, website, hours, booking URL, review histogram, review topics, about attributes, entity IDs, and optional review cards. Do not use this for category searches, local market prospect lists, or requests for multiple GMB/GBP profiles; use maps_search first for those. Split business name from location (e.g. "Elite Roofing Denver CO" => businessName "Elite Roofing", location "Denver, CO"). Pass includeReviews true when the user asks for reviews/customer pain.'),
|
|
@@ -22207,13 +23412,13 @@ function registerPaaExtractorMcpTools(server, executor, options = {}) {
|
|
|
22207
23412
|
}
|
|
22208
23413
|
}, async (input) => formatCreditsInfo(await executor.creditsInfo(input), input));
|
|
22209
23414
|
}
|
|
22210
|
-
var import_mcp,
|
|
23415
|
+
var import_mcp, import_node_fs8, import_node_path12;
|
|
22211
23416
|
var init_paa_mcp_server = __esm({
|
|
22212
23417
|
"src/mcp/paa-mcp-server.ts"() {
|
|
22213
23418
|
"use strict";
|
|
22214
23419
|
import_mcp = require("@modelcontextprotocol/sdk/server/mcp.js");
|
|
22215
|
-
|
|
22216
|
-
|
|
23420
|
+
import_node_fs8 = require("fs");
|
|
23421
|
+
import_node_path12 = require("path");
|
|
22217
23422
|
init_version();
|
|
22218
23423
|
init_mcp_response_formatter();
|
|
22219
23424
|
init_mcp_tool_schemas();
|
|
@@ -22402,6 +23607,12 @@ var init_http_mcp_tool_executor = __esm({
|
|
|
22402
23607
|
facebookVideoTranscribe(input) {
|
|
22403
23608
|
return this.call("/facebook/video-transcribe", input, this.httpTimeoutOverrideMs ?? 24e4);
|
|
22404
23609
|
}
|
|
23610
|
+
instagramProfileContent(input) {
|
|
23611
|
+
return this.call("/instagram/profile-content", input, this.httpTimeoutOverrideMs ?? 24e4);
|
|
23612
|
+
}
|
|
23613
|
+
instagramMediaDownload(input) {
|
|
23614
|
+
return this.call("/instagram/media-download", input, this.httpTimeoutOverrideMs ?? 3e5);
|
|
23615
|
+
}
|
|
22405
23616
|
mapsPlaceIntel(input) {
|
|
22406
23617
|
return this.call("/maps/place", input);
|
|
22407
23618
|
}
|
|
@@ -22495,11 +23706,11 @@ function registerSerpIntelligenceCaptureTools(server, executor) {
|
|
|
22495
23706
|
annotations: liveWebToolAnnotations("SERP Intelligence Page Snapshots")
|
|
22496
23707
|
}, async (input) => formatCaptureSerpPageSnapshots(await executor.captureSerpPageSnapshots(input), input));
|
|
22497
23708
|
}
|
|
22498
|
-
var
|
|
23709
|
+
var import_hono10, import_webStandardStreamableHttp, mcpApp;
|
|
22499
23710
|
var init_mcp_routes = __esm({
|
|
22500
23711
|
"src/mcp/mcp-routes.ts"() {
|
|
22501
23712
|
"use strict";
|
|
22502
|
-
|
|
23713
|
+
import_hono10 = require("hono");
|
|
22503
23714
|
import_webStandardStreamableHttp = require("@modelcontextprotocol/sdk/server/webStandardStreamableHttp.js");
|
|
22504
23715
|
init_paa_mcp_server();
|
|
22505
23716
|
init_http_mcp_tool_executor();
|
|
@@ -22507,7 +23718,7 @@ var init_mcp_routes = __esm({
|
|
|
22507
23718
|
init_db();
|
|
22508
23719
|
init_mcp_tool_schemas();
|
|
22509
23720
|
configureReportSaving(false);
|
|
22510
|
-
mcpApp = new
|
|
23721
|
+
mcpApp = new import_hono10.Hono();
|
|
22511
23722
|
mcpApp.all("/", async (c) => {
|
|
22512
23723
|
try {
|
|
22513
23724
|
const keyOrError = await requireMcpCallerKey(c);
|
|
@@ -23371,19 +24582,19 @@ var init_classify = __esm({
|
|
|
23371
24582
|
});
|
|
23372
24583
|
|
|
23373
24584
|
// src/services/fanout/export.ts
|
|
23374
|
-
function
|
|
23375
|
-
return process.env.MCP_SCRAPER_OUTPUT_DIR?.trim() || (0,
|
|
24585
|
+
function outputBaseDir3() {
|
|
24586
|
+
return process.env.MCP_SCRAPER_OUTPUT_DIR?.trim() || (0, import_node_path13.join)((0, import_node_os8.homedir)(), "Downloads", "mcp-scraper");
|
|
23376
24587
|
}
|
|
23377
24588
|
function safe(value) {
|
|
23378
24589
|
return value.replace(/[^a-zA-Z0-9._-]+/g, "-").replace(/^-+|-+$/g, "").slice(0, 80) || "capture";
|
|
23379
24590
|
}
|
|
23380
24591
|
function writeTable(path6, rows, delimiter) {
|
|
23381
|
-
(0,
|
|
24592
|
+
(0, import_node_fs9.writeFileSync)(path6, import_papaparse3.default.unparse(rows.length ? rows : [{}], { delimiter }));
|
|
23382
24593
|
}
|
|
23383
24594
|
function exportFanout(enriched) {
|
|
23384
24595
|
const stamp = safe(enriched.capturedAt.replace(/[:.]/g, "-"));
|
|
23385
|
-
const dir = (0,
|
|
23386
|
-
(0,
|
|
24596
|
+
const dir = (0, import_node_path13.join)(outputBaseDir3(), "fanout", `${stamp}-${safe(enriched.platform)}`);
|
|
24597
|
+
(0, import_node_fs9.mkdirSync)(dir, { recursive: true });
|
|
23387
24598
|
const queryRows = enriched.queries.map((q, i) => ({ index: i + 1, query: q }));
|
|
23388
24599
|
const citationRows = enriched.aggregates.citationOrder.map((c) => {
|
|
23389
24600
|
const src = enriched.citedUrls.find((s) => s.url === c.url);
|
|
@@ -23393,21 +24604,21 @@ function exportFanout(enriched) {
|
|
|
23393
24604
|
const domainRows = enriched.aggregates.topSites.map((d) => ({ domain: d.domain, count: d.count, cited: d.cited, timesCited: d.timesCited, siteType: d.siteType }));
|
|
23394
24605
|
const paths = {
|
|
23395
24606
|
dir,
|
|
23396
|
-
json: (0,
|
|
23397
|
-
queriesCsv: (0,
|
|
23398
|
-
queriesTsv: (0,
|
|
23399
|
-
citationsCsv: (0,
|
|
23400
|
-
sourcesCsv: (0,
|
|
23401
|
-
domainsCsv: (0,
|
|
23402
|
-
report: (0,
|
|
24607
|
+
json: (0, import_node_path13.join)(dir, "fanout.json"),
|
|
24608
|
+
queriesCsv: (0, import_node_path13.join)(dir, "queries.csv"),
|
|
24609
|
+
queriesTsv: (0, import_node_path13.join)(dir, "queries.tsv"),
|
|
24610
|
+
citationsCsv: (0, import_node_path13.join)(dir, "citations.csv"),
|
|
24611
|
+
sourcesCsv: (0, import_node_path13.join)(dir, "sources.csv"),
|
|
24612
|
+
domainsCsv: (0, import_node_path13.join)(dir, "domains.csv"),
|
|
24613
|
+
report: (0, import_node_path13.join)(dir, "report.html")
|
|
23403
24614
|
};
|
|
23404
|
-
(0,
|
|
24615
|
+
(0, import_node_fs9.writeFileSync)(paths.json, JSON.stringify(enriched, null, 2));
|
|
23405
24616
|
writeTable(paths.queriesCsv, queryRows, ",");
|
|
23406
24617
|
writeTable(paths.queriesTsv, queryRows, " ");
|
|
23407
24618
|
writeTable(paths.citationsCsv, citationRows, ",");
|
|
23408
24619
|
writeTable(paths.sourcesCsv, sourceRows2, ",");
|
|
23409
24620
|
writeTable(paths.domainsCsv, domainRows, ",");
|
|
23410
|
-
(0,
|
|
24621
|
+
(0, import_node_fs9.writeFileSync)(paths.report, renderReportHtml(enriched));
|
|
23411
24622
|
return paths;
|
|
23412
24623
|
}
|
|
23413
24624
|
function esc(s) {
|
|
@@ -23485,13 +24696,13 @@ render();
|
|
|
23485
24696
|
</script>
|
|
23486
24697
|
</body></html>`;
|
|
23487
24698
|
}
|
|
23488
|
-
var
|
|
24699
|
+
var import_node_fs9, import_node_os8, import_node_path13, import_papaparse3;
|
|
23489
24700
|
var init_export = __esm({
|
|
23490
24701
|
"src/services/fanout/export.ts"() {
|
|
23491
24702
|
"use strict";
|
|
23492
|
-
|
|
23493
|
-
|
|
23494
|
-
|
|
24703
|
+
import_node_fs9 = require("fs");
|
|
24704
|
+
import_node_os8 = require("os");
|
|
24705
|
+
import_node_path13 = require("path");
|
|
23495
24706
|
import_papaparse3 = __toESM(require("papaparse"), 1);
|
|
23496
24707
|
}
|
|
23497
24708
|
});
|
|
@@ -23983,7 +25194,7 @@ async function loadOpenSession(id, userId) {
|
|
|
23983
25194
|
return row;
|
|
23984
25195
|
}
|
|
23985
25196
|
function buildBrowserAgentRoutes() {
|
|
23986
|
-
const app2 = new
|
|
25197
|
+
const app2 = new import_hono11.Hono();
|
|
23987
25198
|
app2.use("*", async (c, next) => {
|
|
23988
25199
|
await migrateBrowserAgent();
|
|
23989
25200
|
return next();
|
|
@@ -24340,11 +25551,11 @@ function buildBrowserAgentRoutes() {
|
|
|
24340
25551
|
});
|
|
24341
25552
|
return app2;
|
|
24342
25553
|
}
|
|
24343
|
-
var
|
|
25554
|
+
var import_hono11, auth, DEFAULT_BROWSER_SESSION_LOCK_TTL_SECONDS;
|
|
24344
25555
|
var init_browser_agent_routes = __esm({
|
|
24345
25556
|
"src/api/browser-agent-routes.ts"() {
|
|
24346
25557
|
"use strict";
|
|
24347
|
-
|
|
25558
|
+
import_hono11 = require("hono");
|
|
24348
25559
|
init_api_auth();
|
|
24349
25560
|
init_errors();
|
|
24350
25561
|
init_db();
|
|
@@ -24561,16 +25772,16 @@ var init_browser_agent_console = __esm({
|
|
|
24561
25772
|
});
|
|
24562
25773
|
|
|
24563
25774
|
// src/api/stripe-routes.ts
|
|
24564
|
-
var import_stripe,
|
|
25775
|
+
var import_stripe, import_hono12, stripe, stripeApp;
|
|
24565
25776
|
var init_stripe_routes = __esm({
|
|
24566
25777
|
"src/api/stripe-routes.ts"() {
|
|
24567
25778
|
"use strict";
|
|
24568
25779
|
import_stripe = __toESM(require("stripe"), 1);
|
|
24569
|
-
|
|
25780
|
+
import_hono12 = require("hono");
|
|
24570
25781
|
init_db();
|
|
24571
25782
|
init_rates();
|
|
24572
25783
|
stripe = new import_stripe.default(process.env.STRIPE_SECRET_KEY, { apiVersion: "2026-02-25.clover" });
|
|
24573
|
-
stripeApp = new
|
|
25784
|
+
stripeApp = new import_hono12.Hono();
|
|
24574
25785
|
stripeApp.post("/webhooks", async (c) => {
|
|
24575
25786
|
const sig = c.req.header("stripe-signature");
|
|
24576
25787
|
const body = await c.req.text();
|
|
@@ -24671,31 +25882,31 @@ var init_site_audit_worker = __esm({
|
|
|
24671
25882
|
});
|
|
24672
25883
|
|
|
24673
25884
|
// src/api/billing-schemas.ts
|
|
24674
|
-
var
|
|
25885
|
+
var import_zod28, BillingCheckoutBodySchema, FreeCreditBreakdownSchema, BillingBalanceResponseSchema, MonthlyRefreshSweepResultSchema;
|
|
24675
25886
|
var init_billing_schemas = __esm({
|
|
24676
25887
|
"src/api/billing-schemas.ts"() {
|
|
24677
25888
|
"use strict";
|
|
24678
|
-
|
|
24679
|
-
BillingCheckoutBodySchema =
|
|
24680
|
-
priceId:
|
|
25889
|
+
import_zod28 = require("zod");
|
|
25890
|
+
BillingCheckoutBodySchema = import_zod28.z.object({
|
|
25891
|
+
priceId: import_zod28.z.string().min(1)
|
|
24681
25892
|
});
|
|
24682
|
-
FreeCreditBreakdownSchema =
|
|
24683
|
-
signup_grant_mc:
|
|
24684
|
-
monthly_refresh_mc:
|
|
24685
|
-
total_free_mc:
|
|
24686
|
-
signup_grant_credits:
|
|
24687
|
-
monthly_refresh_credits:
|
|
24688
|
-
total_free_credits:
|
|
25893
|
+
FreeCreditBreakdownSchema = import_zod28.z.object({
|
|
25894
|
+
signup_grant_mc: import_zod28.z.number().int().nonnegative(),
|
|
25895
|
+
monthly_refresh_mc: import_zod28.z.number().int().nonnegative(),
|
|
25896
|
+
total_free_mc: import_zod28.z.number().int().nonnegative(),
|
|
25897
|
+
signup_grant_credits: import_zod28.z.number().nonnegative(),
|
|
25898
|
+
monthly_refresh_credits: import_zod28.z.number().nonnegative(),
|
|
25899
|
+
total_free_credits: import_zod28.z.number().nonnegative()
|
|
24689
25900
|
});
|
|
24690
|
-
BillingBalanceResponseSchema =
|
|
24691
|
-
balance_mc:
|
|
24692
|
-
balance_credits:
|
|
25901
|
+
BillingBalanceResponseSchema = import_zod28.z.object({
|
|
25902
|
+
balance_mc: import_zod28.z.number().int().nonnegative(),
|
|
25903
|
+
balance_credits: import_zod28.z.number().nonnegative(),
|
|
24693
25904
|
free_credits: FreeCreditBreakdownSchema,
|
|
24694
|
-
ledger:
|
|
25905
|
+
ledger: import_zod28.z.array(import_zod28.z.any())
|
|
24695
25906
|
});
|
|
24696
|
-
MonthlyRefreshSweepResultSchema =
|
|
24697
|
-
usersRefreshed:
|
|
24698
|
-
totalMcGranted:
|
|
25907
|
+
MonthlyRefreshSweepResultSchema = import_zod28.z.object({
|
|
25908
|
+
usersRefreshed: import_zod28.z.number().int().nonnegative(),
|
|
25909
|
+
totalMcGranted: import_zod28.z.number().int().nonnegative()
|
|
24699
25910
|
});
|
|
24700
25911
|
}
|
|
24701
25912
|
});
|
|
@@ -25106,7 +26317,7 @@ async function checkHarvestLimits(user, reuseLockId) {
|
|
|
25106
26317
|
}
|
|
25107
26318
|
return null;
|
|
25108
26319
|
}
|
|
25109
|
-
var import_resend,
|
|
26320
|
+
var import_resend, import_hono13, import_hono14, import_factory6, import_cookie, import_stripe2, secureCookies, isProduction2, sessionCookieOptions, requireAllowedOrigin, auth2, adminAuth, sessionAuth, app, STRIPE_API_VERSION, SYNC_HARVEST_TIMEOUT_OVERRIDE_MS;
|
|
25110
26321
|
var init_server = __esm({
|
|
25111
26322
|
"src/api/server.ts"() {
|
|
25112
26323
|
"use strict";
|
|
@@ -25124,14 +26335,15 @@ var init_server = __esm({
|
|
|
25124
26335
|
init_media_extractor();
|
|
25125
26336
|
init_site_mapper();
|
|
25126
26337
|
init_site_extractor();
|
|
25127
|
-
|
|
25128
|
-
|
|
26338
|
+
import_hono13 = require("hono");
|
|
26339
|
+
import_hono14 = require("inngest/hono");
|
|
25129
26340
|
init_client();
|
|
25130
26341
|
init_site_audit();
|
|
25131
26342
|
init_site_audit_routes();
|
|
25132
26343
|
init_youtube_routes();
|
|
25133
26344
|
init_screenshot_routes();
|
|
25134
26345
|
init_facebook_ad_routes();
|
|
26346
|
+
init_instagram_routes();
|
|
25135
26347
|
init_maps_routes();
|
|
25136
26348
|
init_directory_routes();
|
|
25137
26349
|
init_workflow_routes();
|
|
@@ -25199,7 +26411,7 @@ var init_server = __esm({
|
|
|
25199
26411
|
c.set("sessionUser", { ...refreshed, balance_mc: balanceMc });
|
|
25200
26412
|
return next();
|
|
25201
26413
|
});
|
|
25202
|
-
app = new
|
|
26414
|
+
app = new import_hono13.Hono();
|
|
25203
26415
|
STRIPE_API_VERSION = "2026-02-25.clover";
|
|
25204
26416
|
app.use("*", async (c, next) => {
|
|
25205
26417
|
await next();
|
|
@@ -25578,7 +26790,10 @@ var init_server = __esm({
|
|
|
25578
26790
|
const device = screenshotDevice === "mobile" ? "mobile" : "desktop";
|
|
25579
26791
|
const [result, pageData] = await Promise.all([
|
|
25580
26792
|
extractKpo({ url: canonicalUrl, kernelApiKey }),
|
|
25581
|
-
screenshot2 || extractBranding ? capturePageData(canonicalUrl, { kernelApiKey, device, screenshot: !!screenshot2, branding: !!extractBranding }).catch(() =>
|
|
26793
|
+
screenshot2 || extractBranding ? capturePageData(canonicalUrl, { kernelApiKey, device, screenshot: !!screenshot2, branding: !!extractBranding }).catch((err) => {
|
|
26794
|
+
console.error("[extract-url] capturePageData failed:", err instanceof Error ? err.message : err);
|
|
26795
|
+
return null;
|
|
26796
|
+
}) : null
|
|
25582
26797
|
]);
|
|
25583
26798
|
const screenshotBuf = pageData?.screenshot ?? null;
|
|
25584
26799
|
const brandingData = pageData?.branding ?? null;
|
|
@@ -25586,7 +26801,7 @@ var init_server = __esm({
|
|
|
25586
26801
|
if (screenshotBuf) {
|
|
25587
26802
|
screenshotMeta = { base64: screenshotBuf.toString("base64"), sizeBytes: screenshotBuf.length, device };
|
|
25588
26803
|
}
|
|
25589
|
-
const mediaMeta = downloadMedia ? await harvestPageMedia(result.bodyHtml, canonicalUrl, { types: mediaTypes ?? ["image", "video", "audio"] }) : null;
|
|
26804
|
+
const mediaMeta = downloadMedia ? await harvestPageMedia(result.bodyHtml, canonicalUrl, { types: mediaTypes ?? ["image", "video", "audio"], outputDir: null }) : null;
|
|
25590
26805
|
await logRequestEvent({ userId: user.id, source: "extract_url", status: "done", query: canonicalUrl, resultCount: result.headings.length, result });
|
|
25591
26806
|
return c.json({ ...result, screenshot: screenshotMeta, branding: brandingData, media: mediaMeta });
|
|
25592
26807
|
} catch (err) {
|
|
@@ -25889,11 +27104,12 @@ var init_server = __esm({
|
|
|
25889
27104
|
]);
|
|
25890
27105
|
return c.json({ drained: results.length, results, sweepResult, workflowDispatch: workflowDispatchResult });
|
|
25891
27106
|
});
|
|
25892
|
-
app.on(["GET", "POST", "PUT"], "/api/inngest", (0,
|
|
27107
|
+
app.on(["GET", "POST", "PUT"], "/api/inngest", (0, import_hono14.serve)({ client: inngest, functions: [siteAuditFn] }));
|
|
25893
27108
|
app.route("/api/internal/site-architecture-auditor", siteAuditApp);
|
|
25894
27109
|
app.route("/youtube", youtubeApp);
|
|
25895
27110
|
app.route("/screenshot", screenshotApp);
|
|
25896
27111
|
app.route("/facebook", facebookAdApp);
|
|
27112
|
+
app.route("/instagram", instagramApp);
|
|
25897
27113
|
app.route("/maps", mapsApp);
|
|
25898
27114
|
app.route("/directory", directoryApp);
|
|
25899
27115
|
app.route("/workflows", workflowApp);
|
|
@@ -26010,10 +27226,10 @@ ${ATTRIBUTION_PIXEL_SCRIPT}
|
|
|
26010
27226
|
});
|
|
26011
27227
|
|
|
26012
27228
|
// bin/api-server.ts
|
|
26013
|
-
var
|
|
27229
|
+
var import_node_fs10 = require("fs");
|
|
26014
27230
|
function loadDotEnv() {
|
|
26015
27231
|
try {
|
|
26016
|
-
for (const line of (0,
|
|
27232
|
+
for (const line of (0, import_node_fs10.readFileSync)(".env", "utf8").split("\n")) {
|
|
26017
27233
|
const eq = line.indexOf("=");
|
|
26018
27234
|
if (eq < 1 || line.trimStart().startsWith("#")) continue;
|
|
26019
27235
|
const k = line.slice(0, eq).trim();
|