mcp-scraper 0.19.0 → 0.20.0

package/dist/bin/api-server.cjs

@@ -5271,18 +5271,30 @@ async function ledgerExistsForStripePI(stripePI) {
   });
   return res.rows.length > 0;
 }
-async function reconcileBalanceMc(userId) {
+async function reconcileBalanceMc(userId, knownBalanceMc) {
   const db = getDb();
-  await ensureMigrated(Number(userId));
   const res = await db.execute({
-    sql: `SELECT COALESCE(SUM(remaining_mc), 0) AS balance_mc FROM credit_lots WHERE user_id = ? AND remaining_mc > 0 AND expires_at > datetime('now')`,
+    sql: `SELECT
+      COUNT(*) AS lot_count,
+      COALESCE(SUM(CASE WHEN remaining_mc > 0 AND expires_at > datetime('now') THEN remaining_mc ELSE 0 END), 0) AS balance_mc
+      FROM credit_lots WHERE user_id = ?`,
     args: [userId]
   });
-  const balanceMc = Number(res.rows[0]?.balance_mc ?? 0);
-  await db.execute({
-    sql: "UPDATE users SET balance_mc = ? WHERE id = ? AND balance_mc != ?",
-    args: [balanceMc, userId, balanceMc]
-  });
+  let balanceMc = Number(res.rows[0]?.balance_mc ?? 0);
+  if (Number(res.rows[0]?.lot_count ?? 0) === 0) {
+    await ensureMigrated(Number(userId));
+    const migrated = await db.execute({
+      sql: `SELECT COALESCE(SUM(remaining_mc), 0) AS balance_mc FROM credit_lots WHERE user_id = ? AND remaining_mc > 0 AND expires_at > datetime('now')`,
+      args: [userId]
+    });
+    balanceMc = Number(migrated.rows[0]?.balance_mc ?? 0);
+  }
+  if (knownBalanceMc !== balanceMc) {
+    await db.execute({
+      sql: "UPDATE users SET balance_mc = ? WHERE id = ? AND balance_mc != ?",
+      args: [balanceMc, userId, balanceMc]
+    });
+  }
   return balanceMc;
 }
 var import_http, import_node_crypto, import_zod, DB_URL, DB_TOKEN, _db, _rateLimitSchemaReady, CREDIT_LOT_TTL, SiteAuditJobRowSchema, SiteAuditPhaseLogRowSchema;
@@ -6372,8 +6384,8 @@ async function downloadAsset(url, destDir, filename) {
   }
   const writer = (0, import_node_fs.createWriteStream)(dest);
   await (0, import_promises2.pipeline)(import_node_stream.Readable.fromWeb(res.body), writer);
-  const { statSync: statSync2 } = await import("fs");
-  const sizeBytes = statSync2(dest).size;
+  const { statSync: statSync3 } = await import("fs");
+  const sizeBytes = statSync3(dest).size;
   return { savedPath: dest, sizeBytes, mimeType };
 }
 async function harvestPageMedia(html, pageUrl, options = {}) {
@@ -10746,6 +10758,15 @@ function registrableDomain2(host) {
   const parts = h.split(".");
   return parts.length <= 2 ? h : parts.slice(-2).join(".");
 }
+function safeImageFilename(url, index) {
+  try {
+    const u = new URL(url);
+    const base = (0, import_node_path3.basename)(u.pathname).replace(/[^a-zA-Z0-9._-]/g, "_").slice(0, 80);
+    return base || `image-${index}`;
+  } catch {
+    return `image-${index}`;
+  }
+}
 function slugFactory() {
   const counts = /* @__PURE__ */ new Map();
   return (url) => {
@@ -10768,7 +10789,9 @@ async function* pageChunks(jobId) {
 }
 async function assembleExtractArtifacts(job, extras = {}) {
   const dir = (0, import_node_path3.join)((0, import_node_os3.tmpdir)(), `extract-bundle-${job.id}-${Date.now()}`);
+  const downloadImages = job.options.downloadImages === true;
   (0, import_node_fs3.mkdirSync)((0, import_node_path3.join)(dir, "pages"), { recursive: true });
+  if (downloadImages) (0, import_node_fs3.mkdirSync)((0, import_node_path3.join)(dir, "images"), { recursive: true });
   try {
     const metas = [];
     const statusByUrl = /* @__PURE__ */ new Map();
@@ -10802,6 +10825,10 @@ async function assembleExtractArtifacts(job, extras = {}) {
     } catch {
       siteReg = "";
     }
+    const imageLimit = (0, import_p_limit3.default)(IMAGE_DOWNLOAD_CONCURRENCY);
+    const imageDownloads = [];
+    let imagesQueued = 0;
+    let imagesFailed = 0;
     for await (const chunk of pageChunks(job.id)) {
       for (const p of chunk) {
         if (p.bodyMarkdown) {
@@ -10819,6 +10846,20 @@ ${p.bodyMarkdown}
 `
           );
         }
+        if (downloadImages && p.imageLinks?.length && imagesQueued < MAX_IMAGES_PER_SITE) {
+          const pageDir = (0, import_node_path3.join)(dir, "images", slug(p.url));
+          (0, import_node_fs3.mkdirSync)(pageDir, { recursive: true });
+          for (const [idx, imgUrl] of p.imageLinks.slice(0, MAX_IMAGES_PER_PAGE).entries()) {
+            if (imagesQueued >= MAX_IMAGES_PER_SITE) break;
+            imagesQueued++;
+            const filename = safeImageFilename(imgUrl, idx);
+            imageDownloads.push(
+              imageLimit(() => downloadAsset(imgUrl, pageDir, filename)).then(() => void 0, () => {
+                imagesFailed++;
+              })
+            );
+          }
+        }
         const from = normalize2(p.url);
         const oc = { int: 0, ext: 0 };
         for (const l of p.outlinks ?? []) {
@@ -10866,6 +10907,7 @@ ${p.bodyMarkdown}
     }
     gz.end();
     await (0, import_node_events.once)(linksOut, "finish");
+    await Promise.all(imageDownloads);
     const depth = /* @__PURE__ */ new Map();
     const start = normalize2(job.startUrl);
     depth.set(start, 0);
@@ -10957,6 +10999,15 @@ ${renderImageSection(extras.imageAudit)}`;
     }
     if (extras.seoAudit) (0, import_node_fs3.writeFileSync)((0, import_node_path3.join)(dir, "seo-audit.json"), JSON.stringify(extras.seoAudit, null, 2));
     if (extras.branding) (0, import_node_fs3.writeFileSync)((0, import_node_path3.join)(dir, "branding.json"), JSON.stringify(extras.branding, null, 2));
+    if (downloadImages) {
+      (0, import_node_fs3.writeFileSync)((0, import_node_path3.join)(dir, "images-download-summary.json"), JSON.stringify({
+        queued: imagesQueued,
+        downloaded: imagesQueued - imagesFailed,
+        failed: imagesFailed,
+        perPageCap: MAX_IMAGES_PER_PAGE,
+        perSiteCap: MAX_IMAGES_PER_SITE
+      }, null, 2));
+    }
     const artifactFiles = [
       { name: "report.md", type: "text/markdown" },
       { name: "issues.json", type: "application/json" },
@@ -10973,13 +11024,20 @@ ${renderImageSection(extras.imageAudit)}`;
     }
     if (extras.seoAudit) artifactFiles.push({ name: "seo-audit.json", type: "application/json" });
     if (extras.branding) artifactFiles.push({ name: "branding.json", type: "application/json" });
+    if (downloadImages) artifactFiles.push({ name: "images-download-summary.json", type: "application/json" });
     const zip = new import_yazl.ZipFile();
     const zipOut = (0, import_node_fs3.createWriteStream)((0, import_node_path3.join)(dir, "bundle.zip"));
     zip.outputStream.pipe(zipOut);
     for (const f of artifactFiles) zip.addFile((0, import_node_path3.join)(dir, f.name), f.name);
     if (pagesWithBody > 0) {
-      const { readdirSync: readdirSync2 } = await import("fs");
-      for (const f of readdirSync2((0, import_node_path3.join)(dir, "pages"))) zip.addFile((0, import_node_path3.join)(dir, "pages", f), `pages/${f}`);
+      for (const f of (0, import_node_fs3.readdirSync)((0, import_node_path3.join)(dir, "pages"))) zip.addFile((0, import_node_path3.join)(dir, "pages", f), `pages/${f}`);
+    }
+    if (downloadImages && imagesQueued > imagesFailed) {
+      const imagesDir = (0, import_node_path3.join)(dir, "images");
+      for (const rel of (0, import_node_fs3.readdirSync)(imagesDir, { recursive: true })) {
+        const full = (0, import_node_path3.join)(imagesDir, rel);
+        if ((0, import_node_fs3.statSync)(full).isFile()) zip.addFile(full, `images/${rel.split("\\").join("/")}`);
+      }
     }
     zip.end();
     await (0, import_node_events.once)(zipOut, "finish");
@@ -10993,7 +11051,7 @@ ${renderImageSection(extras.imageAudit)}`;
     (0, import_node_fs3.rmSync)(dir, { recursive: true, force: true });
   }
 }
-var import_node_fs3, import_node_path3, import_node_os3, import_node_zlib, import_node_events, import_yazl, CHUNK;
+var import_node_fs3, import_node_path3, import_node_os3, import_node_zlib, import_node_events, import_yazl, import_p_limit3, CHUNK, MAX_IMAGES_PER_PAGE, MAX_IMAGES_PER_SITE, IMAGE_DOWNLOAD_CONCURRENCY;
 var init_extract_bundle = __esm({
   "src/api/extract-bundle.ts"() {
     "use strict";
@@ -11003,12 +11061,17 @@ var init_extract_bundle = __esm({
     import_node_zlib = require("zlib");
     import_node_events = require("events");
     import_yazl = require("yazl");
+    import_p_limit3 = __toESM(require("p-limit"), 1);
     init_db();
     init_blob_store();
     init_seo_issues();
     init_seo_link_report();
     init_image_audit();
+    init_media_extractor();
     CHUNK = 400;
+    MAX_IMAGES_PER_PAGE = 20;
+    MAX_IMAGES_PER_SITE = 500;
+    IMAGE_DOWNLOAD_CONCURRENCY = 8;
   }
 });
 
@@ -14049,6 +14112,7 @@ var init_server_schemas = __esm({
       rotateProxies: import_zod13.z.boolean().optional(),
       rotateProxyEvery: import_zod13.z.number().int().min(1).max(100).optional(),
       background: import_zod13.z.boolean().optional(),
+      downloadImages: import_zod13.z.boolean().optional(),
       formats: import_zod13.z.array(import_zod13.z.enum(["markdown", "links", "json", "images", "branding", "issues"])).optional()
     });
     YoutubeHarvestBodySchema = import_zod13.z.object({
@@ -20035,9 +20099,24 @@ function buildSeoExport(siteUrl, pages, branding) {
     branding: branding ?? void 0
   };
 }
+function formatBackgroundJobStarted(toolLabel, data) {
+  if (data.status !== "pending" || typeof data.jobId !== "string") return null;
+  const jobId = data.jobId;
+  const full = [
+    `# ${toolLabel} \u2014 export started`,
+    `**Job ID:** \`${jobId}\``,
+    `
+Running in the background \u2014 this can take a while for large sites.`,
+    `
+Poll \`check_site_export\` with this jobId to get the download link once it's ready.`
+  ].join("\n");
+  return { content: [{ type: "text", text: full }], structuredContent: { jobId, status: "pending" } };
+}
 async function formatExtractSite(raw, input, ctx) {
   const parsed = parseData(raw);
   if ("error" in parsed) return { content: [{ type: "text", text: parsed.error }], isError: true };
+  const started = formatBackgroundJobStarted("Multi-Page Site Content Crawl", parsed.data);
+  if (started) return started;
   const d = parsed.data;
   const pages = d.pages ?? [];
   const schemaTypesOf = (p) => p.kpo?.type ?? (Array.isArray(p.schema) && p.schema.length ? [`${p.schema.length} block(s)`] : []);
@@ -20155,6 +20234,8 @@ ${pageDetails}${tips}`;
 async function formatAuditSite(raw, input, ctx) {
   const parsed = parseData(raw);
   if ("error" in parsed) return { content: [{ type: "text", text: parsed.error }], isError: true };
+  const started = formatBackgroundJobStarted("Technical SEO Audit", parsed.data);
+  if (started) return started;
   const d = parsed.data;
   const pages = d.pages ?? [];
   const schemaTypesOf = (p) => p.kpo?.type ?? (Array.isArray(p.schema) && p.schema.length ? [`${p.schema.length} block(s)`] : []);
@@ -20226,6 +20307,42 @@ ${imgLine}`
   ].join("\n");
   return { content: [{ type: "text", text: full }], structuredContent };
 }
+function formatCheckSiteExport(raw, input) {
+  const parsed = parseData(raw);
+  if ("error" in parsed) return { content: [{ type: "text", text: parsed.error }], isError: true };
+  const d = parsed.data;
+  const bundle = (d.artifacts ?? []).find((a) => a.key.endsWith("bundle.zip")) ?? null;
+  const progress = d.totalUrls ? `${d.doneUrls ?? 0}/${d.totalUrls} pages` : "starting";
+  const body = d.status === "complete" && bundle ? `
+## \u2705 Ready
+**Download:** ${bundle.url}
+**Size:** ${(bundle.bytes / 1e6).toFixed(1)} MB` : d.status === "complete" ? `
+## \u26A0\uFE0F Complete, but no bundle was produced
+Check the job's artifacts \u2014 nothing matched \`bundle.zip\`.` : d.status === "failed" ? `
+## \u274C Failed
+${d.error ?? "no error message recorded"}` : `
+## \u23F3 Not ready yet
+Status: ${d.status} (${progress}). Poll again shortly.`;
+  const full = [
+    `# Site Export: ${d.startUrl ?? input.jobId}`,
+    `**Job ID:** \`${d.jobId}\``,
+    `**Status:** ${d.status}`,
+    body
+  ].join("\n");
+  return {
+    ...oneBlock(full),
+    structuredContent: {
+      jobId: d.jobId,
+      status: d.status,
+      startUrl: d.startUrl,
+      totalUrls: d.totalUrls,
+      doneUrls: d.doneUrls,
+      bundleUrl: bundle?.url ?? null,
+      bundleBytes: bundle?.bytes ?? null,
+      error: d.error ?? null
+    }
+  };
+}
 function formatYoutubeHarvest(raw, input) {
   const parsed = parseData(raw);
   if ("error" in parsed) return { content: [{ type: "text", text: parsed.error }], isError: true };
@@ -25043,7 +25160,7 @@ async function capturePageSnapshot(target, options = {}) {
 async function capturePageSnapshots(targets, options = {}) {
   const timeoutMs = normalizeTimeoutMs(options.timeoutMs);
   const maxConcurrency = normalizeMaxConcurrency(options.maxConcurrency);
-  const limit = (0, import_p_limit3.default)(maxConcurrency);
+  const limit = (0, import_p_limit4.default)(maxConcurrency);
   const pageSnapshotArtifacts = await Promise.all(
     targets.map((target) => limit(() => capturePageSnapshot(target, { ...options, timeoutMs })))
   );
@@ -25068,12 +25185,12 @@ async function capturePageSnapshots(targets, options = {}) {
     }
   };
 }
-var import_node_crypto10, import_p_limit3, DEFAULT_TIMEOUT_MS, DEFAULT_MAX_CONCURRENCY, DEFAULT_MAX_CONTENT_CHARS;
+var import_node_crypto10, import_p_limit4, DEFAULT_TIMEOUT_MS, DEFAULT_MAX_CONCURRENCY, DEFAULT_MAX_CONTENT_CHARS;
 var init_page_snapshot_extractor = __esm({
   "src/serp-intelligence/page-snapshot-extractor.ts"() {
     "use strict";
     import_node_crypto10 = require("crypto");
-    import_p_limit3 = __toESM(require("p-limit"), 1);
+    import_p_limit4 = __toESM(require("p-limit"), 1);
     init_kpo_extractor();
     init_url_utils();
     DEFAULT_TIMEOUT_MS = 15e3;
@@ -27740,7 +27857,7 @@ var PACKAGE_VERSION;
 var init_version = __esm({
   "src/version.ts"() {
     "use strict";
-    PACKAGE_VERSION = "0.19.0";
+    PACKAGE_VERSION = "0.20.0";
   }
 });
 
@@ -28127,7 +28244,7 @@ var init_meta_ad_creative_media = __esm({
 });
 
 // src/mcp/mcp-tool-schemas.ts
-var import_zod33, HarvestPaaInputSchema, ExtractUrlInputSchema, DiffPageInputSchema, MapSiteUrlsInputSchema, ExtractSiteInputSchema, AuditSiteInputSchema, YoutubeHarvestInputSchema, YoutubeTranscribeInputSchema, FacebookPageIntelInputSchema, FacebookAdSearchInputSchema, RedditThreadInputSchema, VideoFrameAnalysisInputSchema, VideoFrameAnalysisStatusInputSchema, FacebookAdTranscribeInputSchema, FacebookVideoTranscribeInputSchema, GoogleAdsSearchInputSchema, GoogleAdsPageIntelInputSchema, GoogleAdsTranscribeInputSchema, InstagramProfileContentInputSchema, InstagramMediaDownloadInputSchema, MapsPlaceIntelInputSchema, TrustpilotReviewsInputSchema, G2ReviewsInputSchema, ReviewCardSchema, MapsSearchInputSchema, DirectoryWorkflowInputSchema, ArtifactPointerOutputSchema, RankTrackerModeSchema, RankTrackerBlueprintInputSchema, NullableString, MapsSearchAttemptOutput, MapsSearchOutputSchema, DirectoryMapsBusinessOutput, DirectoryWorkflowOutputSchema, RankTrackerToolPlanOutput, RankTrackerTableOutput, RankTrackerCronJobOutput, RankTrackerBlueprintOutputSchema, OrganicResultOutput, AiOverviewOutput, EntityIdsOutput, HarvestPaaOutputSchema, SearchSerpOutputSchema, ExtractUrlOutputSchema, DiffPageOutputSchema, ExtractSiteOutputSchema, AuditSiteOutputSchema, MapsPlaceIntelOutputSchema, TrustpilotReviewsOutputSchema, G2ReviewsOutputSchema, CreditsInfoOutputSchema, MapSiteUrlsOutputSchema, YoutubeHarvestOutputSchema, FacebookAdSearchOutputSchema, VideoFrameAnalysisOutputSchema, VideoFrameAnalysisStatusOutputSchema, RedditThreadOutputSchema, FacebookPageIntelOutputSchema, GoogleAdsSearchOutputSchema, GoogleAdsPageIntelOutputSchema, TranscriptSignalOutput, FacebookVideoTranscribeOutputSchema, TranscriptChunkOutput, InstagramBrowserOutput, InstagramPaginationOutput, InstagramProfileContentOutputSchema, InstagramMediaTrackOutput, InstagramDownloadOutput, InstagramMediaDownloadOutputSchema, YoutubeTranscribeOutputSchema, FacebookAdTranscribeOutputSchema, GoogleAdsTranscribeOutputSchema, CaptureSerpSnapshotOutputSchema, CaptureSerpPageSnapshotsOutputSchema, CreditsInfoInputSchema, WorkflowIdSchema2, WorkflowListInputSchema, WorkflowSuggestInputSchema, WorkflowRunInputSchema, WorkflowStepInputSchema, WorkflowStatusInputSchema, WorkflowArtifactReadInputSchema, WorkflowRecipeOutput, WorkflowDefinitionOutput, WorkflowArtifactOutput, WorkflowListOutputSchema, WorkflowSuggestOutputSchema, WorkflowRunOutputSchema, WorkflowStepOutputSchema, WorkflowStatusOutputSchema, WorkflowArtifactReadOutputSchema, SearchSerpInputSchema, CaptureSerpSnapshotInputSchema, ScreenshotInputSchema, CaptureSerpPageSnapshotsInputSchema, ReportArtifactReadInputSchema, ReportArtifactReadOutputSchema, ListServiceConnectionsInputSchema, ListServiceConnectionsOutputSchema, ReadServiceConnectionInputSchema, ReadServiceConnectionOutputSchema, MetaAdCreativeMediaInputSchema, MetaAdCreativeMediaOutputSchema, ImportServiceConnectionToMemoryInputSchema, ImportServiceConnectionToMemoryOutputSchema, DescribeServiceConnectionToolInputSchema, DescribeServiceConnectionToolOutputSchema, ConnectedDataContinuationSchema, ExportConnectedServiceDataInputSchema, ConnectedDataArtifactSchema, ExportConnectedServiceDataOutputSchema, RenewConnectedDataExportDownloadInputSchema, RenewConnectedDataExportDownloadOutputSchema, CallServiceConnectionActionInputSchema, CallServiceConnectionActionOutputSchema, SetScheduledActionConnectionsInputSchema, SetScheduledActionConnectionsOutputSchema, SlackSendMessageInputSchema, SlackSendMessageOutputSchema, GmailSendMessageInputSchema, GmailSendMessageOutputSchema, GoogleCalendarCreateEventInputSchema, GoogleCalendarCreateEventOutputSchema, ZoomCreateMeetingInputSchema, ZoomCreateMeetingOutputSchema;
+var import_zod33, HarvestPaaInputSchema, ExtractUrlInputSchema, DiffPageInputSchema, MapSiteUrlsInputSchema, ExtractSiteInputSchema, AuditSiteInputSchema, CheckSiteExportInputSchema, YoutubeHarvestInputSchema, YoutubeTranscribeInputSchema, FacebookPageIntelInputSchema, FacebookAdSearchInputSchema, RedditThreadInputSchema, VideoFrameAnalysisInputSchema, VideoFrameAnalysisStatusInputSchema, FacebookAdTranscribeInputSchema, FacebookVideoTranscribeInputSchema, GoogleAdsSearchInputSchema, GoogleAdsPageIntelInputSchema, GoogleAdsTranscribeInputSchema, InstagramProfileContentInputSchema, InstagramMediaDownloadInputSchema, MapsPlaceIntelInputSchema, TrustpilotReviewsInputSchema, G2ReviewsInputSchema, ReviewCardSchema, MapsSearchInputSchema, DirectoryWorkflowInputSchema, ArtifactPointerOutputSchema, RankTrackerModeSchema, RankTrackerBlueprintInputSchema, NullableString, MapsSearchAttemptOutput, MapsSearchOutputSchema, DirectoryMapsBusinessOutput, DirectoryWorkflowOutputSchema, RankTrackerToolPlanOutput, RankTrackerTableOutput, RankTrackerCronJobOutput, RankTrackerBlueprintOutputSchema, OrganicResultOutput, AiOverviewOutput, EntityIdsOutput, HarvestPaaOutputSchema, SearchSerpOutputSchema, ExtractUrlOutputSchema, DiffPageOutputSchema, ExtractSiteOutputSchema, AuditSiteOutputSchema, CheckSiteExportOutputSchema, MapsPlaceIntelOutputSchema, TrustpilotReviewsOutputSchema, G2ReviewsOutputSchema, CreditsInfoOutputSchema, MapSiteUrlsOutputSchema, YoutubeHarvestOutputSchema, FacebookAdSearchOutputSchema, VideoFrameAnalysisOutputSchema, VideoFrameAnalysisStatusOutputSchema, RedditThreadOutputSchema, FacebookPageIntelOutputSchema, GoogleAdsSearchOutputSchema, GoogleAdsPageIntelOutputSchema, TranscriptSignalOutput, FacebookVideoTranscribeOutputSchema, TranscriptChunkOutput, InstagramBrowserOutput, InstagramPaginationOutput, InstagramProfileContentOutputSchema, InstagramMediaTrackOutput, InstagramDownloadOutput, InstagramMediaDownloadOutputSchema, YoutubeTranscribeOutputSchema, FacebookAdTranscribeOutputSchema, GoogleAdsTranscribeOutputSchema, CaptureSerpSnapshotOutputSchema, CaptureSerpPageSnapshotsOutputSchema, CreditsInfoInputSchema, WorkflowIdSchema2, WorkflowListInputSchema, WorkflowSuggestInputSchema, WorkflowRunInputSchema, WorkflowStepInputSchema, WorkflowStatusInputSchema, WorkflowArtifactReadInputSchema, WorkflowRecipeOutput, WorkflowDefinitionOutput, WorkflowArtifactOutput, WorkflowListOutputSchema, WorkflowSuggestOutputSchema, WorkflowRunOutputSchema, WorkflowStepOutputSchema, WorkflowStatusOutputSchema, WorkflowArtifactReadOutputSchema, SearchSerpInputSchema, CaptureSerpSnapshotInputSchema, ScreenshotInputSchema, CaptureSerpPageSnapshotsInputSchema, ReportArtifactReadInputSchema, ReportArtifactReadOutputSchema, ListServiceConnectionsInputSchema, ListServiceConnectionsOutputSchema, ReadServiceConnectionInputSchema, ReadServiceConnectionOutputSchema, MetaAdCreativeMediaInputSchema, MetaAdCreativeMediaOutputSchema, ImportServiceConnectionToMemoryInputSchema, ImportServiceConnectionToMemoryOutputSchema, DescribeServiceConnectionToolInputSchema, DescribeServiceConnectionToolOutputSchema, ConnectedDataContinuationSchema, ExportConnectedServiceDataInputSchema, ConnectedDataArtifactSchema, ExportConnectedServiceDataOutputSchema, RenewConnectedDataExportDownloadInputSchema, RenewConnectedDataExportDownloadOutputSchema, CallServiceConnectionActionInputSchema, CallServiceConnectionActionOutputSchema, SetScheduledActionConnectionsInputSchema, SetScheduledActionConnectionsOutputSchema, SlackSendMessageInputSchema, SlackSendMessageOutputSchema, GmailSendMessageInputSchema, GmailSendMessageOutputSchema, GoogleCalendarCreateEventInputSchema, GoogleCalendarCreateEventOutputSchema, ZoomCreateMeetingInputSchema, ZoomCreateMeetingOutputSchema;
 var init_mcp_tool_schemas = __esm({
   "src/mcp/mcp-tool-schemas.ts"() {
     "use strict";
@@ -28169,13 +28286,20 @@ var init_mcp_tool_schemas = __esm({
       maxPages: import_zod33.z.number().int().min(1).max(1e4).optional().describe("Maximum pages to extract. Bulk crawls (over 25 pages) switch to folder mode: each page saved as its own Markdown file, with a summary plus folder path returned instead of inlining content."),
       rotateProxies: import_zod33.z.boolean().optional().describe("Route page fetches through rotating residential proxies to defeat rate-limiting and bot blocks (403/429). Slower and pricier \u2014 use only when a site blocks normal crawling."),
       rotateProxyEvery: import_zod33.z.number().int().min(1).max(100).optional().describe("When rotateProxies is on, pages fetched per proxy before rotating. Default 30."),
-      formats: import_zod33.z.array(import_zod33.z.enum(["markdown", "links", "json", "images", "branding"])).optional().describe("Per-page output formats: markdown, links, json, images are captured cheaply from HTML; branding (site-level logo/colors/fonts) requires a browser and adds time. Defaults to markdown+links.")
+      formats: import_zod33.z.array(import_zod33.z.enum(["markdown", "links", "json", "images", "branding"])).optional().describe("Per-page output formats: markdown, links, json, images are captured cheaply from HTML; branding (site-level logo/colors/fonts) requires a browser and adds time. Defaults to markdown+links."),
+      background: import_zod33.z.boolean().default(false).describe("Run the crawl as a background job instead of blocking this call, returning a jobId immediately \u2014 poll it with check_site_export to get a downloadable zip (all page content, plus real image files if downloadImages is set) once ready. Use for large sites where a synchronous call would be slow."),
+      downloadImages: import_zod33.z.boolean().default(false).describe("Download every discovered image as a real file into the export bundle (not just image URLs/stats). OFF by default \u2014 must be explicitly set true. Implies background regardless of the background flag, since downloading a whole site's images is too slow to run synchronously. Capped at 20 images/page and 500 images/site.")
     };
     AuditSiteInputSchema = {
       url: import_zod33.z.string().url().describe("Public website URL or domain for a full technical SEO audit (issues, link graph, indexability, headings, images). For plain content use extract_site instead."),
       maxPages: import_zod33.z.number().int().min(1).max(1e4).optional().describe("Maximum pages to crawl and audit. Always writes a folder of analysis files plus per-page content, returning a summary plus the folder path."),
       rotateProxies: import_zod33.z.boolean().optional().describe("Route page fetches through rotating residential proxies to defeat rate-limiting and bot blocks. Slower/pricier \u2014 use only when a site blocks normal crawling."),
-      rotateProxyEvery: import_zod33.z.number().int().min(1).max(100).optional().describe("When rotateProxies is on, pages fetched per proxy before rotating. Default 30.")
+      rotateProxyEvery: import_zod33.z.number().int().min(1).max(100).optional().describe("When rotateProxies is on, pages fetched per proxy before rotating. Default 30."),
+      background: import_zod33.z.boolean().default(false).describe("Run the audit as a background job instead of blocking this call, returning a jobId immediately \u2014 poll it with check_site_export to get a downloadable zip (full audit report, all page content, plus real image files if downloadImages is set) once ready. Use for large sites where a synchronous call would be slow."),
+      downloadImages: import_zod33.z.boolean().default(false).describe("Download every discovered image as a real file into the export bundle (not just image URLs/stats). OFF by default \u2014 must be explicitly set true. Implies background regardless of the background flag, since downloading a whole site's images is too slow to run synchronously. Capped at 20 images/page and 500 images/site.")
+    };
+    CheckSiteExportInputSchema = {
+      jobId: import_zod33.z.string().min(1).describe('The jobId returned by extract_site or audit_site when called with background (or downloadImages) set \u2014 poll this until status is "complete" (or "failed").')
     };
     YoutubeHarvestInputSchema = {
       mode: import_zod33.z.enum(["search", "channel"]).describe("Use search for topic/keyword requests. Use channel when the user provides @handle, channel ID, or channel URL."),
@@ -28578,36 +28702,52 @@ var init_mcp_tool_schemas = __esm({
     };
     ExtractSiteOutputSchema = {
       url: import_zod33.z.string(),
-      pageCount: import_zod33.z.number().int().min(0),
+      pageCount: import_zod33.z.number().int().min(0).optional().describe("Absent when background is true \u2014 the crawl has not finished yet."),
       pages: import_zod33.z.array(import_zod33.z.object({
         url: import_zod33.z.string(),
         title: NullableString,
         schemaTypes: import_zod33.z.array(import_zod33.z.string())
-      })),
-      durationMs: import_zod33.z.number().min(0),
+      })).optional().describe("Absent when background is true \u2014 the crawl has not finished yet."),
+      durationMs: import_zod33.z.number().min(0).optional().describe("Absent when background is true \u2014 the crawl has not finished yet."),
       truncatedCount: import_zod33.z.number().int().min(0).optional(),
-      artifact: ArtifactPointerOutputSchema.optional()
+      artifact: ArtifactPointerOutputSchema.optional(),
+      jobId: import_zod33.z.string().optional().describe("Present when background (or downloadImages) was set \u2014 poll with check_site_export."),
+      status: import_zod33.z.enum(["pending"]).optional().describe("Present when background (or downloadImages) was set."),
+      statusUrl: import_zod33.z.string().optional().describe("Present when background (or downloadImages) was set \u2014 informational; use check_site_export with jobId, not this URL directly.")
     };
     AuditSiteOutputSchema = {
       url: import_zod33.z.string(),
-      pageCount: import_zod33.z.number().int().min(0),
-      durationMs: import_zod33.z.number().min(0),
-      bulkFolder: import_zod33.z.string().nullable(),
-      issues: import_zod33.z.record(import_zod33.z.string(), import_zod33.z.number()),
+      pageCount: import_zod33.z.number().int().min(0).optional().describe("Absent when background is true \u2014 the audit has not finished yet."),
+      durationMs: import_zod33.z.number().min(0).optional().describe("Absent when background is true \u2014 the audit has not finished yet."),
+      bulkFolder: import_zod33.z.string().nullable().optional().describe("Absent when background is true \u2014 the audit has not finished yet."),
+      issues: import_zod33.z.record(import_zod33.z.string(), import_zod33.z.number()).optional().describe("Absent when background is true \u2014 the audit has not finished yet."),
       images: import_zod33.z.object({
         unique: import_zod33.z.number().int().min(0),
         totalBytes: import_zod33.z.number().min(0),
         over100kb: import_zod33.z.number().int().min(0),
         legacyFormat: import_zod33.z.number().int().min(0)
-      }),
+      }).optional().describe("Absent when background is true \u2014 the audit has not finished yet."),
       links: import_zod33.z.object({
         internal: import_zod33.z.number().int().min(0),
         external: import_zod33.z.number().int().min(0),
         orphans: import_zod33.z.number().int().min(0),
         brokenInternal: import_zod33.z.number().int().min(0),
         externalDomains: import_zod33.z.number().int().min(0)
-      }),
-      artifact: ArtifactPointerOutputSchema.optional()
+      }).optional().describe("Absent when background is true \u2014 the audit has not finished yet."),
+      artifact: ArtifactPointerOutputSchema.optional(),
+      jobId: import_zod33.z.string().optional().describe("Present when background (or downloadImages) was set \u2014 poll with check_site_export."),
+      status: import_zod33.z.enum(["pending"]).optional().describe("Present when background (or downloadImages) was set."),
+      statusUrl: import_zod33.z.string().optional().describe("Present when background (or downloadImages) was set \u2014 informational; use check_site_export with jobId, not this URL directly.")
+    };
+    CheckSiteExportOutputSchema = {
+      jobId: import_zod33.z.string(),
+      status: import_zod33.z.enum(["pending", "running", "complete", "failed"]),
+      startUrl: import_zod33.z.string().optional(),
+      totalUrls: import_zod33.z.number().int().min(0).optional(),
+      doneUrls: import_zod33.z.number().int().min(0).optional(),
+      bundleUrl: import_zod33.z.string().nullable().describe("Downloadable zip URL once status is complete; null otherwise."),
+      bundleBytes: import_zod33.z.number().int().min(0).nullable().describe("Zip size in bytes once status is complete; null otherwise."),
+      error: import_zod33.z.string().nullable().optional().describe("Present with a message when status is failed.")
     };
     MapsPlaceIntelOutputSchema = {
       name: import_zod33.z.string(),
@@ -29919,6 +30059,13 @@ function registerPaaExtractorMcpTools(server, executor, options = {}) {
     outputSchema: recordOutputSchema("audit_site", AuditSiteOutputSchema),
     annotations: liveWebToolAnnotations("Technical SEO Audit")
   }, async (input) => formatAuditSite(await executor.auditSite(input), input, ctx));
+  server.registerTool("check_site_export", {
+    title: "Check Site Export",
+    description: "Poll the status of a background extract_site or audit_site job (one started with background or downloadImages set). Returns a downloadable zip URL (all page content, plus real image files if downloadImages was set) once status is complete.",
+    inputSchema: CheckSiteExportInputSchema,
+    outputSchema: recordOutputSchema("check_site_export", CheckSiteExportOutputSchema),
+    annotations: liveWebToolAnnotations("Check Site Export")
+  }, async (input) => formatCheckSiteExport(await executor.checkSiteExport(input), input));
   server.registerTool("youtube_harvest", {
     title: "YouTube Video Harvest",
     description: 'Harvest YouTube video metadata by topic search or channel library. Use mode "search" for keyword/topic requests, mode "channel" for @handles/channel IDs/URLs. Returns titles, views, durations, and videoIds.',
@@ -30436,6 +30583,9 @@ var init_http_mcp_tool_executor = __esm({
       auditSite(input) {
         return this.call("/extract-site", input);
       }
+      checkSiteExport(input) {
+        return this.getJson(`/extract-site/status/${encodeURIComponent(input.jobId)}`);
+      }
       youtubeHarvest(input) {
         return this.call("/youtube/harvest", input);
       }
@@ -41906,7 +42056,7 @@ var init_server = __esm({
       const user = await getUserByApiKey(key);
       if (!user) return c.json({ error: "Invalid or inactive API key" }, 401);
       const refreshed = await applyMonthlyFreeRefresh(user);
-      const balanceMc = await reconcileBalanceMc(refreshed.id);
+      const balanceMc = await reconcileBalanceMc(refreshed.id, refreshed.balance_mc);
       c.set("user", { ...refreshed, balance_mc: balanceMc });
       return next();
     });
@@ -41925,7 +42075,7 @@ var init_server = __esm({
       const user = await getUserById(userId);
       if (!user) return c.json({ error: "User not found" }, 401);
       const refreshed = await applyMonthlyFreeRefresh(user);
-      const balanceMc = await reconcileBalanceMc(refreshed.id);
+      const balanceMc = await reconcileBalanceMc(refreshed.id, refreshed.balance_mc);
       c.set("sessionUser", { ...refreshed, balance_mc: balanceMc });
       return next();
     });
@@ -42115,7 +42265,7 @@ var init_server = __esm({
       }
       const refreshed = await applyMonthlyFreeRefresh(foundUser);
       const [balanceMc, stats] = await Promise.all([
-        reconcileBalanceMc(refreshed.id),
+        reconcileBalanceMc(refreshed.id, refreshed.balance_mc),
         getUserStats(refreshed.id)
       ]);
       const user = { ...refreshed, balance_mc: balanceMc };
@@ -43383,7 +43533,8 @@ var init_server = __esm({
       if (checked.error || !checked.parsed) return c.json({ error: checked.error ?? "Invalid URL" }, 400);
       const parsed = checked.parsed;
       const user = c.get("user");
-      if (body.background === true) {
+      const downloadImages = body.downloadImages === true;
+      if (body.background === true || downloadImages) {
         const affordablePages = Math.floor(user.balance_mc / MC_COSTS.page_scrape);
         if (affordablePages < 1) return c.json(insufficientBalanceResponse(user.balance_mc, MC_COSTS.page_scrape), 402);
         const maxPages = Math.min(Math.min(1e4, Math.max(1, body.maxPages ?? 1e4)), affordablePages);
@@ -43398,6 +43549,7 @@ var init_server = __esm({
             concurrency: concurrencyLimitForUser(user),
             urlsPerBrowser: body.rotateProxyEvery ?? 10,
             formats: body.formats,
+            downloadImages,
             heldMc
           });
           jobCreated = true;
@@ -43706,7 +43858,7 @@ var init_server = __esm({
     });
     app.get("/billing/balance", auth2, async (c) => {
       const user = c.get("user");
-      const balanceMc = await reconcileBalanceMc(user.id);
+      const balanceMc = user.balance_mc;
       const ledger = await getLedger(user.id, 20);
       const freeCredits = await getFreeCreditBreakdown(user.id);
       return c.json({
@@ -43718,7 +43870,7 @@ var init_server = __esm({
     });
     app.post("/billing/credits", auth2, async (c) => {
       const user = c.get("user");
-      const balanceMc = await reconcileBalanceMc(user.id);
+      const balanceMc = user.balance_mc;
       const body = await c.req.json().catch(() => ({}));
       const query = body.item?.trim().toLowerCase();
       const costs = CREDIT_COST_CATALOG.map(({ aliases, ...cost }) => cost);