mcp-scraper 0.19.0 → 0.21.0

package/dist/{site-extract-repository-NVSZH35Y.js → site-extract-repository-YNPXEZCB.js}

@@ -10,10 +10,10 @@ import {
   saveExtractPages,
   setExtractJobTotal,
   settleExtractJob
-} from "./chunk-RUGJE5EB.js";
-import "./chunk-HL33CGJF.js";
+} from "./chunk-TUSLATZY.js";
+import "./chunk-HXGZMMW5.js";
 import "./chunk-M2S27J6Z.js";
-import "./chunk-GL4BW4CP.js";
+import "./chunk-DZBGULWW.js";
 export {
   completeExtractJob,
   countSuccessfulPages,
@@ -27,4 +27,4 @@ export {
   setExtractJobTotal,
   settleExtractJob
 };
-//# sourceMappingURL=site-extract-repository-NVSZH35Y.js.map
+//# sourceMappingURL=site-extract-repository-YNPXEZCB.js.map

package/dist/{worker-AM2DHUWG.js → worker-45WEWCMQ.js}

@@ -3,19 +3,19 @@ import {
   createHarvestAttemptRecorder,
   harvestProblemResponse,
   serializeHarvestProblem
-} from "./chunk-HE2LQPJ2.js";
+} from "./chunk-JBME5WJR.js";
 import {
   harvest
-} from "./chunk-2HDMYW4B.js";
+} from "./chunk-ZJDANZTQ.js";
 import {
   browserServiceApiKey,
   runWithCostContext
-} from "./chunk-D7ZT27HY.js";
+} from "./chunk-GMZFQRH2.js";
 import "./chunk-K443GQY5.js";
 import "./chunk-XGIPATLV.js";
 import {
   MC_COSTS
-} from "./chunk-HL33CGJF.js";
+} from "./chunk-HXGZMMW5.js";
 import "./chunk-M2S27J6Z.js";
 import {
   claimPendingJob,
@@ -24,7 +24,7 @@ import {
   debitMc,
   failJob,
   listHarvestAttempts
-} from "./chunk-GL4BW4CP.js";
+} from "./chunk-DZBGULWW.js";
 
 // src/api/webhook.ts
 async function deliverWebhook(url, payload, retries = 3) {
@@ -132,4 +132,4 @@ export {
   startWorker,
   tickOnce
 };
-//# sourceMappingURL=worker-AM2DHUWG.js.map
+//# sourceMappingURL=worker-45WEWCMQ.js.map

package/docs/mcp-tool-manifest.generated.json

@@ -1,12 +1,12 @@
 {
-  "generatedAt": "2026-07-13T04:02:37.913Z",
+  "generatedAt": "2026-07-14T17:00:17.517Z",
   "generatedFrom": "dist/bin/mcp-stdio-server.js",
   "serverInfo": {
     "name": "mcp-scraper",
-    "version": "0.19.0"
+    "version": "0.21.0"
   },
   "counts": {
-    "unified_stdio": 158
+    "unified_stdio": 159
   },
   "surfaces": {
     "unified_stdio": [
@@ -55,6 +55,7 @@
       "call_service_connection_action",
       "capture_serp_page_snapshots",
       "capture_serp_snapshot",
+      "check_site_export",
       "cost-usage",
       "create-channel",
       "create-scheduled-action",
@@ -1702,6 +1703,16 @@
             "minimum": 1,
             "maximum": 100,
             "description": "When rotateProxies is on, pages fetched per proxy before rotating. Default 30."
+          },
+          "background": {
+            "type": "boolean",
+            "default": false,
+            "description": "Run the audit as a background job instead of blocking this call, returning a jobId immediately — poll it with check_site_export to get a downloadable zip (full audit report, all page content, plus real image files if downloadImages is set) once ready. Use for large sites where a synchronous call would be slow."
+          },
+          "downloadImages": {
+            "type": "boolean",
+            "default": false,
+            "description": "Download every discovered image as a real file into the export bundle (not just image URLs/stats). OFF by default — must be explicitly set true. Implies background regardless of the background flag, since downloading a whole site's images is too slow to run synchronously. Capped at 20 images/page and 500 images/site."
           }
         },
         "required": [
@@ -1718,23 +1729,27 @@
           },
           "pageCount": {
             "type": "integer",
-            "minimum": 0
+            "minimum": 0,
+            "description": "Absent when background is true — the audit has not finished yet."
           },
           "durationMs": {
             "type": "number",
-            "minimum": 0
+            "minimum": 0,
+            "description": "Absent when background is true — the audit has not finished yet."
           },
           "bulkFolder": {
             "type": [
               "string",
               "null"
-            ]
+            ],
+            "description": "Absent when background is true — the audit has not finished yet."
           },
           "issues": {
             "type": "object",
             "additionalProperties": {
               "type": "number"
-            }
+            },
+            "description": "Absent when background is true — the audit has not finished yet."
           },
           "images": {
             "type": "object",
@@ -1762,7 +1777,8 @@
               "over100kb",
               "legacyFormat"
             ],
-            "additionalProperties": false
+            "additionalProperties": false,
+            "description": "Absent when background is true — the audit has not finished yet."
           },
           "links": {
             "type": "object",
@@ -1795,7 +1811,8 @@
               "brokenInternal",
               "externalDomains"
             ],
-            "additionalProperties": false
+            "additionalProperties": false,
+            "description": "Absent when background is true — the audit has not finished yet."
           },
           "artifact": {
             "type": "object",
@@ -1821,16 +1838,25 @@
               "preview"
             ],
             "additionalProperties": false
+          },
+          "jobId": {
+            "type": "string",
+            "description": "Present when background (or downloadImages) was set — poll with check_site_export."
+          },
+          "status": {
+            "type": "string",
+            "enum": [
+              "pending"
+            ],
+            "description": "Present when background (or downloadImages) was set."
+          },
+          "statusUrl": {
+            "type": "string",
+            "description": "Present when background (or downloadImages) was set — informational; use check_site_export with jobId, not this URL directly."
           }
         },
         "required": [
-          "url",
-          "pageCount",
-          "durationMs",
-          "bulkFolder",
-          "issues",
-          "images",
-          "links"
+          "url"
         ],
         "additionalProperties": false,
         "$schema": "http://json-schema.org/draft-07/schema#"
@@ -4557,6 +4583,98 @@
         "taskSupport": "forbidden"
       }
     },
+    {
+      "name": "check_site_export",
+      "title": "Check Site Export",
+      "description": "Poll the status of a background extract_site or audit_site job (one started with background or downloadImages set). Returns a downloadable zip URL (all page content, plus real image files if downloadImages was set) once status is complete.",
+      "inputSchema": {
+        "type": "object",
+        "properties": {
+          "jobId": {
+            "type": "string",
+            "minLength": 1,
+            "description": "The jobId returned by extract_site or audit_site when called with background (or downloadImages) set — poll this until status is \"complete\" (or \"failed\")."
+          }
+        },
+        "required": [
+          "jobId"
+        ],
+        "additionalProperties": false,
+        "$schema": "http://json-schema.org/draft-07/schema#"
+      },
+      "outputSchema": {
+        "type": "object",
+        "properties": {
+          "jobId": {
+            "type": "string"
+          },
+          "status": {
+            "type": "string",
+            "enum": [
+              "pending",
+              "running",
+              "complete",
+              "failed"
+            ]
+          },
+          "startUrl": {
+            "type": "string"
+          },
+          "totalUrls": {
+            "type": "integer",
+            "minimum": 0
+          },
+          "doneUrls": {
+            "type": "integer",
+            "minimum": 0
+          },
+          "bundleUrl": {
+            "type": [
+              "string",
+              "null"
+            ],
+            "description": "Downloadable zip URL once status is complete; null otherwise."
+          },
+          "bundleBytes": {
+            "anyOf": [
+              {
+                "type": "integer",
+                "minimum": 0
+              },
+              {
+                "type": "null"
+              }
+            ],
+            "description": "Zip size in bytes once status is complete; null otherwise."
+          },
+          "error": {
+            "type": [
+              "string",
+              "null"
+            ],
+            "description": "Present with a message when status is failed."
+          }
+        },
+        "required": [
+          "jobId",
+          "status",
+          "bundleUrl",
+          "bundleBytes"
+        ],
+        "additionalProperties": false,
+        "$schema": "http://json-schema.org/draft-07/schema#"
+      },
+      "annotations": {
+        "title": "Check Site Export",
+        "readOnlyHint": true,
+        "destructiveHint": false,
+        "idempotentHint": false,
+        "openWorldHint": true
+      },
+      "execution": {
+        "taskSupport": "forbidden"
+      }
+    },
     {
       "name": "cost-usage",
       "title": "Cost & Usage",
@@ -4799,7 +4917,7 @@
     {
       "name": "create-scheduled-action",
       "title": "Create Scheduled Action",
-      "description": "Create a scheduled action in agent mode (default) or connection_sync mode. Agent mode asks an agent to follow the description and write a result into the target vault. connection_sync deterministically runs the approved read-only tools on bound service connections and ingests their data; it requires at least one connection to be bound before execution. Cadence 'once' runs a single time then completes permanently. Requires an active scheduling subscription and write access to the target vault.",
+      "description": "Create a Credit-metered scheduled action in agent mode (default) or connection_sync mode. Each execution has a 75-Credit base charge; agent model usage is added at 1.5 times OpenRouter's actual reported cost. Agent mode follows the description and writes a result into the target vault. connection_sync deterministically runs approved read-only tools on bound service connections and ingests their data. Cadence 'once' runs a single time then completes permanently. Requires write access to the target vault.",
       "inputSchema": {
         "type": "object",
         "properties": {
@@ -4884,7 +5002,7 @@
           },
           "code": {
             "type": "string",
-            "description": "Machine-readable denial code: not_enabled when no scheduling subscription is active."
+            "description": "Machine-readable denial code when creation is refused."
           }
         },
         "required": [
@@ -6749,6 +6867,16 @@
               ]
             },
             "description": "Per-page output formats: markdown, links, json, images are captured cheaply from HTML; branding (site-level logo/colors/fonts) requires a browser and adds time. Defaults to markdown+links."
+          },
+          "background": {
+            "type": "boolean",
+            "default": false,
+            "description": "Run the crawl as a background job instead of blocking this call, returning a jobId immediately — poll it with check_site_export to get a downloadable zip (all page content, plus real image files if downloadImages is set) once ready. Use for large sites where a synchronous call would be slow."
+          },
+          "downloadImages": {
+            "type": "boolean",
+            "default": false,
+            "description": "Download every discovered image as a real file into the export bundle (not just image URLs/stats). OFF by default — must be explicitly set true. Implies background regardless of the background flag, since downloading a whole site's images is too slow to run synchronously. Capped at 20 images/page and 500 images/site."
           }
         },
         "required": [
@@ -6765,7 +6893,8 @@
           },
           "pageCount": {
             "type": "integer",
-            "minimum": 0
+            "minimum": 0,
+            "description": "Absent when background is true — the crawl has not finished yet."
           },
           "pages": {
             "type": "array",
@@ -6794,11 +6923,13 @@
                 "schemaTypes"
               ],
               "additionalProperties": false
-            }
+            },
+            "description": "Absent when background is true — the crawl has not finished yet."
           },
           "durationMs": {
             "type": "number",
-            "minimum": 0
+            "minimum": 0,
+            "description": "Absent when background is true — the crawl has not finished yet."
           },
           "truncatedCount": {
             "type": "integer",
@@ -6828,13 +6959,25 @@
               "preview"
             ],
             "additionalProperties": false
+          },
+          "jobId": {
+            "type": "string",
+            "description": "Present when background (or downloadImages) was set — poll with check_site_export."
+          },
+          "status": {
+            "type": "string",
+            "enum": [
+              "pending"
+            ],
+            "description": "Present when background (or downloadImages) was set."
+          },
+          "statusUrl": {
+            "type": "string",
+            "description": "Present when background (or downloadImages) was set — informational; use check_site_export with jobId, not this URL directly."
           }
         },
         "required": [
-          "url",
-          "pageCount",
-          "pages",
-          "durationMs"
+          "url"
         ],
         "additionalProperties": false,
         "$schema": "http://json-schema.org/draft-07/schema#"
@@ -8249,7 +8392,7 @@
     {
       "name": "get-schedule-status",
       "title": "Get Schedule Status",
-      "description": "Get your own Scheduled Actions subscription status: whether it is active, your monthly quota, and how much you have used this period.",
+      "description": "Get the Credit-metered Scheduled Actions policy. Scheduling is included rather than sold as a separate subscription: each execution has a 75-Credit base charge, and agent model usage is billed at 1.5 times OpenRouter's actual reported cost.",
       "inputSchema": {
         "type": "object",
         "properties": {},
@@ -8264,6 +8407,16 @@
           "enabled": {
             "type": "boolean"
           },
+          "billingMode": {
+            "type": "string",
+            "const": "credits"
+          },
+          "runBaseCredits": {
+            "type": "number"
+          },
+          "llmCostMultiplier": {
+            "type": "number"
+          },
           "quotaPerPeriod": {
             "type": "number"
           },
@@ -16552,7 +16705,7 @@
     {
       "name": "set-schedule-entitlement",
       "title": "Set Schedule Entitlement",
-      "description": "Set whether an identity has an active Scheduled Actions subscription, its monthly execution quota, and (optionally) an mcp-scraper API key for scheduled actions to call mcp-scraper on their behalf. Called by mcp-scraper's Stripe webhook on subscribe/renew/cancel. Requires admin scope.",
+      "description": "Legacy migration and account-recovery control for old Scheduled Actions entitlements. New scheduling is Credit-metered and does not require a separate subscription. This admin-only tool may preserve a legacy API-key delegation or recover historical entitlement rows during migration.",
       "inputSchema": {
         "type": "object",
         "properties": {
@@ -16567,7 +16720,7 @@
           },
           "quotaPerPeriod": {
             "type": "number",
-            "description": "Monthly execution quota. Optional; defaults to 1000, or leaves the existing value unchanged if already set."
+            "description": "Legacy monthly execution quota retained only for historical-row compatibility; it does not price new Credit-metered runs."
           },
           "mcpScraperApiKey": {
             "type": "string",

package/docs/specs/meta-ad-creative-media-resolution-spec.md

@@ -2,20 +2,20 @@
 
 - **Request:** Replace the brittle manual path from a connected Meta ad-account ad to visible creative images, playable/transcribable video, and trustworthy analysis signals. Preserve the generic connected-service tools, but make the correct path directly discoverable and callable from the external MCP and scheduled Mastra agents.
 - **Lane:** MCP Build using Mastra, with a provider-native Nango read primitive and an external MCP composition tool.
-- **Capability:** `meta_ad_creative_media` for media buyers, ad directors, analysts, and agents. Given a tenant-owned Meta connection plus an `adId` or creative reference, return the resolved creative media, render bounded images as MCP image content, and optionally transcribe or start frame analysis for the video.
+- **Capability:** `meta_ad_creative_media` for media buyers, ad directors, analysts, and agents. Given a tenant-owned Meta connection plus an `adId`, return the resolved creative media, render bounded images as MCP image content, and provide exact explicit follow-up calls for video transcription or frame analysis when video media is present.
 - **Inputs:** This report; the MCP Scraper, scheduler, Nango integration, and SDK repositories; current Meta Graph API behavior; current MCP content/result specification; existing `facebook_page_intel`, `facebook_ad_transcribe`, `facebook_video_transcribe`, `video_frame_analysis`, `get-ad`, `get-ad-creative`, `get-ad-image`, and `get-ad-video` implementations.
 - **Missing Context:** None that blocks implementation. Default behavior is read-only resolution with image inclusion. Transcription and frame analysis are explicit opt-ins because they spend credits; ad mutations remain outside this tool.
 - **Evidence Plan:** Inspect current source and live schemas; verify Meta media fields against primary sources; verify MCP image content against the protocol/SDK; test a connected Meta ad when a safe fixture or user-owned ad ID is available; never run a write operation during verification.
 - **Target Runtime:** `mcp-scraper` hosted MCP and npm stdio package; `mcp-scraper-scheduler` on Vercel/Mastra; source-controlled Nango functions deployed to the Meta Marketing integration; generated Node/Python/CLI/cURL SDK contracts.
 - **Agents:** Existing scheduled-action Mastra agent receives the provider-native resolver through its per-run connection toolset. No new supervisor or model route is required; creative interpretation remains with the calling vision-capable model.
 - **Skills:** Existing MCP Scraper and Mastra operating guidance. No new workspace skill is required.
-- **Workflows:** Resolve ad and creative deterministically; collect bounded media descriptors; derive durable organic post/reel candidates; hydrate image/video metadata; attach safe image bytes; optionally call the appropriate transcription route or start the existing asynchronous video-frame analysis workflow.
+- **Workflows:** Resolve ad and creative deterministically; collect bounded media descriptors; derive organic post/reel candidates; hydrate image/video metadata; attach safe image bytes; recommend the appropriate explicit transcription route or existing asynchronous video-frame analysis workflow without spending those credits inside the read tool.
 - **Tools:**
   - Provider read: `resolve-ad-creative-media` in the Meta Nango integration.
   - External MCP composition: `meta_ad_creative_media`.
   - Existing lower-level tools remain available and cross-reference the new path.
 - **MCP:** `meta_ad_creative_media` returns text plus structured content. Successfully fetched creative images are additional MCP `image` content blocks with MIME type and base64 data. Videos remain URL/metadata plus transcript or asynchronous analysis receipt; raw video is not embedded in context.
-- **Data Shape:** Stable media descriptor with ad/creative IDs, account/page/post IDs, public post URL, media type, image assets, video assets, selected transcription route, transcript signal, warnings, and next actions. Image content indices map descriptors to MCP content blocks.
+- **Data Shape:** Stable media descriptor with ad/creative IDs, account/page/post IDs, unverified public-post candidate, media type, image assets, video assets, selected follow-up route, warnings, limitations, and next actions. Image content indices map descriptors to MCP content blocks. Transcript calls separately return their transcript signal.
 - **Signals:** Not required. Video frame analysis continues through its existing asynchronous status tool.
 - **Background Tasks:** Resolution and transcription are foreground bounded calls. Deep video analysis uses the existing asynchronous `video_frame_analysis` run/status path.
 - **Guardrails:** Tenant ownership and live provider policy; read-only provider function; explicit opt-in for credit-spending work; SSRF-safe HTTPS image fetching; Meta/CDN host allowlist; redirect validation; MIME allowlist; per-image and image-count limits; no tokens or signed query strings in logs; untrusted-provider-data labeling; action switch is not required for reads.