mcpscraper-sdk 0.28.0 → 0.30.0

package/README.md

@@ -45,7 +45,7 @@ Current Google search pricing is 60 Credits per SERP search and 400 Credits plus
 
 ## API surface
 
-`client.tools` is the generated, typed 252-tool MCP surface. It includes 151 MCP Scraper tools and all 101 mirrored Memory tools from `contracts/mcp.tools.json`.
+`client.tools` is the generated, typed 259-tool MCP surface. It includes 158 MCP Scraper tools and all 101 mirrored Memory tools from `contracts/mcp.tools.json`.
 
 For multimodal results such as `meta_ad_creative_media`, call `client.tools.callToolResult(...)` to preserve native MCP image/audio/resource blocks. `callTool(...)` remains backward-compatible and returns the parsed structured or text value.
 
@@ -66,13 +66,28 @@ const inbox = await client.tools.schedule.listScheduledRuns({ view: 'inbox' })
 const templates = await client.tools.schedule.listArtifactTemplates({ status: 'active' })
 ```
 
+Lead enrichment is available through the same generated surface. Supply mapped rows directly, or call `client.tools.leads.import(...)` first for CSV/TSV/XLSX input:
+
+```ts
+const job = await client.tools.leads.enrich({
+  idempotencyKey: crypto.randomUUID(),
+  source: { kind: 'rows', rows: [{ Business: 'White Rock Roofing', City: 'Dallas', Website: 'https://roofwhiterock.com' }] },
+  columnMap: { name: 'Business', city: 'City', websiteUrl: 'Website' },
+  defaultEntityType: 'business',
+  emailSearchFallback: 'serp_snippets',
+  peopleDiscovery: 'owners',
+  peopleQueryTemplates: ['{business} owner of company {city}', '{business} founder {city}'],
+  outputFormats: ['csv', 'xlsx'],
+})
+```
+
 The connected-data export performs bounded Gmail, Calendar, Google Search Console, Zoom, Meta Marketing, or Resend pagination server-side and returns small results inline or a private seven-day JSONL artifact. Use `exportConnectedServiceData({ dataset: 'search_console_performance' })` for a fresh Search Console API extract. A scheduled `connection_sync` maintains a typed `gsc_performance_*` table exposed as `listServiceConnections().tableName`; use `exportSearchConsoleTableData` for a server-filtered download from that persisted table. Use `meta_ads_insights` for daily account, campaign, ad-set, and ad reporting across connected Meta ad accounts. Resend can aggregate sent/received mail, logs, contacts, broadcasts, and templates with `resend_data`. Resume partial exports with the returned `continuation` object; renew an expired signed URL with `client.tools.connections.renewConnectedDataDownload({ artifactId })`. Use `listServiceConnections` for verified grants and per-tool permission blockers, then `describeServiceConnectionTool` for the exact provider-native schema before calling through the generic connection bridges.
 
 Search Console also provides six API-only batches through those connection bridges: reads `inspect-urls` and `query-search-analytics-batch`, and gated actions `add-sites-batch`, `submit-sitemaps-batch`, `delete-sites-batch`, and `delete-sitemaps-batch`. They require no database, return per-item receipts, and can be bound to scheduled agent runs. Delete batches default to dry-run and require the live schema's exact confirmation token for execution; all actions require the connection's action switch.
 
 Integrations are included with an active Starter plan or higher: OAuth connect/reconnect and direct connected-service reads, approved actions, exports, and snapshots do not currently have an extra connection-operation debit. Scheduled Actions use the shared Credit balance at 75 Credits per started occurrence; agent-mode runs also add 1.5 times OpenRouter's actual reported model cost. Inspect the live policy with `await client.tools.schedule.getScheduleStatus()`.
 
-Core operations are flat on the client: `searchSerp`, `harvestPaa`, `extractUrl`, `mapSiteUrls`, `extractSite`, `auditSite`, `getExtractSiteStatus`, `listJobs`, `getJob`, `getHistory`, `getLedger`.
+Core operations are flat on the client: `searchSerp`, `harvestPaa`, `extractUrl`, `mapSiteUrls`, `extractSite`, `auditSite`, `getExtractSiteStatus`, `readExtractSiteExport`, `readExtractSiteImage`, `listJobs`, `getJob`, `getHistory`, `getLedger`.
 
 Everything else is namespaced by product area, matching the OpenAPI spec's tags: `client.youtube`, `client.screenshot`, `client.facebook`, `client.googleAds`, `client.instagram`, `client.reddit`, `client.video`, `client.maps`, `client.directory`, `client.serpIntelligence`, `client.workflows`.
 

package/dist/index.cjs

@@ -48,14 +48,31 @@ var PUBLIC_ERROR_CODES = /* @__PURE__ */ new Set([
   "proxy_unavailable",
   "harvest_timeout",
   "mcp_request_timeout",
+  "mcp_http_error",
   "extraction_failed",
   "concurrency_limit_exceeded",
   "unauthorized",
   "forbidden",
   "invalid_request",
   "rate_limited",
+  "upstream_rate_limited",
   "insufficient_balance",
-  "service_unavailable"
+  "service_unavailable",
+  "response_lost",
+  "vendor_unavailable",
+  "page_not_found",
+  "page_forbidden",
+  "page_rate_limited",
+  "page_server_error",
+  "page_http_error",
+  "bot_check_unresolved",
+  "page_too_large",
+  "page_unreachable",
+  "browser_session_interrupted",
+  "site_export_not_found",
+  "site_export_format_unavailable",
+  "site_export_read_failed",
+  "site_export_image_not_found"
 ]);
 var PUBLIC_ERROR_FIELDS = /* @__PURE__ */ new Set([
   "error",
@@ -153,19 +170,37 @@ var ScraperApiError = class extends Error {
 // src/client.ts
 var import_mcpscraper_memory_sdk = require("mcpscraper-memory-sdk");
 var Requester = class {
-  constructor(apiKey, baseUrl, fetchImpl) {
+  constructor(apiKey, baseUrl, fetchImpl, timeoutMs) {
     this.apiKey = apiKey;
     this.baseUrl = baseUrl;
     this.fetchImpl = fetchImpl;
+    this.timeoutMs = timeoutMs;
   }
   apiKey;
   baseUrl;
   fetchImpl;
+  timeoutMs;
+  async request(url, init) {
+    try {
+      return await this.fetchImpl(url, { ...init, signal: AbortSignal.timeout(this.timeoutMs) });
+    } catch (error) {
+      if (error instanceof DOMException && (error.name === "TimeoutError" || error.name === "AbortError")) {
+        throw new ScraperApiError(408, {
+          error: "mcp_request_timeout",
+          error_code: "mcp_request_timeout",
+          error_type: "timeout",
+          message: `MCP Scraper request exceeded ${Math.round(this.timeoutMs / 1e3)}s and was cancelled client-side. The operation may still be running; use the same Idempotency-Key or poll a returned job before starting a new paid run.`,
+          retryable: true
+        });
+      }
+      throw error;
+    }
+  }
   async call(method, path, body, extraHeaders = {}) {
     return (await this.callWithReceipt(method, path, body, extraHeaders)).data;
   }
   async callWithReceipt(method, path, body, extraHeaders = {}) {
-    const res = await this.fetchImpl(`${this.baseUrl}${path}`, {
+    const res = await this.request(`${this.baseUrl}${path}`, {
       method,
       headers: {
         "x-api-key": this.apiKey,
@@ -179,7 +214,7 @@ var Requester = class {
     return { data, headers: res.headers };
   }
   async callRaw(method, path) {
-    const res = await this.fetchImpl(`${this.baseUrl}${path}`, { method, headers: { "x-api-key": this.apiKey } });
+    const res = await this.request(`${this.baseUrl}${path}`, { method, headers: { "x-api-key": this.apiKey } });
     if (!res.ok) {
       const data = await res.json().catch(() => void 0);
       throw new ScraperApiError(res.status, data);
@@ -422,7 +457,9 @@ var ScraperClient = class {
   memoryTools;
   tools;
   constructor(options) {
-    this.r = new Requester(options.apiKey, options.baseUrl ?? "https://mcpscraper.dev", options.fetch ?? globalThis.fetch);
+    const requestTimeoutMs = options.requestTimeoutMs ?? 3e5;
+    if (!Number.isFinite(requestTimeoutMs) || requestTimeoutMs <= 0) throw new TypeError("requestTimeoutMs must be a positive number");
+    this.r = new Requester(options.apiKey, options.baseUrl ?? "https://mcpscraper.dev", options.fetch ?? globalThis.fetch, requestTimeoutMs);
     this.youtube = new YoutubeNamespace(this.r);
     this.screenshot = new ScreenshotNamespace(this.r);
     this.facebook = new FacebookNamespace(this.r);
@@ -438,7 +475,8 @@ var ScraperClient = class {
     this.tools = new import_mcpscraper_memory_sdk.McpToolsClient({
       apiKey: options.apiKey,
       baseUrl: options.baseUrl ?? "https://mcpscraper.dev",
-      fetch: options.fetch
+      fetch: options.fetch,
+      requestTimeoutMs
     });
   }
   async callMemoryTool(toolName, args) {
@@ -458,21 +496,27 @@ var ScraperClient = class {
   harvestPaa(params) {
     return this.r.call("POST", "/harvest/sync", params);
   }
-  extractUrl(params) {
-    return this.r.call("POST", "/extract-url", params);
+  extractUrl(params, options = {}) {
+    return this.r.call("POST", "/extract-url", params, options.idempotencyKey ? { "Idempotency-Key": options.idempotencyKey } : {});
   }
   mapSiteUrls(params) {
     return this.r.call("POST", "/map-urls", params);
   }
-  extractSite(params) {
-    return this.r.call("POST", "/extract-site", params);
+  extractSite(params, options = {}) {
+    return this.r.call("POST", "/extract-site", params, options.idempotencyKey ? { "Idempotency-Key": options.idempotencyKey } : {});
   }
-  auditSite(params) {
-    return this.extractSite(params);
+  auditSite(params, options = {}) {
+    return this.extractSite(params, options);
   }
   getExtractSiteStatus(id) {
     return this.r.call("GET", `/extract-site/status/${encodeURIComponent(id)}`);
   }
+  readExtractSiteExport(params) {
+    return this.r.call("POST", "/extract-site/read", params);
+  }
+  readExtractSiteImage(params) {
+    return this.r.call("POST", "/extract-site/image", params);
+  }
   listJobs() {
     return this.r.call("GET", "/jobs");
   }

package/dist/index.d.cts

@@ -178,6 +178,40 @@ interface paths {
         patch?: never;
         trace?: never;
     };
+    "/extract-site/read": {
+        parameters: {
+            query?: never;
+            header?: never;
+            path?: never;
+            cookie?: never;
+        };
+        get?: never;
+        put?: never;
+        /** List a site export or read complete page JSON, HTML, or Markdown */
+        post: operations["readExtractSiteExport"];
+        delete?: never;
+        options?: never;
+        head?: never;
+        patch?: never;
+        trace?: never;
+    };
+    "/extract-site/image": {
+        parameters: {
+            query?: never;
+            header?: never;
+            path?: never;
+            cookie?: never;
+        };
+        get?: never;
+        put?: never;
+        /** Read one owner-scoped downloaded image with provenance */
+        post: operations["readExtractSiteImage"];
+        delete?: never;
+        options?: never;
+        head?: never;
+        patch?: never;
+        trace?: never;
+    };
     "/youtube/harvest": {
         parameters: {
             query?: never;
@@ -844,7 +878,7 @@ interface components {
             details?: components["schemas"]["PublicErrorDetails"];
         };
         /** @enum {string} */
-        PublicChargeStatus: "not_charged" | "refunded" | "charged";
+        PublicChargeStatus: "not_charged" | "refund_pending" | "refunded" | "charged";
         /** @description Bounded, vendor-neutral recovery and account details safe for public clients. */
         PublicErrorDetails: {
             [key: string]: string | number | boolean | (string | null);
@@ -852,7 +886,7 @@ interface components {
         /** @description Stable public failure envelope shared across REST and MCP operations. */
         PublicErrorEnvelope: {
             /** @enum {string} */
-            error_code: "request_aborted" | "captcha_exhausted" | "captcha_or_blocked" | "location_mismatch" | "proxy_tunnel_failed" | "proxy_unavailable" | "harvest_timeout" | "mcp_request_timeout" | "extraction_failed" | "concurrency_limit_exceeded" | "unauthorized" | "forbidden" | "invalid_request" | "rate_limited" | "insufficient_balance" | "service_unavailable";
+            error_code: "request_aborted" | "captcha_exhausted" | "captcha_or_blocked" | "location_mismatch" | "proxy_tunnel_failed" | "proxy_unavailable" | "harvest_timeout" | "mcp_request_timeout" | "mcp_http_error" | "extraction_failed" | "concurrency_limit_exceeded" | "unauthorized" | "forbidden" | "invalid_request" | "rate_limited" | "upstream_rate_limited" | "insufficient_balance" | "service_unavailable" | "response_lost" | "vendor_unavailable" | "page_not_found" | "page_forbidden" | "page_rate_limited" | "page_server_error" | "page_http_error" | "bot_check_unresolved" | "page_too_large" | "page_unreachable" | "browser_session_interrupted" | "site_export_not_found" | "site_export_format_unavailable" | "site_export_read_failed" | "site_export_image_not_found";
             /** @enum {string} */
             error_type: "request_aborted" | "captcha" | "location_mismatch" | "connection" | "timeout" | "extraction" | "concurrency_limit" | "unauthorized" | "forbidden" | "invalid_request" | "rate_limit" | "billing" | "service_unavailable";
             message: string;
@@ -1042,9 +1076,19 @@ interface components {
                 [key: string]: unknown;
             } | null;
             memory?: components["schemas"]["VaultDepositResult"];
+            warnings?: components["schemas"]["PublicWarning"][];
         } & {
             [key: string]: unknown;
         };
+        /** @description A component-scoped optional-capability failure that does not invalidate the core extraction. */
+        PublicWarning: {
+            /** @enum {string} */
+            code: "render_unavailable" | "screenshot_unavailable" | "branding_unavailable" | "media_partial" | "media_packaging_failed";
+            /** @enum {string} */
+            component: "rendering" | "screenshot" | "branding" | "media";
+            message: string;
+            retryable: boolean;
+        };
         /**
          * @description Present on ExtractUrlResponse only when the request set depositToVault: true.
          *     Result of depositing a scrape into the caller's mcp-memory vault. Falls back to a
@@ -1104,6 +1148,11 @@ interface components {
              * @default false
              */
             downloadImages: boolean;
+            /**
+             * @description Preferred alias for downloadImages. Downloads supported images into owner-scoped artifacts and the ZIP.
+             * @default false
+             */
+            preserveMedia: boolean;
             /** @default false */
             rotateProxies: boolean;
             /** @default 30 */
@@ -1116,7 +1165,7 @@ interface components {
              *     `images` performs the image audit and returns `imageAudit` — on the synchronous
              *     path too, not just background jobs.
              */
-            formats?: ("markdown" | "links" | "json" | "images" | "branding" | "issues")[];
+            formats?: ("markdown" | "html" | "links" | "json" | "images" | "branding" | "issues")[];
         };
         /** @description Full synchronous crawl result. */
         ExtractSiteResponse: {
@@ -1254,8 +1303,52 @@ interface components {
                 [key: string]: unknown;
             }[];
             error?: string | null;
+            error_code?: string;
+            error_type?: string;
+            retryable?: boolean;
+            retry_after_seconds?: number;
+            charge_status?: components["schemas"]["PublicChargeStatus"];
             updatedAt?: string;
         };
+        SiteExportReadRequest: {
+            jobId: string;
+            pageId?: string;
+            /**
+             * @default manifest
+             * @enum {string}
+             */
+            format: "manifest" | "json" | "html" | "markdown";
+            /** @default 0 */
+            offset: number;
+            /** @default 64000 */
+            maxBytes: number;
+        };
+        SiteExportReadResponse: {
+            jobId: string;
+            pageId: string | null;
+            /** @enum {string} */
+            format: "manifest" | "json" | "html" | "markdown";
+            sha256: string;
+            text: string;
+            offset: number;
+            totalBytes: number;
+            nextOffset: number | null;
+        };
+        SiteExportImageRequest: {
+            jobId: string;
+            imageId: string;
+        };
+        SiteExportImageResponse: {
+            jobId: string;
+            imageId: string;
+            sourcePage?: string;
+            sourceUrl?: string;
+            mimeType: string;
+            bytes: number;
+            sha256?: string;
+            /** Format: byte */
+            dataBase64: string;
+        };
         YoutubeHarvestRequest: {
             /** @enum {string} */
             mode: "search" | "channel";
@@ -1878,6 +1971,60 @@ interface operations {
             404: components["responses"]["NotFound"];
         };
     };
+    readExtractSiteExport: {
+        parameters: {
+            query?: never;
+            header?: never;
+            path?: never;
+            cookie?: never;
+        };
+        requestBody: {
+            content: {
+                "application/json": components["schemas"]["SiteExportReadRequest"];
+            };
+        };
+        responses: {
+            /** @description One bounded UTF-8 window. Continue from nextOffset until null. */
+            200: {
+                headers: {
+                    [name: string]: unknown;
+                };
+                content: {
+                    "application/json": components["schemas"]["SiteExportReadResponse"];
+                };
+            };
+            400: components["responses"]["BadRequest"];
+            404: components["responses"]["NotFound"];
+            500: components["responses"]["ServerError"];
+        };
+    };
+    readExtractSiteImage: {
+        parameters: {
+            query?: never;
+            header?: never;
+            path?: never;
+            cookie?: never;
+        };
+        requestBody: {
+            content: {
+                "application/json": components["schemas"]["SiteExportImageRequest"];
+            };
+        };
+        responses: {
+            /** @description Downloaded image bytes encoded as base64 plus source provenance. */
+            200: {
+                headers: {
+                    [name: string]: unknown;
+                };
+                content: {
+                    "application/json": components["schemas"]["SiteExportImageResponse"];
+                };
+            };
+            400: components["responses"]["BadRequest"];
+            404: components["responses"]["NotFound"];
+            500: components["responses"]["ServerError"];
+        };
+    };
     youtubeHarvest: {
         parameters: {
             query?: never;
@@ -3337,6 +3484,11 @@ interface ScraperClientOptions {
     apiKey: string;
     baseUrl?: string;
     fetch?: typeof globalThis.fetch;
+    /** Bounded HTTP deadline for every request. Defaults to 300 seconds. */
+    requestTimeoutMs?: number;
+}
+interface IdempotentRequestOptions {
+    idempotencyKey?: string;
 }
 type SerpIntelligenceCaptureDefaultedKey = 'gl' | 'hl' | 'device' | 'proxyMode' | 'pages' | 'debug' | 'includePageSnapshots' | 'pageSnapshotLimit';
 type SerpIntelligenceCaptureParams = Omit<RequestBodyOf<'serpIntelligenceCapture'>, SerpIntelligenceCaptureDefaultedKey> & Partial<Pick<RequestBodyOf<'serpIntelligenceCapture'>, SerpIntelligenceCaptureDefaultedKey>>;
@@ -3351,7 +3503,9 @@ declare class Requester {
     private readonly apiKey;
     private readonly baseUrl;
     private readonly fetchImpl;
-    constructor(apiKey: string, baseUrl: string, fetchImpl: typeof globalThis.fetch);
+    private readonly timeoutMs;
+    constructor(apiKey: string, baseUrl: string, fetchImpl: typeof globalThis.fetch, timeoutMs: number);
+    private request;
     call<K extends OperationId>(method: string, path: string, body?: RequestBodyOf<K>, extraHeaders?: Record<string, string>): Promise<SuccessBodyOf<K>>;
     callWithReceipt<K extends OperationId>(method: string, path: string, body?: RequestBodyOf<K>, extraHeaders?: Record<string, string>): Promise<{
         data: SuccessBodyOf<K>;
@@ -3667,7 +3821,7 @@ declare class ScraperClient {
         };
         error?: string;
     }>;
-    extractUrl(params: RequestBodyOf<'extractUrl'>): Promise<{
+    extractUrl(params: RequestBodyOf<'extractUrl'>, options?: IdempotentRequestOptions): Promise<{
         title?: string | null;
         bodyMarkdown?: string;
         bodyHtml?: string;
@@ -3686,6 +3840,7 @@ declare class ScraperClient {
             [key: string]: unknown;
         } | null;
         memory?: components["schemas"]["VaultDepositResult"];
+        warnings?: components["schemas"]["PublicWarning"][];
     } & {
         [key: string]: unknown;
     }>;
@@ -3694,7 +3849,7 @@ declare class ScraperClient {
     } & {
         [key: string]: unknown;
     }>;
-    extractSite(params: RequestBodyOf<'extractSite'>): Promise<({
+    extractSite(params: RequestBodyOf<'extractSite'>, options?: IdempotentRequestOptions): Promise<({
         pages?: ({
             inSitemap?: boolean | null;
         } & {
@@ -3783,7 +3938,7 @@ declare class ScraperClient {
         status: "pending";
         statusUrl: string;
     }>;
-    auditSite(params: RequestBodyOf<'extractSite'>): Promise<({
+    auditSite(params: RequestBodyOf<'extractSite'>, options?: IdempotentRequestOptions): Promise<({
         pages?: ({
             inSitemap?: boolean | null;
         } & {
@@ -3882,8 +4037,33 @@ declare class ScraperClient {
             [key: string]: unknown;
         }[];
         error?: string | null;
+        error_code?: string;
+        error_type?: string;
+        retryable?: boolean;
+        retry_after_seconds?: number;
+        charge_status?: components["schemas"]["PublicChargeStatus"];
         updatedAt?: string;
     }>;
+    readExtractSiteExport(params: RequestBodyOf<'readExtractSiteExport'>): Promise<{
+        jobId: string;
+        pageId: string | null;
+        format: "manifest" | "json" | "html" | "markdown";
+        sha256: string;
+        text: string;
+        offset: number;
+        totalBytes: number;
+        nextOffset: number | null;
+    }>;
+    readExtractSiteImage(params: RequestBodyOf<'readExtractSiteImage'>): Promise<{
+        jobId: string;
+        imageId: string;
+        sourcePage?: string;
+        sourceUrl?: string;
+        mimeType: string;
+        bytes: number;
+        sha256?: string;
+        dataBase64: string;
+    }>;
     listJobs(): Promise<{
         id?: string;
         status?: string;
@@ -3938,4 +4118,4 @@ declare class ScraperApiError extends Error {
     };
 }
 
-export { type ConcurrencyAccountViewBody, type ConcurrencyLimitBody, type ConcurrencyPackBody, type InsufficientBalanceBody, type PublicErrorEnvelopeBody, ScraperApiError, ScraperClient, type ScraperClientOptions, type SerpIntelligenceCaptureOptions, type SerpIntelligenceCaptureParams, type SerpIntelligenceCaptureReceipt, type StructuredErrorBody, type components, type operations, type paths };
+export { type ConcurrencyAccountViewBody, type ConcurrencyLimitBody, type ConcurrencyPackBody, type IdempotentRequestOptions, type InsufficientBalanceBody, type PublicErrorEnvelopeBody, ScraperApiError, ScraperClient, type ScraperClientOptions, type SerpIntelligenceCaptureOptions, type SerpIntelligenceCaptureParams, type SerpIntelligenceCaptureReceipt, type StructuredErrorBody, type components, type operations, type paths };

package/dist/index.d.ts

@@ -178,6 +178,40 @@ interface paths {
         patch?: never;
         trace?: never;
     };
+    "/extract-site/read": {
+        parameters: {
+            query?: never;
+            header?: never;
+            path?: never;
+            cookie?: never;
+        };
+        get?: never;
+        put?: never;
+        /** List a site export or read complete page JSON, HTML, or Markdown */
+        post: operations["readExtractSiteExport"];
+        delete?: never;
+        options?: never;
+        head?: never;
+        patch?: never;
+        trace?: never;
+    };
+    "/extract-site/image": {
+        parameters: {
+            query?: never;
+            header?: never;
+            path?: never;
+            cookie?: never;
+        };
+        get?: never;
+        put?: never;
+        /** Read one owner-scoped downloaded image with provenance */
+        post: operations["readExtractSiteImage"];
+        delete?: never;
+        options?: never;
+        head?: never;
+        patch?: never;
+        trace?: never;
+    };
     "/youtube/harvest": {
         parameters: {
             query?: never;
@@ -844,7 +878,7 @@ interface components {
             details?: components["schemas"]["PublicErrorDetails"];
         };
         /** @enum {string} */
-        PublicChargeStatus: "not_charged" | "refunded" | "charged";
+        PublicChargeStatus: "not_charged" | "refund_pending" | "refunded" | "charged";
         /** @description Bounded, vendor-neutral recovery and account details safe for public clients. */
         PublicErrorDetails: {
             [key: string]: string | number | boolean | (string | null);
@@ -852,7 +886,7 @@ interface components {
         /** @description Stable public failure envelope shared across REST and MCP operations. */
         PublicErrorEnvelope: {
             /** @enum {string} */
-            error_code: "request_aborted" | "captcha_exhausted" | "captcha_or_blocked" | "location_mismatch" | "proxy_tunnel_failed" | "proxy_unavailable" | "harvest_timeout" | "mcp_request_timeout" | "extraction_failed" | "concurrency_limit_exceeded" | "unauthorized" | "forbidden" | "invalid_request" | "rate_limited" | "insufficient_balance" | "service_unavailable";
+            error_code: "request_aborted" | "captcha_exhausted" | "captcha_or_blocked" | "location_mismatch" | "proxy_tunnel_failed" | "proxy_unavailable" | "harvest_timeout" | "mcp_request_timeout" | "mcp_http_error" | "extraction_failed" | "concurrency_limit_exceeded" | "unauthorized" | "forbidden" | "invalid_request" | "rate_limited" | "upstream_rate_limited" | "insufficient_balance" | "service_unavailable" | "response_lost" | "vendor_unavailable" | "page_not_found" | "page_forbidden" | "page_rate_limited" | "page_server_error" | "page_http_error" | "bot_check_unresolved" | "page_too_large" | "page_unreachable" | "browser_session_interrupted" | "site_export_not_found" | "site_export_format_unavailable" | "site_export_read_failed" | "site_export_image_not_found";
             /** @enum {string} */
             error_type: "request_aborted" | "captcha" | "location_mismatch" | "connection" | "timeout" | "extraction" | "concurrency_limit" | "unauthorized" | "forbidden" | "invalid_request" | "rate_limit" | "billing" | "service_unavailable";
             message: string;
@@ -1042,9 +1076,19 @@ interface components {
                 [key: string]: unknown;
             } | null;
             memory?: components["schemas"]["VaultDepositResult"];
+            warnings?: components["schemas"]["PublicWarning"][];
         } & {
             [key: string]: unknown;
         };
+        /** @description A component-scoped optional-capability failure that does not invalidate the core extraction. */
+        PublicWarning: {
+            /** @enum {string} */
+            code: "render_unavailable" | "screenshot_unavailable" | "branding_unavailable" | "media_partial" | "media_packaging_failed";
+            /** @enum {string} */
+            component: "rendering" | "screenshot" | "branding" | "media";
+            message: string;
+            retryable: boolean;
+        };
         /**
          * @description Present on ExtractUrlResponse only when the request set depositToVault: true.
          *     Result of depositing a scrape into the caller's mcp-memory vault. Falls back to a
@@ -1104,6 +1148,11 @@ interface components {
              * @default false
              */
             downloadImages: boolean;
+            /**
+             * @description Preferred alias for downloadImages. Downloads supported images into owner-scoped artifacts and the ZIP.
+             * @default false
+             */
+            preserveMedia: boolean;
             /** @default false */
             rotateProxies: boolean;
             /** @default 30 */
@@ -1116,7 +1165,7 @@ interface components {
              *     `images` performs the image audit and returns `imageAudit` — on the synchronous
              *     path too, not just background jobs.
              */
-            formats?: ("markdown" | "links" | "json" | "images" | "branding" | "issues")[];
+            formats?: ("markdown" | "html" | "links" | "json" | "images" | "branding" | "issues")[];
         };
         /** @description Full synchronous crawl result. */
         ExtractSiteResponse: {
@@ -1254,8 +1303,52 @@ interface components {
                 [key: string]: unknown;
             }[];
             error?: string | null;
+            error_code?: string;
+            error_type?: string;
+            retryable?: boolean;
+            retry_after_seconds?: number;
+            charge_status?: components["schemas"]["PublicChargeStatus"];
             updatedAt?: string;
         };
+        SiteExportReadRequest: {
+            jobId: string;
+            pageId?: string;
+            /**
+             * @default manifest
+             * @enum {string}
+             */
+            format: "manifest" | "json" | "html" | "markdown";
+            /** @default 0 */
+            offset: number;
+            /** @default 64000 */
+            maxBytes: number;
+        };
+        SiteExportReadResponse: {
+            jobId: string;
+            pageId: string | null;
+            /** @enum {string} */
+            format: "manifest" | "json" | "html" | "markdown";
+            sha256: string;
+            text: string;
+            offset: number;
+            totalBytes: number;
+            nextOffset: number | null;
+        };
+        SiteExportImageRequest: {
+            jobId: string;
+            imageId: string;
+        };
+        SiteExportImageResponse: {
+            jobId: string;
+            imageId: string;
+            sourcePage?: string;
+            sourceUrl?: string;
+            mimeType: string;
+            bytes: number;
+            sha256?: string;
+            /** Format: byte */
+            dataBase64: string;
+        };
         YoutubeHarvestRequest: {
             /** @enum {string} */
             mode: "search" | "channel";
@@ -1878,6 +1971,60 @@ interface operations {
             404: components["responses"]["NotFound"];
         };
     };
+    readExtractSiteExport: {
+        parameters: {
+            query?: never;
+            header?: never;
+            path?: never;
+            cookie?: never;
+        };
+        requestBody: {
+            content: {
+                "application/json": components["schemas"]["SiteExportReadRequest"];
+            };
+        };
+        responses: {
+            /** @description One bounded UTF-8 window. Continue from nextOffset until null. */
+            200: {
+                headers: {
+                    [name: string]: unknown;
+                };
+                content: {
+                    "application/json": components["schemas"]["SiteExportReadResponse"];
+                };
+            };
+            400: components["responses"]["BadRequest"];
+            404: components["responses"]["NotFound"];
+            500: components["responses"]["ServerError"];
+        };
+    };
+    readExtractSiteImage: {
+        parameters: {
+            query?: never;
+            header?: never;
+            path?: never;
+            cookie?: never;
+        };
+        requestBody: {
+            content: {
+                "application/json": components["schemas"]["SiteExportImageRequest"];
+            };
+        };
+        responses: {
+            /** @description Downloaded image bytes encoded as base64 plus source provenance. */
+            200: {
+                headers: {
+                    [name: string]: unknown;
+                };
+                content: {
+                    "application/json": components["schemas"]["SiteExportImageResponse"];
+                };
+            };
+            400: components["responses"]["BadRequest"];
+            404: components["responses"]["NotFound"];
+            500: components["responses"]["ServerError"];
+        };
+    };
     youtubeHarvest: {
         parameters: {
             query?: never;
@@ -3337,6 +3484,11 @@ interface ScraperClientOptions {
     apiKey: string;
     baseUrl?: string;
     fetch?: typeof globalThis.fetch;
+    /** Bounded HTTP deadline for every request. Defaults to 300 seconds. */
+    requestTimeoutMs?: number;
+}
+interface IdempotentRequestOptions {
+    idempotencyKey?: string;
 }
 type SerpIntelligenceCaptureDefaultedKey = 'gl' | 'hl' | 'device' | 'proxyMode' | 'pages' | 'debug' | 'includePageSnapshots' | 'pageSnapshotLimit';
 type SerpIntelligenceCaptureParams = Omit<RequestBodyOf<'serpIntelligenceCapture'>, SerpIntelligenceCaptureDefaultedKey> & Partial<Pick<RequestBodyOf<'serpIntelligenceCapture'>, SerpIntelligenceCaptureDefaultedKey>>;
@@ -3351,7 +3503,9 @@ declare class Requester {
     private readonly apiKey;
     private readonly baseUrl;
     private readonly fetchImpl;
-    constructor(apiKey: string, baseUrl: string, fetchImpl: typeof globalThis.fetch);
+    private readonly timeoutMs;
+    constructor(apiKey: string, baseUrl: string, fetchImpl: typeof globalThis.fetch, timeoutMs: number);
+    private request;
     call<K extends OperationId>(method: string, path: string, body?: RequestBodyOf<K>, extraHeaders?: Record<string, string>): Promise<SuccessBodyOf<K>>;
     callWithReceipt<K extends OperationId>(method: string, path: string, body?: RequestBodyOf<K>, extraHeaders?: Record<string, string>): Promise<{
         data: SuccessBodyOf<K>;
@@ -3667,7 +3821,7 @@ declare class ScraperClient {
         };
         error?: string;
     }>;
-    extractUrl(params: RequestBodyOf<'extractUrl'>): Promise<{
+    extractUrl(params: RequestBodyOf<'extractUrl'>, options?: IdempotentRequestOptions): Promise<{
         title?: string | null;
         bodyMarkdown?: string;
         bodyHtml?: string;
@@ -3686,6 +3840,7 @@ declare class ScraperClient {
             [key: string]: unknown;
         } | null;
         memory?: components["schemas"]["VaultDepositResult"];
+        warnings?: components["schemas"]["PublicWarning"][];
     } & {
         [key: string]: unknown;
     }>;
@@ -3694,7 +3849,7 @@ declare class ScraperClient {
     } & {
         [key: string]: unknown;
     }>;
-    extractSite(params: RequestBodyOf<'extractSite'>): Promise<({
+    extractSite(params: RequestBodyOf<'extractSite'>, options?: IdempotentRequestOptions): Promise<({
         pages?: ({
             inSitemap?: boolean | null;
         } & {
@@ -3783,7 +3938,7 @@ declare class ScraperClient {
         status: "pending";
         statusUrl: string;
     }>;
-    auditSite(params: RequestBodyOf<'extractSite'>): Promise<({
+    auditSite(params: RequestBodyOf<'extractSite'>, options?: IdempotentRequestOptions): Promise<({
         pages?: ({
             inSitemap?: boolean | null;
         } & {
@@ -3882,8 +4037,33 @@ declare class ScraperClient {
             [key: string]: unknown;
         }[];
         error?: string | null;
+        error_code?: string;
+        error_type?: string;
+        retryable?: boolean;
+        retry_after_seconds?: number;
+        charge_status?: components["schemas"]["PublicChargeStatus"];
         updatedAt?: string;
     }>;
+    readExtractSiteExport(params: RequestBodyOf<'readExtractSiteExport'>): Promise<{
+        jobId: string;
+        pageId: string | null;
+        format: "manifest" | "json" | "html" | "markdown";
+        sha256: string;
+        text: string;
+        offset: number;
+        totalBytes: number;
+        nextOffset: number | null;
+    }>;
+    readExtractSiteImage(params: RequestBodyOf<'readExtractSiteImage'>): Promise<{
+        jobId: string;
+        imageId: string;
+        sourcePage?: string;
+        sourceUrl?: string;
+        mimeType: string;
+        bytes: number;
+        sha256?: string;
+        dataBase64: string;
+    }>;
     listJobs(): Promise<{
         id?: string;
         status?: string;
@@ -3938,4 +4118,4 @@ declare class ScraperApiError extends Error {
     };
 }
 
-export { type ConcurrencyAccountViewBody, type ConcurrencyLimitBody, type ConcurrencyPackBody, type InsufficientBalanceBody, type PublicErrorEnvelopeBody, ScraperApiError, ScraperClient, type ScraperClientOptions, type SerpIntelligenceCaptureOptions, type SerpIntelligenceCaptureParams, type SerpIntelligenceCaptureReceipt, type StructuredErrorBody, type components, type operations, type paths };
+export { type ConcurrencyAccountViewBody, type ConcurrencyLimitBody, type ConcurrencyPackBody, type IdempotentRequestOptions, type InsufficientBalanceBody, type PublicErrorEnvelopeBody, ScraperApiError, ScraperClient, type ScraperClientOptions, type SerpIntelligenceCaptureOptions, type SerpIntelligenceCaptureParams, type SerpIntelligenceCaptureReceipt, type StructuredErrorBody, type components, type operations, type paths };

package/dist/index.js

@@ -16,14 +16,31 @@ var PUBLIC_ERROR_CODES = /* @__PURE__ */ new Set([
   "proxy_unavailable",
   "harvest_timeout",
   "mcp_request_timeout",
+  "mcp_http_error",
   "extraction_failed",
   "concurrency_limit_exceeded",
   "unauthorized",
   "forbidden",
   "invalid_request",
   "rate_limited",
+  "upstream_rate_limited",
   "insufficient_balance",
-  "service_unavailable"
+  "service_unavailable",
+  "response_lost",
+  "vendor_unavailable",
+  "page_not_found",
+  "page_forbidden",
+  "page_rate_limited",
+  "page_server_error",
+  "page_http_error",
+  "bot_check_unresolved",
+  "page_too_large",
+  "page_unreachable",
+  "browser_session_interrupted",
+  "site_export_not_found",
+  "site_export_format_unavailable",
+  "site_export_read_failed",
+  "site_export_image_not_found"
 ]);
 var PUBLIC_ERROR_FIELDS = /* @__PURE__ */ new Set([
   "error",
@@ -138,19 +155,37 @@ import {
   McpToolsClient
 } from "mcpscraper-memory-sdk";
 var Requester = class {
-  constructor(apiKey, baseUrl, fetchImpl) {
+  constructor(apiKey, baseUrl, fetchImpl, timeoutMs) {
     this.apiKey = apiKey;
     this.baseUrl = baseUrl;
     this.fetchImpl = fetchImpl;
+    this.timeoutMs = timeoutMs;
   }
   apiKey;
   baseUrl;
   fetchImpl;
+  timeoutMs;
+  async request(url, init) {
+    try {
+      return await this.fetchImpl(url, { ...init, signal: AbortSignal.timeout(this.timeoutMs) });
+    } catch (error) {
+      if (error instanceof DOMException && (error.name === "TimeoutError" || error.name === "AbortError")) {
+        throw new ScraperApiError(408, {
+          error: "mcp_request_timeout",
+          error_code: "mcp_request_timeout",
+          error_type: "timeout",
+          message: `MCP Scraper request exceeded ${Math.round(this.timeoutMs / 1e3)}s and was cancelled client-side. The operation may still be running; use the same Idempotency-Key or poll a returned job before starting a new paid run.`,
+          retryable: true
+        });
+      }
+      throw error;
+    }
+  }
   async call(method, path, body, extraHeaders = {}) {
     return (await this.callWithReceipt(method, path, body, extraHeaders)).data;
   }
   async callWithReceipt(method, path, body, extraHeaders = {}) {
-    const res = await this.fetchImpl(`${this.baseUrl}${path}`, {
+    const res = await this.request(`${this.baseUrl}${path}`, {
       method,
       headers: {
         "x-api-key": this.apiKey,
@@ -164,7 +199,7 @@ var Requester = class {
     return { data, headers: res.headers };
   }
   async callRaw(method, path) {
-    const res = await this.fetchImpl(`${this.baseUrl}${path}`, { method, headers: { "x-api-key": this.apiKey } });
+    const res = await this.request(`${this.baseUrl}${path}`, { method, headers: { "x-api-key": this.apiKey } });
     if (!res.ok) {
       const data = await res.json().catch(() => void 0);
       throw new ScraperApiError(res.status, data);
@@ -407,7 +442,9 @@ var ScraperClient = class {
   memoryTools;
   tools;
   constructor(options) {
-    this.r = new Requester(options.apiKey, options.baseUrl ?? "https://mcpscraper.dev", options.fetch ?? globalThis.fetch);
+    const requestTimeoutMs = options.requestTimeoutMs ?? 3e5;
+    if (!Number.isFinite(requestTimeoutMs) || requestTimeoutMs <= 0) throw new TypeError("requestTimeoutMs must be a positive number");
+    this.r = new Requester(options.apiKey, options.baseUrl ?? "https://mcpscraper.dev", options.fetch ?? globalThis.fetch, requestTimeoutMs);
     this.youtube = new YoutubeNamespace(this.r);
     this.screenshot = new ScreenshotNamespace(this.r);
     this.facebook = new FacebookNamespace(this.r);
@@ -423,7 +460,8 @@ var ScraperClient = class {
     this.tools = new McpToolsClient({
       apiKey: options.apiKey,
       baseUrl: options.baseUrl ?? "https://mcpscraper.dev",
-      fetch: options.fetch
+      fetch: options.fetch,
+      requestTimeoutMs
     });
   }
   async callMemoryTool(toolName, args) {
@@ -443,21 +481,27 @@ var ScraperClient = class {
   harvestPaa(params) {
     return this.r.call("POST", "/harvest/sync", params);
   }
-  extractUrl(params) {
-    return this.r.call("POST", "/extract-url", params);
+  extractUrl(params, options = {}) {
+    return this.r.call("POST", "/extract-url", params, options.idempotencyKey ? { "Idempotency-Key": options.idempotencyKey } : {});
   }
   mapSiteUrls(params) {
     return this.r.call("POST", "/map-urls", params);
   }
-  extractSite(params) {
-    return this.r.call("POST", "/extract-site", params);
+  extractSite(params, options = {}) {
+    return this.r.call("POST", "/extract-site", params, options.idempotencyKey ? { "Idempotency-Key": options.idempotencyKey } : {});
   }
-  auditSite(params) {
-    return this.extractSite(params);
+  auditSite(params, options = {}) {
+    return this.extractSite(params, options);
   }
   getExtractSiteStatus(id) {
     return this.r.call("GET", `/extract-site/status/${encodeURIComponent(id)}`);
   }
+  readExtractSiteExport(params) {
+    return this.r.call("POST", "/extract-site/read", params);
+  }
+  readExtractSiteImage(params) {
+    return this.r.call("POST", "/extract-site/image", params);
+  }
   listJobs() {
     return this.r.call("GET", "/jobs");
   }

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "mcpscraper-sdk",
-  "version": "0.28.0",
-  "description": "Official Node.js client for all 252 mcpscraper.dev MCP tools and the REST API",
+  "version": "0.30.0",
+  "description": "Official Node.js client for all 259 mcpscraper.dev MCP tools and the REST API",
   "license": "MIT",
   "type": "module",
   "main": "./dist/index.cjs",
@@ -32,7 +32,7 @@
     "seo-api"
   ],
   "dependencies": {
-    "mcpscraper-memory-sdk": "^0.28.0"
+    "mcpscraper-memory-sdk": "^0.31.0"
   },
   "devDependencies": {
     "openapi-typescript": "^7.13.0"