mcpscraper-sdk 0.5.0 → 0.7.0

package/README.md

@@ -40,13 +40,20 @@ Every non-2xx response throws a `ScraperApiError` with `status`, `code`, and the
 
 ## API surface
 
-`client.tools` is the generated, typed 145-tool MCP surface. It includes every scraper, browser, workflow, artifact, billing, and memory tool from `contracts/mcp.tools.json`.
+`client.tools` is the generated, typed 155-tool MCP surface. It includes every scraper, browser, workflow, artifact, billing, and memory tool from `contracts/mcp.tools.json`.
 
 ```ts
 await client.tools.search.searchSerp({ query: 'roof repair Denver' })
 await client.tools.memory.search({ query: 'roofing warranty terms' })
+await client.tools.connections.exportConnectedServiceData({
+  connectionId: 'conn_123',
+  dataset: 'emails',
+  lastDays: 7,
+})
 ```
 
+The connected-data export performs bounded provider pagination server-side and returns small results inline or a private seven-day JSONL artifact. Resume partial exports with the returned `continuation` object; renew an expired signed URL with `client.tools.connections.renewConnectedDataDownload({ artifactId })`.
+
 Core operations are flat on the client: `searchSerp`, `harvestPaa`, `extractUrl`, `mapSiteUrls`, `extractSite`, `auditSite`, `getExtractSiteStatus`, `listJobs`, `getJob`, `getHistory`, `getLedger`.
 
 Everything else is namespaced by product area, matching the OpenAPI spec's tags: `client.youtube`, `client.screenshot`, `client.facebook`, `client.googleAds`, `client.instagram`, `client.reddit`, `client.video`, `client.maps`, `client.directory`, `client.serpIntelligence`, `client.workflows`.
@@ -81,4 +88,4 @@ npm run generate
 
 ## See also
 
-[Repo README](../../README.md) (multi-language examples with real sample output) · [`mcpscraper-memory-sdk`](../memory) (Node, full 85-tool typed surface) · [`mcpscraper-sdk` Python source package](../scraper-python) · [`mcpscraper-cli`](../cli)
+[Repo README](../../README.md) (multi-language examples with real sample output) · [`mcpscraper-memory-sdk`](../memory) (Node, full 86-tool typed surface) · [`mcpscraper-sdk` Python source package](../scraper-python) · [`mcpscraper-cli`](../cli)

package/dist/index.d.cts

@@ -786,7 +786,7 @@ interface paths {
         put?: never;
         /**
          * Call one memory.mcpscraper.dev tool by name, using this account's mcpscraper.dev API key
-         * @description Generic dispatch: names one of the 85 tools documented in this repo's
+         * @description Generic dispatch: names one of the 86 tools documented in this repo's
          *     `contracts/memory.tools.json` and forwards `args` to it, using a memory identity
          *     auto-provisioned for the calling mcpscraper.dev account. The response shape depends
          *     entirely on which tool was called — see that tool's `outputSchema` in the manifest.
@@ -1035,13 +1035,123 @@ interface components {
             browserFallback: boolean;
             /** @default false */
             kernelFallback: boolean;
-            formats?: ("markdown" | "links" | "json" | "images" | "branding")[];
+            /**
+             * @description Output formats to produce. `issues` runs a site-wide SEO audit over the crawl and
+             *     returns `seoAudit` (in background mode, writes the `seo-audit.json` artifact).
+             *     `images` performs the image audit and returns `imageAudit` — on the synchronous
+             *     path too, not just background jobs.
+             */
+            formats?: ("markdown" | "links" | "json" | "images" | "branding" | "issues")[];
         };
         /** @description Full synchronous crawl result. */
         ExtractSiteResponse: {
-            pages?: {
+            pages?: ({
+                /** @description Whether this page's URL appears in the site's sitemap(s); null when no sitemap was found. */
+                inSitemap?: boolean | null;
+            } & {
                 [key: string]: unknown;
-            }[];
+            })[];
+            /** @description Crawl/URL-discovery metadata. */
+            spider?: {
+                /** @description Sitemap URLs discovered and parsed during the crawl. */
+                sitemapUrls?: string[];
+            } & {
+                [key: string]: unknown;
+            };
+            /** @description Site-wide SEO audit. Present (non-null) only when `formats` includes `issues`. */
+            seoAudit?: {
+                /**
+                 * @description Map of issue key → occurrence detail. The 31 issue keys: title.missing,
+                 *     title.duplicate, title.tooLong, title.tooShort, title.sameAsH1, meta.missing,
+                 *     meta.duplicate, meta.tooLong, meta.tooShort, h1.missing, h1.multiple,
+                 *     h1.duplicate, h1.tooLong, h2.missing, indexability.nonIndexable,
+                 *     indexability.noindex, canonical.missing, canonical.canonicalised,
+                 *     response.broken4xx, response.error5xx, response.redirect3xx,
+                 *     response.noResponse, content.thin, content.exactDuplicate, images.missingAlt,
+                 *     schema.missing, url.tooLong, url.uppercase, url.underscores, links.orphan,
+                 *     links.brokenInternal.
+                 */
+                issues?: {
+                    [key: string]: {
+                        count: number;
+                        urls: string[];
+                    };
+                };
+                linkSummary?: {
+                    internal?: {
+                        totalLinks?: number;
+                        pages?: number;
+                        orphans?: number;
+                        brokenInternal?: number;
+                        avgInlinks?: number;
+                        avgOutlinks?: number;
+                        distribution?: {
+                            zero?: number;
+                            oneToTwo?: number;
+                            threeToTen?: number;
+                            elevenPlus?: number;
+                        };
+                        topByInlinks?: {
+                            url?: string;
+                            inlinks?: number;
+                            outlinksInternal?: number;
+                            outlinksExternal?: number;
+                        }[];
+                    };
+                    external?: {
+                        totalLinks?: number;
+                        uniqueDomains?: number;
+                        topDomains?: {
+                            domain?: string;
+                            links?: number;
+                            nofollow?: number;
+                            pages?: number;
+                        }[];
+                    };
+                };
+                pageMetrics?: {
+                    url?: string;
+                    inboundInternal?: number;
+                    outboundInternal?: number;
+                    orphan?: boolean;
+                }[];
+            } | null;
+            /**
+             * @description Site-wide image audit. Present (non-null) only when `formats` includes `images` —
+             *     the synchronous path performs this audit too, not just background jobs.
+             */
+            imageAudit?: {
+                rows?: {
+                    url?: string;
+                    /** @description HTTP status returned when probing the image; null when the probe failed. */
+                    status?: number | null;
+                    bytes?: number | null;
+                    /** @description Human-readable size, e.g. "1.2 MB". */
+                    size?: string | null;
+                    contentType?: string | null;
+                    /** @description Detected image format; "unknown" when undetectable. */
+                    format?: string;
+                    over100kb?: boolean;
+                    legacyFormat?: boolean;
+                    /** @description Present when probing the image failed. */
+                    error?: string;
+                }[];
+                summary?: {
+                    unique?: number;
+                    /** @description How many rows have a known byte size. */
+                    sized?: number;
+                    totalBytes?: number;
+                    /** @description Human-readable total, e.g. "4.5 MB". */
+                    totalSize?: string;
+                    avgSize?: string;
+                    over100kb?: number;
+                    legacyFormat?: number;
+                    /** @description Map of format → image count. */
+                    formatCounts?: {
+                        [key: string]: number;
+                    };
+                };
+            } | null;
         } & {
             [key: string]: unknown;
         };
@@ -1057,6 +1167,11 @@ interface components {
             startUrl?: string;
             totalUrls?: number;
             doneUrls?: number;
+            /**
+             * @description Produced files (name + MIME type). When the job requested formats including
+             *     `issues`, this includes `seo-audit.json` (application/json — the same object as
+             *     the synchronous `seoAudit`), and rows in `pages.jsonl` include `inSitemap`.
+             */
             artifacts?: {
                 [key: string]: unknown;
             }[];
@@ -3386,9 +3501,87 @@ declare class ScraperClient {
         [key: string]: unknown;
     }>;
     extractSite(params: RequestBodyOf<'extractSite'>): Promise<({
-        pages?: {
+        pages?: ({
+            inSitemap?: boolean | null;
+        } & {
             [key: string]: unknown;
-        }[];
+        })[];
+        spider?: {
+            sitemapUrls?: string[];
+        } & {
+            [key: string]: unknown;
+        };
+        seoAudit?: {
+            issues?: {
+                [key: string]: {
+                    count: number;
+                    urls: string[];
+                };
+            };
+            linkSummary?: {
+                internal?: {
+                    totalLinks?: number;
+                    pages?: number;
+                    orphans?: number;
+                    brokenInternal?: number;
+                    avgInlinks?: number;
+                    avgOutlinks?: number;
+                    distribution?: {
+                        zero?: number;
+                        oneToTwo?: number;
+                        threeToTen?: number;
+                        elevenPlus?: number;
+                    };
+                    topByInlinks?: {
+                        url?: string;
+                        inlinks?: number;
+                        outlinksInternal?: number;
+                        outlinksExternal?: number;
+                    }[];
+                };
+                external?: {
+                    totalLinks?: number;
+                    uniqueDomains?: number;
+                    topDomains?: {
+                        domain?: string;
+                        links?: number;
+                        nofollow?: number;
+                        pages?: number;
+                    }[];
+                };
+            };
+            pageMetrics?: {
+                url?: string;
+                inboundInternal?: number;
+                outboundInternal?: number;
+                orphan?: boolean;
+            }[];
+        } | null;
+        imageAudit?: {
+            rows?: {
+                url?: string;
+                status?: number | null;
+                bytes?: number | null;
+                size?: string | null;
+                contentType?: string | null;
+                format?: string;
+                over100kb?: boolean;
+                legacyFormat?: boolean;
+                error?: string;
+            }[];
+            summary?: {
+                unique?: number;
+                sized?: number;
+                totalBytes?: number;
+                totalSize?: string;
+                avgSize?: string;
+                over100kb?: number;
+                legacyFormat?: number;
+                formatCounts?: {
+                    [key: string]: number;
+                };
+            };
+        } | null;
     } & {
         [key: string]: unknown;
     }) | {
@@ -3397,9 +3590,87 @@ declare class ScraperClient {
         statusUrl: string;
     }>;
     auditSite(params: RequestBodyOf<'extractSite'>): Promise<({
-        pages?: {
+        pages?: ({
+            inSitemap?: boolean | null;
+        } & {
             [key: string]: unknown;
-        }[];
+        })[];
+        spider?: {
+            sitemapUrls?: string[];
+        } & {
+            [key: string]: unknown;
+        };
+        seoAudit?: {
+            issues?: {
+                [key: string]: {
+                    count: number;
+                    urls: string[];
+                };
+            };
+            linkSummary?: {
+                internal?: {
+                    totalLinks?: number;
+                    pages?: number;
+                    orphans?: number;
+                    brokenInternal?: number;
+                    avgInlinks?: number;
+                    avgOutlinks?: number;
+                    distribution?: {
+                        zero?: number;
+                        oneToTwo?: number;
+                        threeToTen?: number;
+                        elevenPlus?: number;
+                    };
+                    topByInlinks?: {
+                        url?: string;
+                        inlinks?: number;
+                        outlinksInternal?: number;
+                        outlinksExternal?: number;
+                    }[];
+                };
+                external?: {
+                    totalLinks?: number;
+                    uniqueDomains?: number;
+                    topDomains?: {
+                        domain?: string;
+                        links?: number;
+                        nofollow?: number;
+                        pages?: number;
+                    }[];
+                };
+            };
+            pageMetrics?: {
+                url?: string;
+                inboundInternal?: number;
+                outboundInternal?: number;
+                orphan?: boolean;
+            }[];
+        } | null;
+        imageAudit?: {
+            rows?: {
+                url?: string;
+                status?: number | null;
+                bytes?: number | null;
+                size?: string | null;
+                contentType?: string | null;
+                format?: string;
+                over100kb?: boolean;
+                legacyFormat?: boolean;
+                error?: string;
+            }[];
+            summary?: {
+                unique?: number;
+                sized?: number;
+                totalBytes?: number;
+                totalSize?: string;
+                avgSize?: string;
+                over100kb?: number;
+                legacyFormat?: number;
+                formatCounts?: {
+                    [key: string]: number;
+                };
+            };
+        } | null;
     } & {
         [key: string]: unknown;
     }) | {

package/dist/index.d.ts

@@ -786,7 +786,7 @@ interface paths {
         put?: never;
         /**
          * Call one memory.mcpscraper.dev tool by name, using this account's mcpscraper.dev API key
-         * @description Generic dispatch: names one of the 85 tools documented in this repo's
+         * @description Generic dispatch: names one of the 86 tools documented in this repo's
          *     `contracts/memory.tools.json` and forwards `args` to it, using a memory identity
          *     auto-provisioned for the calling mcpscraper.dev account. The response shape depends
          *     entirely on which tool was called — see that tool's `outputSchema` in the manifest.
@@ -1035,13 +1035,123 @@ interface components {
             browserFallback: boolean;
             /** @default false */
             kernelFallback: boolean;
-            formats?: ("markdown" | "links" | "json" | "images" | "branding")[];
+            /**
+             * @description Output formats to produce. `issues` runs a site-wide SEO audit over the crawl and
+             *     returns `seoAudit` (in background mode, writes the `seo-audit.json` artifact).
+             *     `images` performs the image audit and returns `imageAudit` — on the synchronous
+             *     path too, not just background jobs.
+             */
+            formats?: ("markdown" | "links" | "json" | "images" | "branding" | "issues")[];
         };
         /** @description Full synchronous crawl result. */
         ExtractSiteResponse: {
-            pages?: {
+            pages?: ({
+                /** @description Whether this page's URL appears in the site's sitemap(s); null when no sitemap was found. */
+                inSitemap?: boolean | null;
+            } & {
                 [key: string]: unknown;
-            }[];
+            })[];
+            /** @description Crawl/URL-discovery metadata. */
+            spider?: {
+                /** @description Sitemap URLs discovered and parsed during the crawl. */
+                sitemapUrls?: string[];
+            } & {
+                [key: string]: unknown;
+            };
+            /** @description Site-wide SEO audit. Present (non-null) only when `formats` includes `issues`. */
+            seoAudit?: {
+                /**
+                 * @description Map of issue key → occurrence detail. The 31 issue keys: title.missing,
+                 *     title.duplicate, title.tooLong, title.tooShort, title.sameAsH1, meta.missing,
+                 *     meta.duplicate, meta.tooLong, meta.tooShort, h1.missing, h1.multiple,
+                 *     h1.duplicate, h1.tooLong, h2.missing, indexability.nonIndexable,
+                 *     indexability.noindex, canonical.missing, canonical.canonicalised,
+                 *     response.broken4xx, response.error5xx, response.redirect3xx,
+                 *     response.noResponse, content.thin, content.exactDuplicate, images.missingAlt,
+                 *     schema.missing, url.tooLong, url.uppercase, url.underscores, links.orphan,
+                 *     links.brokenInternal.
+                 */
+                issues?: {
+                    [key: string]: {
+                        count: number;
+                        urls: string[];
+                    };
+                };
+                linkSummary?: {
+                    internal?: {
+                        totalLinks?: number;
+                        pages?: number;
+                        orphans?: number;
+                        brokenInternal?: number;
+                        avgInlinks?: number;
+                        avgOutlinks?: number;
+                        distribution?: {
+                            zero?: number;
+                            oneToTwo?: number;
+                            threeToTen?: number;
+                            elevenPlus?: number;
+                        };
+                        topByInlinks?: {
+                            url?: string;
+                            inlinks?: number;
+                            outlinksInternal?: number;
+                            outlinksExternal?: number;
+                        }[];
+                    };
+                    external?: {
+                        totalLinks?: number;
+                        uniqueDomains?: number;
+                        topDomains?: {
+                            domain?: string;
+                            links?: number;
+                            nofollow?: number;
+                            pages?: number;
+                        }[];
+                    };
+                };
+                pageMetrics?: {
+                    url?: string;
+                    inboundInternal?: number;
+                    outboundInternal?: number;
+                    orphan?: boolean;
+                }[];
+            } | null;
+            /**
+             * @description Site-wide image audit. Present (non-null) only when `formats` includes `images` —
+             *     the synchronous path performs this audit too, not just background jobs.
+             */
+            imageAudit?: {
+                rows?: {
+                    url?: string;
+                    /** @description HTTP status returned when probing the image; null when the probe failed. */
+                    status?: number | null;
+                    bytes?: number | null;
+                    /** @description Human-readable size, e.g. "1.2 MB". */
+                    size?: string | null;
+                    contentType?: string | null;
+                    /** @description Detected image format; "unknown" when undetectable. */
+                    format?: string;
+                    over100kb?: boolean;
+                    legacyFormat?: boolean;
+                    /** @description Present when probing the image failed. */
+                    error?: string;
+                }[];
+                summary?: {
+                    unique?: number;
+                    /** @description How many rows have a known byte size. */
+                    sized?: number;
+                    totalBytes?: number;
+                    /** @description Human-readable total, e.g. "4.5 MB". */
+                    totalSize?: string;
+                    avgSize?: string;
+                    over100kb?: number;
+                    legacyFormat?: number;
+                    /** @description Map of format → image count. */
+                    formatCounts?: {
+                        [key: string]: number;
+                    };
+                };
+            } | null;
         } & {
             [key: string]: unknown;
         };
@@ -1057,6 +1167,11 @@ interface components {
             startUrl?: string;
             totalUrls?: number;
             doneUrls?: number;
+            /**
+             * @description Produced files (name + MIME type). When the job requested formats including
+             *     `issues`, this includes `seo-audit.json` (application/json — the same object as
+             *     the synchronous `seoAudit`), and rows in `pages.jsonl` include `inSitemap`.
+             */
             artifacts?: {
                 [key: string]: unknown;
             }[];
@@ -3386,9 +3501,87 @@ declare class ScraperClient {
         [key: string]: unknown;
     }>;
     extractSite(params: RequestBodyOf<'extractSite'>): Promise<({
-        pages?: {
+        pages?: ({
+            inSitemap?: boolean | null;
+        } & {
             [key: string]: unknown;
-        }[];
+        })[];
+        spider?: {
+            sitemapUrls?: string[];
+        } & {
+            [key: string]: unknown;
+        };
+        seoAudit?: {
+            issues?: {
+                [key: string]: {
+                    count: number;
+                    urls: string[];
+                };
+            };
+            linkSummary?: {
+                internal?: {
+                    totalLinks?: number;
+                    pages?: number;
+                    orphans?: number;
+                    brokenInternal?: number;
+                    avgInlinks?: number;
+                    avgOutlinks?: number;
+                    distribution?: {
+                        zero?: number;
+                        oneToTwo?: number;
+                        threeToTen?: number;
+                        elevenPlus?: number;
+                    };
+                    topByInlinks?: {
+                        url?: string;
+                        inlinks?: number;
+                        outlinksInternal?: number;
+                        outlinksExternal?: number;
+                    }[];
+                };
+                external?: {
+                    totalLinks?: number;
+                    uniqueDomains?: number;
+                    topDomains?: {
+                        domain?: string;
+                        links?: number;
+                        nofollow?: number;
+                        pages?: number;
+                    }[];
+                };
+            };
+            pageMetrics?: {
+                url?: string;
+                inboundInternal?: number;
+                outboundInternal?: number;
+                orphan?: boolean;
+            }[];
+        } | null;
+        imageAudit?: {
+            rows?: {
+                url?: string;
+                status?: number | null;
+                bytes?: number | null;
+                size?: string | null;
+                contentType?: string | null;
+                format?: string;
+                over100kb?: boolean;
+                legacyFormat?: boolean;
+                error?: string;
+            }[];
+            summary?: {
+                unique?: number;
+                sized?: number;
+                totalBytes?: number;
+                totalSize?: string;
+                avgSize?: string;
+                over100kb?: number;
+                legacyFormat?: number;
+                formatCounts?: {
+                    [key: string]: number;
+                };
+            };
+        } | null;
     } & {
         [key: string]: unknown;
     }) | {
@@ -3397,9 +3590,87 @@ declare class ScraperClient {
         statusUrl: string;
     }>;
     auditSite(params: RequestBodyOf<'extractSite'>): Promise<({
-        pages?: {
+        pages?: ({
+            inSitemap?: boolean | null;
+        } & {
             [key: string]: unknown;
-        }[];
+        })[];
+        spider?: {
+            sitemapUrls?: string[];
+        } & {
+            [key: string]: unknown;
+        };
+        seoAudit?: {
+            issues?: {
+                [key: string]: {
+                    count: number;
+                    urls: string[];
+                };
+            };
+            linkSummary?: {
+                internal?: {
+                    totalLinks?: number;
+                    pages?: number;
+                    orphans?: number;
+                    brokenInternal?: number;
+                    avgInlinks?: number;
+                    avgOutlinks?: number;
+                    distribution?: {
+                        zero?: number;
+                        oneToTwo?: number;
+                        threeToTen?: number;
+                        elevenPlus?: number;
+                    };
+                    topByInlinks?: {
+                        url?: string;
+                        inlinks?: number;
+                        outlinksInternal?: number;
+                        outlinksExternal?: number;
+                    }[];
+                };
+                external?: {
+                    totalLinks?: number;
+                    uniqueDomains?: number;
+                    topDomains?: {
+                        domain?: string;
+                        links?: number;
+                        nofollow?: number;
+                        pages?: number;
+                    }[];
+                };
+            };
+            pageMetrics?: {
+                url?: string;
+                inboundInternal?: number;
+                outboundInternal?: number;
+                orphan?: boolean;
+            }[];
+        } | null;
+        imageAudit?: {
+            rows?: {
+                url?: string;
+                status?: number | null;
+                bytes?: number | null;
+                size?: string | null;
+                contentType?: string | null;
+                format?: string;
+                over100kb?: boolean;
+                legacyFormat?: boolean;
+                error?: string;
+            }[];
+            summary?: {
+                unique?: number;
+                sized?: number;
+                totalBytes?: number;
+                totalSize?: string;
+                avgSize?: string;
+                over100kb?: number;
+                legacyFormat?: number;
+                formatCounts?: {
+                    [key: string]: number;
+                };
+            };
+        } | null;
     } & {
         [key: string]: unknown;
     }) | {

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "mcpscraper-sdk",
-  "version": "0.5.0",
-  "description": "Official Node.js client for all 145 mcpscraper.dev MCP tools and the REST API",
+  "version": "0.7.0",
+  "description": "Official Node.js client for all 155 mcpscraper.dev MCP tools and the REST API",
   "license": "MIT",
   "type": "module",
   "main": "./dist/index.cjs",
@@ -31,7 +31,7 @@
     "seo-api"
   ],
   "dependencies": {
-    "mcpscraper-memory-sdk": "^0.3.0"
+    "mcpscraper-memory-sdk": "^0.6.0"
   },
   "devDependencies": {
     "openapi-typescript": "^7.13.0"