mcpscraper-cli 0.27.0 → 0.28.0

package/README.md

@@ -4,7 +4,7 @@ Command-line interface for [mcpscraper.dev](https://mcpscraper.dev) and [memory.
 
 [Release history](https://github.com/VilovietaSEO/mcpscraper-sdk/blob/main/CHANGELOG.md)
 
-The CLI provides friendly shortcuts for common operations plus universal discovery and invocation for every one of the 252 unified MCP tools (151 MCP Scraper plus 101 mirrored Memory tools).
+The CLI provides friendly shortcuts for common operations plus universal discovery and invocation for every one of the 254 unified MCP tools (153 MCP Scraper plus 101 mirrored Memory tools).
 
 ## Install
 
@@ -37,7 +37,7 @@ mcpscraper tools call archive_read --args '{"url":"https://github.com/octocat/He
 mcpscraper tools call list_scheduled_runs --args '{"view":"inbox","limit":30}' --json
 ```
 
-`tools list` contains exactly 252 manifest-backed names. `tools call` accepts any one of them. Tools marked destructive require `--yes`.
+`tools list` contains exactly 254 manifest-backed names. `tools call` accepts any one of them. Tools marked destructive require `--yes`.
 
 Google SERP searches cost 60 Credits. PAA harvests cost 400 Credits plus 10 Credits per returned question. For additional throughput, one optional $5/month concurrency pack adds two browser slots; pack quantity scales both slots and monthly price linearly.
 
@@ -71,7 +71,16 @@ Options: `--screenshot`, `--deposit-to-vault`, `--vault <name>`, `--json`.
 
 ### `mcpscraper crawl <url>`
 
-Crawls and extracts every page of a site. Options: `--max-pages <n>` (default 100), `--json`.
+Starts a durable background export that retains complete JSON, acquired HTML, and Markdown for every successful page. Options: `--max-pages <n>` (default 100), `--preserve-media`, `--idempotency-key <key>`, `--json`.
+
+Poll and read the export without downloading and unpacking the ZIP:
+
+```bash
+mcpscraper export-status <job-id>
+mcpscraper export-read <job-id> --representation manifest
+mcpscraper export-read <job-id> --representation html --page-id <page-id> --output page.html
+mcpscraper export-image <job-id> <image-id> --output image.jpg
+```
 
 ### `mcpscraper map <url>`
 

package/dist/bin/cli.js

@@ -5,11 +5,12 @@ import { Command } from "commander";
 import { fileURLToPath } from "url";
 import { mkdirSync, realpathSync, writeFileSync } from "fs";
 import { homedir } from "os";
-import { join } from "path";
+import { dirname, join } from "path";
+import { randomUUID } from "crypto";
 import { ScraperClient, ScraperApiError } from "mcpscraper-sdk";
 
 // src/version.ts
-var CLI_VERSION = "0.27.0";
+var CLI_VERSION = "0.28.0";
 
 // src/generated-tools.ts
 var MCP_TOOL_CATALOG = [
@@ -513,7 +514,7 @@ var MCP_TOOL_CATALOG = [
     "name": "extract_site",
     "category": "web",
     "title": "Multi-Page Site Content Crawl",
-    "description": "Crawl a public website and return page CONTENT (Markdown) across multiple pages. Each page is fetched with a plain HTTP request first; a browser retries only the pages that were blocked or hit a bot check, so a per-page bot_check_unresolved result means that specific page already failed the browser retry too. A Wayback replay URL produces one archived site snapshot. The optional wayback plan produces whole-site, single-page, or selected-page timelines across explicit months or a month range, all in one export with a capture matrix. Pass a new idempotencyKey for each intended crawl and reuse it only when retrying that call. Every MCP crawl starts a durable export; poll check_site_export for honest outcome counters and the saved ZIP. Content only \u2014 for a technical SEO audit use audit_site instead.",
+    "description": "Crawl a public website and durably retain complete per-page JSON, acquired HTML, and Markdown. Each page is fetched with a plain HTTP request first; a browser retries only pages that were blocked or hit a bot check. A Wayback replay URL produces one archived site snapshot; the optional wayback plan produces timelines across explicit months or a month range. Pass a new idempotencyKey for each intended crawl and reuse it only when retrying that call. Poll check_site_export, then use site_export_read for complete page representations and site_export_image for preserved images; a saved ZIP is also produced. For a technical SEO audit use audit_site instead.",
     "inputSchema": {
       "type": "object",
       "properties": {
@@ -588,12 +589,13 @@ var MCP_TOOL_CATALOG = [
           "maximum": 100
         },
         "formats": {
-          "description": "Per-page output formats: markdown, links, json, images are captured cheaply from HTML; branding (site-level logo/colors/fonts) requires a browser and adds time. Defaults to markdown+links.",
+          "description": "Requested export views. Every successful page is durably retained as complete JSON, acquired HTML, and Markdown; links and image references are included in page JSON. branding adds a browser-based site summary.",
           "type": "array",
           "items": {
             "type": "string",
             "enum": [
               "markdown",
+              "html",
               "links",
               "json",
               "images",
@@ -603,7 +605,7 @@ var MCP_TOOL_CATALOG = [
         },
         "background": {
           "default": true,
-          "description": "MCP multi-page crawls always run as durable background jobs. Poll check_site_export for progress, outcome counters, and the hosted ZIP.",
+          "description": "MCP multi-page crawls always run as durable background jobs. Poll check_site_export, then use site_export_read for complete JSON/HTML/Markdown and site_export_image for downloaded images.",
           "type": "boolean",
           "const": true
         },
@@ -718,7 +720,7 @@ var MCP_TOOL_CATALOG = [
     "name": "check_site_export",
     "category": "web",
     "title": "Check Site Export",
-    "description": "Poll a background extract_site or audit_site job. Reports discovered, attempted, successful, failed, and remaining pages. Complete and partial jobs return a downloadable ZIP; partial bundles include successful content plus per-page failure reasons.",
+    "description": "Poll a background extract_site or audit_site job. Reports discovered, attempted, successful, failed, and remaining pages plus a stable public error envelope when terminal. Complete and partial jobs expose direct AI readback through site_export_read/site_export_image and also return a downloadable ZIP.",
     "inputSchema": {
       "type": "object",
       "properties": {
@@ -17819,6 +17821,96 @@ var MCP_TOOL_CATALOG = [
       "idempotentHint": true,
       "openWorldHint": true
     }
+  },
+  {
+    "name": "site_export_image",
+    "category": "web",
+    "title": "Read Site Export Image",
+    "description": "Return one downloaded site-export image as a standard MCP image content block with source-page and source-URL provenance. Obtain imageId from site_export_read manifest.",
+    "inputSchema": {
+      "type": "object",
+      "properties": {
+        "jobId": {
+          "type": "string",
+          "minLength": 1,
+          "description": "Site export job ID returned by extract_site or audit_site."
+        },
+        "imageId": {
+          "type": "string",
+          "pattern": "^[a-f0-9]{64}$",
+          "description": "Downloaded image ID returned by a site_export_read manifest."
+        }
+      },
+      "required": [
+        "jobId",
+        "imageId"
+      ],
+      "$schema": "https://json-schema.org/draft/2020-12/schema"
+    },
+    "annotations": {
+      "title": "Read Site Export Image",
+      "readOnlyHint": true,
+      "destructiveHint": false,
+      "idempotentHint": false,
+      "openWorldHint": true
+    }
+  },
+  {
+    "name": "site_export_read",
+    "category": "web",
+    "title": "Read Site Export Page",
+    "description": "List a site export manifest or read complete JSON, HTML, or Markdown for one page in resumable UTF-8 windows. Uses owner-scoped durable content directly and does not load the final ZIP.",
+    "inputSchema": {
+      "type": "object",
+      "properties": {
+        "jobId": {
+          "type": "string",
+          "minLength": 1,
+          "description": "Site export job ID returned by extract_site or audit_site."
+        },
+        "pageId": {
+          "description": "Page ID returned by a manifest read. Omit to list the export manifest.",
+          "type": "string",
+          "pattern": "^[a-f0-9]{64}$"
+        },
+        "format": {
+          "default": "manifest",
+          "description": "manifest lists page/image IDs; JSON, HTML, and Markdown read one page representation.",
+          "type": "string",
+          "enum": [
+            "manifest",
+            "json",
+            "html",
+            "markdown"
+          ]
+        },
+        "offset": {
+          "default": 0,
+          "description": "UTF-8 byte offset. Continue from nextOffset until it is null.",
+          "type": "integer",
+          "minimum": 0,
+          "maximum": 9007199254740991
+        },
+        "maxBytes": {
+          "default": 64e3,
+          "description": "Maximum UTF-8 bytes returned in this window.",
+          "type": "integer",
+          "minimum": 1,
+          "maximum": 1e6
+        }
+      },
+      "required": [
+        "jobId"
+      ],
+      "$schema": "https://json-schema.org/draft/2020-12/schema"
+    },
+    "annotations": {
+      "title": "Read Site Export Page",
+      "readOnlyHint": true,
+      "destructiveHint": false,
+      "idempotentHint": false,
+      "openWorldHint": true
+    }
   }
 ];
 var MCP_TOOL_NAMES = MCP_TOOL_CATALOG.map((tool) => tool.name);
@@ -18015,21 +18107,68 @@ Vault deposit: ${page.memory.deposited ? `saved to ${page.memory.vault}` : "fail
       });
     });
   });
-  program.command("crawl <url>").description("Crawl and extract every page of a site").option("--max-pages <n>", "maximum pages to crawl", (v) => parseInt(v, 10)).option("--json", "print raw JSON").action(async (url, opts, cmd) => {
+  program.command("crawl <url>").description("Start a durable site export with complete per-page JSON, acquired HTML, and Markdown").option("--max-pages <n>", "maximum pages to crawl", (v) => parseInt(v, 10)).option("--idempotency-key <key>", "stable retry key; defaults to a new UUID for this intended crawl").option("--preserve-media", "download supported images into the export").option("--json", "print raw JSON").action(async (url, opts, cmd) => {
     await run(async () => {
       const result = await client(cmd).extractSite({
         url,
         maxPages: opts.maxPages ?? 100,
-        background: false,
+        background: true,
         downloadImages: false,
+        preserveMedia: Boolean(opts.preserveMedia),
         rotateProxies: false,
         rotateProxyEvery: 30,
-        browserFallback: false
-      });
+        browserFallback: false,
+        formats: ["json", "html", "markdown", "links"]
+      }, { idempotencyKey: opts.idempotencyKey ?? randomUUID() });
       printResult(result, Boolean(opts.json), (r) => {
-        const pages = r.pages ?? [];
-        console.log(`Crawled ${pages.length} page(s).`);
+        const job = r;
+        console.log(`Export ${job.status ?? "started"}: ${job.jobId ?? "(missing job id)"}`);
+        console.log(`Poll: mcpscraper export-status ${job.jobId ?? "<job-id>"}`);
+      });
+    });
+  });
+  program.command("export-status <job-id>").description("Poll a durable site export with progress, public errors, and artifact metadata").option("--json", "print raw JSON").action(async (jobId, opts, cmd) => {
+    await run(async () => {
+      const result = await client(cmd).getExtractSiteStatus(jobId);
+      printResult(result, Boolean(opts.json), (value) => {
+        const status = value;
+        console.log(`${status.status ?? "unknown"} \xB7 discovered ${status.discovered ?? 0} \xB7 attempted ${status.attempted ?? 0} \xB7 successful ${status.successful ?? 0} \xB7 failed ${status.failed ?? 0} \xB7 remaining ${status.remaining ?? 0}`);
+        if (status.error) console.log(status.error);
+        if (["complete", "partial", "failed"].includes(status.status ?? "")) console.log(`Manifest: mcpscraper export-read ${jobId}`);
+      });
+    });
+  });
+  program.command("export-read <job-id>").description("List an export manifest or read one complete JSON, HTML, or Markdown representation").option("--page-id <id>", "page ID returned by the manifest").option("--format <format>", "manifest, json, html, or markdown", "manifest").option("--offset <n>", "UTF-8 byte offset", (value) => parseInt(value, 10), 0).option("--max-bytes <n>", "maximum bytes in this window", (value) => parseInt(value, 10), 64e3).option("--output <path>", "write the returned text window to a file").option("--json", "print response metadata and text as JSON").action(async (jobId, opts, cmd) => {
+    await run(async () => {
+      if (!["manifest", "json", "html", "markdown"].includes(opts.format)) throw new Error("--format must be manifest, json, html, or markdown");
+      const result = await client(cmd).readExtractSiteExport({
+        jobId,
+        pageId: opts.pageId,
+        format: opts.format,
+        offset: opts.offset,
+        maxBytes: opts.maxBytes
       });
+      if (opts.output) writeFileSync(opts.output, result.text, { mode: 384 });
+      if (opts.json) console.log(JSON.stringify(result, null, 2));
+      else {
+        if (opts.output) console.log(`Saved ${Buffer.byteLength(result.text)} bytes to ${opts.output}`);
+        else process.stdout.write(result.text);
+        if (result.nextOffset != null) console.error(`
+Continue with --offset ${result.nextOffset}`);
+      }
+    });
+  });
+  program.command("export-image <job-id> <image-id>").description("Download one preserved site-export image by image ID").option("--output <path>", "output path; defaults under ~/Downloads/mcp-scraper").option("--json", "print image metadata after saving").action(async (jobId, imageId, opts, cmd) => {
+    await run(async () => {
+      const result = await client(cmd).readExtractSiteImage({ jobId, imageId });
+      const target = opts.output ?? join(homedir(), "Downloads", "mcp-scraper", `${imageId}.${mediaExtension(result.mimeType)}`);
+      mkdirSync(dirname(target), { recursive: true });
+      const bytes = decodeBase64(result.dataBase64);
+      writeFileSync(target, bytes, { mode: 384 });
+      const metadata = { ...result, dataBase64: void 0, savedPath: target };
+      if (opts.json) console.log(JSON.stringify(metadata, null, 2));
+      else console.log(`Saved ${bytes.length} bytes to ${target}
+Source: ${result.sourceUrl ?? "unknown"}`);
     });
   });
   program.command("map <url>").description("Discover all URLs on a site").option("--json", "print raw JSON").action(async (url, opts, cmd) => {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "mcpscraper-cli",
-  "version": "0.27.0",
+  "version": "0.28.0",
   "description": "Command-line interface for mcpscraper.dev and memory.mcpscraper.dev",
   "license": "MIT",
   "type": "module",
@@ -19,7 +19,7 @@
   },
   "dependencies": {
     "commander": "^12.0.0",
-    "mcpscraper-sdk": "^0.27.0"
+    "mcpscraper-sdk": "^0.29.0"
   },
   "devDependencies": {
     "@types/node": "^22.0.0"