mcpscraper-cli 0.26.0 → 0.28.0

package/README.md

@@ -4,7 +4,7 @@ Command-line interface for [mcpscraper.dev](https://mcpscraper.dev) and [memory.
 
 [Release history](https://github.com/VilovietaSEO/mcpscraper-sdk/blob/main/CHANGELOG.md)
 
-The CLI provides friendly shortcuts for common operations plus universal discovery and invocation for every one of the 249 unified MCP tools (148 MCP Scraper plus 101 mirrored Memory tools).
+The CLI provides friendly shortcuts for common operations plus universal discovery and invocation for every one of the 254 unified MCP tools (153 MCP Scraper plus 101 mirrored Memory tools).
 
 ## Install
 
@@ -37,7 +37,7 @@ mcpscraper tools call archive_read --args '{"url":"https://github.com/octocat/He
 mcpscraper tools call list_scheduled_runs --args '{"view":"inbox","limit":30}' --json
 ```
 
-`tools list` contains exactly 249 manifest-backed names. `tools call` accepts any one of them. Tools marked destructive require `--yes`.
+`tools list` contains exactly 254 manifest-backed names. `tools call` accepts any one of them. Tools marked destructive require `--yes`.
 
 Google SERP searches cost 60 Credits. PAA harvests cost 400 Credits plus 10 Credits per returned question. For additional throughput, one optional $5/month concurrency pack adds two browser slots; pack quantity scales both slots and monthly price linearly.
 
@@ -71,7 +71,16 @@ Options: `--screenshot`, `--deposit-to-vault`, `--vault <name>`, `--json`.
 
 ### `mcpscraper crawl <url>`
 
-Crawls and extracts every page of a site. Options: `--max-pages <n>` (default 100), `--json`.
+Starts a durable background export that retains complete JSON, acquired HTML, and Markdown for every successful page. Options: `--max-pages <n>` (default 100), `--preserve-media`, `--idempotency-key <key>`, `--json`.
+
+Poll and read the export without downloading and unpacking the ZIP:
+
+```bash
+mcpscraper export-status <job-id>
+mcpscraper export-read <job-id> --representation manifest
+mcpscraper export-read <job-id> --representation html --page-id <page-id> --output page.html
+mcpscraper export-image <job-id> <image-id> --output image.jpg
+```
 
 ### `mcpscraper map <url>`
 

package/dist/bin/cli.js

@@ -5,11 +5,12 @@ import { Command } from "commander";
 import { fileURLToPath } from "url";
 import { mkdirSync, realpathSync, writeFileSync } from "fs";
 import { homedir } from "os";
-import { join } from "path";
+import { dirname, join } from "path";
+import { randomUUID } from "crypto";
 import { ScraperClient, ScraperApiError } from "mcpscraper-sdk";
 
 // src/version.ts
-var CLI_VERSION = "0.26.0";
+var CLI_VERSION = "0.28.0";
 
 // src/generated-tools.ts
 var MCP_TOOL_CATALOG = [
@@ -58,6 +59,12 @@ var MCP_TOOL_CATALOG = [
             "mobile"
           ]
         },
+        "idempotencyKey": {
+          "description": "Retry key: reuse after a timeout to avoid re-billing. New key per harvest.",
+          "type": "string",
+          "minLength": 8,
+          "maxLength": 200
+        },
         "serpIdentity": {
           "description": "Optional persistent SERP identity created with serp_identity_create. Reuses the same saved browser state and fixed network address across calls.",
           "type": "string",
@@ -145,6 +152,12 @@ var MCP_TOOL_CATALOG = [
             "mobile"
           ]
         },
+        "idempotencyKey": {
+          "description": "Retry key: reuse after a timeout to avoid re-billing. New key per search.",
+          "type": "string",
+          "minLength": 8,
+          "maxLength": 200
+        },
         "serpIdentity": {
           "description": "Optional persistent SERP identity created with serp_identity_create. Reuses the same saved browser state and fixed network address across calls.",
           "type": "string",
@@ -215,7 +228,7 @@ var MCP_TOOL_CATALOG = [
     "name": "extract_url",
     "category": "web",
     "title": "Single URL Extract",
-    "description": "Extract structured data from one public URL: content, schema, headings, metadata, screenshots, branding, featured image, or media assets. Wayback replay URLs automatically return the archived page copy without playback chrome. Use delivery:auto for bounded inline results with automatic artifact offload, delivery:artifact for a durable owner-scoped report, or delivery:memory to save the full page into hosted MCP Memory. preserveMedia is the preferred media-retention flag; depositToVault and downloadMedia remain temporary compatibility aliases.",
+    "description": "Extract structured data from one public URL: content, schema, headings, metadata, screenshots, branding, featured image, or media assets. Tries a plain HTTP fetch first and only opens a stealth browser when that fetch is blocked, hits a bot check, or returns unusably thin content \u2014 most calls never need the browser, and a bot_check_unresolved error means the browser already tried and failed, not that a retry will help. Wayback replay URLs automatically return the archived page copy without playback chrome. Use delivery:auto for bounded inline results with automatic artifact offload, delivery:artifact for a durable owner-scoped report, or delivery:memory to save the full page into hosted MCP Memory. preserveMedia is the preferred media-retention flag; depositToVault and downloadMedia remain temporary compatibility aliases.",
     "inputSchema": {
       "type": "object",
       "properties": {
@@ -226,12 +239,12 @@ var MCP_TOOL_CATALOG = [
         },
         "screenshot": {
           "default": false,
-          "description": "Capture a full-page screenshot and return it through the result contract. Large captures may be offloaded to an owned artifact.",
+          "description": "Capture a full-page screenshot. Large captures may be offloaded to an owned artifact.",
           "type": "boolean"
         },
         "screenshotDevice": {
           "default": "desktop",
-          "description": "Viewport for screenshot. desktop = 1440\xD7900, mobile = 390\xD7844.",
+          "description": "Viewport: desktop = 1440\xD7900, mobile = 390\xD7844.",
           "type": "string",
           "enum": [
             "desktop",
@@ -240,16 +253,16 @@ var MCP_TOOL_CATALOG = [
         },
         "extractBranding": {
           "default": false,
-          "description": "Extract brand colors, fonts, logo, and favicon via a rendered browser session.",
+          "description": "Extract brand colors, fonts, logo, and favicon via a rendered session.",
           "type": "boolean"
         },
         "includeFeaturedImage": {
           "default": false,
-          "description": "Return the best featured image from Open Graph, Twitter, JSON-LD, or page content. For Wayback replay URLs, also returns the timestamp-matched archived image URL when available.",
+          "description": "Return the best featured image from Open Graph, Twitter, JSON-LD, or page content. Wayback URLs get the timestamp-matched archived image.",
           "type": "boolean"
         },
         "downloadMedia": {
-          "description": "Deprecated alias for preserveMedia. Omit when using preserveMedia; when omitted, media preservation defaults to false.",
+          "description": "Deprecated alias for preserveMedia; omit when using preserveMedia.",
           "type": "boolean"
         },
         "mediaTypes": {
@@ -258,7 +271,7 @@ var MCP_TOOL_CATALOG = [
             "video",
             "audio"
           ],
-          "description": "Which media types to download. Default all three.",
+          "description": "Media types to download. Default all three.",
           "type": "array",
           "items": {
             "type": "string",
@@ -271,21 +284,21 @@ var MCP_TOOL_CATALOG = [
         },
         "maxMediaAssets": {
           "default": 100,
-          "description": "Maximum media records to retain and attempt to download after filtering and responsive-variant collapse.",
+          "description": "Maximum media records to retain after filtering and variant collapse.",
           "type": "integer",
           "minimum": 1,
           "maximum": 250
         },
         "maxInlineImages": {
           "default": 3,
-          "description": "Maximum downloaded images to attach as AI-readable image content blocks. All successfully downloaded media remains available in the ZIP.",
+          "description": "Maximum images attached as AI-readable content blocks. All downloaded media stays available in the ZIP.",
           "type": "integer",
           "minimum": 0,
           "maximum": 5
         },
         "delivery": {
           "default": "auto",
-          "description": "Where to deliver the result. auto keeps small results inline and offloads large ones; artifact always returns an owned artifact; memory stores the full page in hosted Memory; inline returns a bounded response.",
+          "description": "auto offloads large results; artifact always returns an artifact; memory stores in hosted Memory; inline is bounded.",
           "type": "string",
           "enum": [
             "auto",
@@ -296,19 +309,25 @@ var MCP_TOOL_CATALOG = [
         },
         "preserveMedia": {
           "default": false,
-          "description": "Collect media from static source plus a rendered, lazy-loaded page; collapse responsive variants; return provenance and completeness; attach bounded image previews; and create an owner-scoped ZIP readable with archive_read.",
+          "description": "Collect media from static plus rendered page content, collapse responsive variants, attach bounded previews, and create an owner-scoped ZIP readable with archive_read.",
           "type": "boolean"
         },
         "depositToVault": {
           "default": false,
-          "description": "Save the full page content into the user's MCP Memory vault server-side, embedded for semantic recall \u2014 the full body is NOT returned to chat.",
+          "description": "Save the full page content into the user's MCP Memory vault, embedded for semantic recall \u2014 the full body is NOT returned to chat.",
           "type": "boolean"
         },
         "vaultName": {
-          "description": "Optional vault to deposit into. Defaults to the user's personal vault.",
+          "description": "Optional vault to deposit into; defaults to the personal vault.",
           "type": "string",
           "minLength": 1,
           "maxLength": 120
+        },
+        "idempotencyKey": {
+          "description": "Retry key: reuse after a timeout to avoid re-billing. New key per extraction.",
+          "type": "string",
+          "minLength": 8,
+          "maxLength": 200
         }
       },
       "required": [
@@ -495,7 +514,7 @@ var MCP_TOOL_CATALOG = [
     "name": "extract_site",
     "category": "web",
     "title": "Multi-Page Site Content Crawl",
-    "description": "Crawl a public website and return page CONTENT (Markdown) across multiple pages. A Wayback replay URL produces one archived site snapshot. The optional wayback plan produces whole-site, single-page, or selected-page timelines across explicit months or a month range, all in one export with a capture matrix. Pass a new idempotencyKey for each intended crawl and reuse it only when retrying that call. Every MCP crawl starts a durable export; poll check_site_export for honest outcome counters and the saved ZIP. Content only \u2014 for a technical SEO audit use audit_site instead.",
+    "description": "Crawl a public website and durably retain complete per-page JSON, acquired HTML, and Markdown. Each page is fetched with a plain HTTP request first; a browser retries only pages that were blocked or hit a bot check. A Wayback replay URL produces one archived site snapshot; the optional wayback plan produces timelines across explicit months or a month range. Pass a new idempotencyKey for each intended crawl and reuse it only when retrying that call. Poll check_site_export, then use site_export_read for complete page representations and site_export_image for preserved images; a saved ZIP is also produced. For a technical SEO audit use audit_site instead.",
     "inputSchema": {
       "type": "object",
       "properties": {
@@ -570,12 +589,13 @@ var MCP_TOOL_CATALOG = [
           "maximum": 100
         },
         "formats": {
-          "description": "Per-page output formats: markdown, links, json, images are captured cheaply from HTML; branding (site-level logo/colors/fonts) requires a browser and adds time. Defaults to markdown+links.",
+          "description": "Requested export views. Every successful page is durably retained as complete JSON, acquired HTML, and Markdown; links and image references are included in page JSON. branding adds a browser-based site summary.",
           "type": "array",
           "items": {
             "type": "string",
             "enum": [
               "markdown",
+              "html",
               "links",
               "json",
               "images",
@@ -585,7 +605,7 @@ var MCP_TOOL_CATALOG = [
         },
         "background": {
           "default": true,
-          "description": "MCP multi-page crawls always run as durable background jobs. Poll check_site_export for progress, outcome counters, and the hosted ZIP.",
+          "description": "MCP multi-page crawls always run as durable background jobs. Poll check_site_export, then use site_export_read for complete JSON/HTML/Markdown and site_export_image for downloaded images.",
           "type": "boolean",
           "const": true
         },
@@ -700,7 +720,7 @@ var MCP_TOOL_CATALOG = [
     "name": "check_site_export",
     "category": "web",
     "title": "Check Site Export",
-    "description": "Poll a background extract_site or audit_site job. Reports discovered, attempted, successful, failed, and remaining pages. Complete and partial jobs return a downloadable ZIP; partial bundles include successful content plus per-page failure reasons.",
+    "description": "Poll a background extract_site or audit_site job. Reports discovered, attempted, successful, failed, and remaining pages plus a stable public error envelope when terminal. Complete and partial jobs expose direct AI readback through site_export_read/site_export_image and also return a downloadable ZIP.",
     "inputSchema": {
       "type": "object",
       "properties": {
@@ -1760,10 +1780,19 @@ var MCP_TOOL_CATALOG = [
     "name": "commons_search_entities",
     "category": "commons",
     "title": "Transparent Commons Entity Search",
-    "description": "Search the Transparent Commons public wiki graph using the same fields the public /wiki/ frontend uses: title, description, tags, keywords, JSON-LD, article body, citations, media, source metadata, related entities, trails, and saved account filters. This reads published Commons entities only.",
+    "description": "Search the Transparent Commons public wiki graph. Hybrid by default: meaning-based candidates fused with exact matching over title, description, tags, keywords, JSON-LD, body, citations, media, source metadata, related entities, trails, and saved filters, ranked title-first. Set mode:lexical for a quoted exact name or phrase. Reads published Commons entities only; never a personal Memory vault.",
     "inputSchema": {
       "type": "object",
       "properties": {
+        "mode": {
+          "description": "Retrieval mode. hybrid (default) fuses meaning-based candidates with exact term matching; lexical matches terms only, which is what you want when the caller quoted an exact name or phrase; semantic ignores term matching entirely.",
+          "type": "string",
+          "enum": [
+            "lexical",
+            "semantic",
+            "hybrid"
+          ]
+        },
         "query": {
           "description": "Search text matched against title, description, tags, keywords, JSON-LD, source metadata, citations, media, and article body.",
           "type": "string",
@@ -3032,7 +3061,7 @@ var MCP_TOOL_CATALOG = [
     "name": "commons_submit_entity",
     "category": "commons",
     "title": "Transparent Commons Governed Entity Write",
-    "description": "Create or propose an edit to a Transparent Commons public wiki entity through the governed MCP Scraper write plane. A new entity costs 10 Credits; an edit to an existing entity costs 2 Credits. Preparation and validation are free, and idempotent retries are not charged twice. This never edits rendered HTML directly. Normal workflow is commons_prepare_entity, compose, commons_validate_entity, then submit. Choose a precise entityType and write a neutral encyclopedia projection with the matching entity structure, not a blog article or raw scrape dump. Use H2 sections and H3/H4/H5 subitems; every heading appears in the public page menu. Omit unsupported sections instead of publishing empty/filler headings. It writes a proposal, records a contribution ledger entry when accepted, and auto-publishes only safe non-conflicting changes with a featured image and source/body evidence. Existing entity edits require baseRevision to auto-publish. Requires idempotencyKey.",
+    "description": "Create or propose an edit to a Transparent Commons public wiki entity through the governed MCP Scraper write plane. A new entity costs 10 Credits; an edit to an existing entity costs 2 Credits. Preparation and validation are free, and idempotent retries are not charged twice. This never edits rendered HTML directly. Normal workflow is commons_prepare_entity, compose, commons_validate_entity, then submit. commons_prepare_entity returns the entity profile contract and section guidance to compose against. Transparent Commons is an open wiki: any account may edit any entity, and every edit publishes and is recorded on the public contribution ledger. Publishing requires a featured image and body/source evidence. Pass baseRevision when editing so the ledger can show whether the edit was written on top of a newer revision. Requires idempotencyKey.",
     "inputSchema": {
       "type": "object",
       "properties": {
@@ -3914,6 +3943,15 @@ var MCP_TOOL_CATALOG = [
         "filter": {
           "type": "object",
           "properties": {
+            "mode": {
+              "description": "Retrieval mode. hybrid (default) fuses meaning-based candidates with exact term matching; lexical matches terms only, which is what you want when the caller quoted an exact name or phrase; semantic ignores term matching entirely.",
+              "type": "string",
+              "enum": [
+                "lexical",
+                "semantic",
+                "hybrid"
+              ]
+            },
             "query": {
               "description": "Search text matched against title, description, tags, keywords, JSON-LD, source metadata, citations, media, and article body.",
               "type": "string",
@@ -4932,7 +4970,7 @@ var MCP_TOOL_CATALOG = [
     "name": "local_sourcebook_submission_status",
     "category": "directory",
     "title": "Local Sourcebook Submission Status",
-    "description": "Read the authenticated caller\u2019s listing draft, enrichment coverage, immutable revision number, publication state, and exact live LocalSourcebook.com profile and reviews URLs.",
+    "description": "Read the caller\u2019s listing draft, enrichment coverage, revision number, publication state, and live LocalSourcebook.com profile and reviews URLs. When status is failed, acquisitionError says why \u2014 read it before retrying, since a refresh repeats the same acquisition. Coverage counters are written only on success, so zeroed counters on a failed listing do not mean the crawl never ran.",
     "inputSchema": {
       "type": "object",
       "properties": {
@@ -4959,7 +4997,7 @@ var MCP_TOOL_CATALOG = [
     "name": "local_sourcebook_refresh",
     "category": "directory",
     "title": "Refresh a Local Sourcebook Listing",
-    "description": "Queue a new broad crawl and review/media acquisition pass for a listing owned by the authenticated MCP Scraper account. A refresh costs 2 Credits total, including acquisition; idempotent retries are not charged twice. The last published revision remains public until the refreshed evidence revision completes and auto-publishes.",
+    "description": "Queue a new broad crawl and review/media acquisition pass for a listing owned by the authenticated MCP Scraper account. A refresh costs 2 Credits total, including acquisition; idempotent retries are not charged twice, and a replayed retry still re-runs the acquisition, so a failed listing is always recoverable. Read local_sourcebook_submission_status first: a refresh repeats the same acquisition, so fix whatever acquisitionError reports before spending another pass. The last published revision remains public until the refreshed evidence revision completes and auto-publishes.",
     "inputSchema": {
       "type": "object",
       "properties": {
@@ -16698,10 +16736,14 @@ var MCP_TOOL_CATALOG = [
     "name": "commons_get_publication",
     "category": "commons",
     "title": "Get Transparent Commons Publication",
-    "description": "Read a Transparent Commons publication and its latest edition revisions. Omit subdomain to inspect the caller-owned publication; pass a name to inspect a public publication. Returns the permanent public and archive URLs needed for sharing or later edits.",
+    "description": "Read a Transparent Commons publication and its latest edition revisions. Omit subdomain to inspect the caller-owned publication; pass a name to inspect a public publication. Returns the permanent public and archive URLs needed for sharing or later edits. Set includeArticles to also return every stored article\u2019s Markdown source and slug, which is what commons_update_editorial_article edits \u2014 the payload is large, so request it only when you intend to edit.",
     "inputSchema": {
       "type": "object",
       "properties": {
+        "includeArticles": {
+          "description": "Return every stored article\u2019s Markdown source alongside each edition. Large payload; request it only when you intend to edit an article. Owner-only: a publication read by name never returns sources.",
+          "type": "boolean"
+        },
         "subdomain": {
           "description": "Public publication name to inspect. Omit to return the publication owned by the authenticated account.",
           "type": "string",
@@ -17616,6 +17658,259 @@ var MCP_TOOL_CATALOG = [
       "idempotentHint": false,
       "openWorldHint": true
     }
+  },
+  {
+    "name": "commons_get_proposal",
+    "category": "commons",
+    "title": "Transparent Commons Proposal Status",
+    "description": "Read one commons_submit_entity proposal owned by the calling account. Use it when submit returned a proposal that was held for review instead of published, to see the review reason and whether it has since been applied.",
+    "inputSchema": {
+      "type": "object",
+      "properties": {
+        "proposalId": {
+          "type": "string",
+          "minLength": 1,
+          "maxLength": 120,
+          "description": "Proposal id returned by commons_submit_entity, in the form commons-proposal-<uuid>."
+        }
+      },
+      "required": [
+        "proposalId"
+      ],
+      "$schema": "https://json-schema.org/draft/2020-12/schema"
+    },
+    "annotations": {
+      "title": "Transparent Commons Proposal Status",
+      "readOnlyHint": true,
+      "destructiveHint": false,
+      "idempotentHint": true,
+      "openWorldHint": false
+    }
+  },
+  {
+    "name": "commons_host_image",
+    "category": "commons",
+    "title": "Host a Transparent Commons Image",
+    "description": "Host an image from a public https URL or base64 bytes, returning a permanent Commons URL. Commons never serves a third-party image URL \u2014 commons_submit_entity hosts external images automatically, so use this only to host one before or independently of a write. JPEG, PNG, GIF, WebP up to 10 MB; SVG rejected. Identical bytes are stored once.",
+    "inputSchema": {
+      "type": "object",
+      "properties": {
+        "sourceUrl": {
+          "description": "Public https image URL to download and host. The wiki never serves a third-party URL directly, so pass the original source here rather than putting it on the entity.",
+          "type": "string",
+          "format": "uri"
+        },
+        "imageBase64": {
+          "description": "Base64 image bytes for an image you already hold. Use for images under about 3 MB; larger files should be published from a URL.",
+          "type": "string",
+          "minLength": 16
+        },
+        "alt": {
+          "description": "Alt text describing the image for readers who cannot see it.",
+          "type": "string",
+          "maxLength": 500
+        },
+        "license": {
+          "description": "License or usage terms of the original image, preserved with the hosted copy.",
+          "type": "string",
+          "maxLength": 240
+        },
+        "attribution": {
+          "description": "Credit line for the original photographer, publication, or archive.",
+          "type": "string",
+          "maxLength": 500
+        }
+      },
+      "$schema": "https://json-schema.org/draft/2020-12/schema"
+    },
+    "annotations": {
+      "title": "Host a Transparent Commons Image",
+      "readOnlyHint": false,
+      "destructiveHint": false,
+      "idempotentHint": true,
+      "openWorldHint": true
+    }
+  },
+  {
+    "name": "commons_update_editorial_article",
+    "category": "commons",
+    "title": "Edit One Transparent Commons Article",
+    "description": "Replace the Markdown of one article in a published edition without resupplying the others: it reads the stored sources, swaps the article you name, and publishes the next revision with the rest byte-identical. Read commons_get_publication with includeArticles first for the current source and revision. commons_publish_editorial is the full-replace path. Requires baseRevision and idempotencyKey.",
+    "inputSchema": {
+      "type": "object",
+      "properties": {
+        "publicationSubdomain": {
+          "type": "string",
+          "minLength": 3,
+          "maxLength": 50,
+          "description": "Publication owned by the authenticated account."
+        },
+        "editionSlug": {
+          "description": "Edition holding the article. Defaults to the publication\u2019s latest edition.",
+          "type": "string",
+          "maxLength": 80,
+          "pattern": "^[a-z0-9]+(?:-[a-z0-9]+)*$"
+        },
+        "articleSlug": {
+          "type": "string",
+          "maxLength": 80,
+          "pattern": "^[a-z0-9]+(?:-[a-z0-9]+)*$",
+          "description": "Slug of the one article to replace, from commons_get_publication with includeArticles."
+        },
+        "idempotencyKey": {
+          "type": "string",
+          "minLength": 8,
+          "maxLength": 200,
+          "description": "Unique key for this intended edit. Reuse it only when retrying the same edit."
+        },
+        "baseRevision": {
+          "type": "integer",
+          "exclusiveMinimum": 0,
+          "maximum": 9007199254740991,
+          "description": "Current edition revision from commons_get_publication. The edit is refused if the edition moved on."
+        },
+        "markdown": {
+          "description": "Replacement Markdown body for this article. Omit to change only the metadata fields below.",
+          "type": "string",
+          "minLength": 1,
+          "maxLength": 4e5
+        },
+        "title": {
+          "description": "Replacement article title.",
+          "type": "string",
+          "minLength": 1,
+          "maxLength": 180
+        },
+        "summary": {
+          "description": "Replacement one or two sentence summary.",
+          "type": "string",
+          "minLength": 1,
+          "maxLength": 500
+        },
+        "category": {
+          "description": "Replacement section label.",
+          "type": "string",
+          "minLength": 1,
+          "maxLength": 80
+        },
+        "kicker": {
+          "description": "Replacement framing line above the title.",
+          "type": "string",
+          "minLength": 1,
+          "maxLength": 140
+        },
+        "order": {
+          "description": "Replacement reading-order position.",
+          "type": "integer",
+          "minimum": 1,
+          "maximum": 1e3
+        }
+      },
+      "required": [
+        "publicationSubdomain",
+        "articleSlug",
+        "idempotencyKey",
+        "baseRevision"
+      ],
+      "$schema": "https://json-schema.org/draft/2020-12/schema"
+    },
+    "annotations": {
+      "title": "Edit One Transparent Commons Article",
+      "readOnlyHint": false,
+      "destructiveHint": false,
+      "idempotentHint": true,
+      "openWorldHint": true
+    }
+  },
+  {
+    "name": "site_export_image",
+    "category": "web",
+    "title": "Read Site Export Image",
+    "description": "Return one downloaded site-export image as a standard MCP image content block with source-page and source-URL provenance. Obtain imageId from site_export_read manifest.",
+    "inputSchema": {
+      "type": "object",
+      "properties": {
+        "jobId": {
+          "type": "string",
+          "minLength": 1,
+          "description": "Site export job ID returned by extract_site or audit_site."
+        },
+        "imageId": {
+          "type": "string",
+          "pattern": "^[a-f0-9]{64}$",
+          "description": "Downloaded image ID returned by a site_export_read manifest."
+        }
+      },
+      "required": [
+        "jobId",
+        "imageId"
+      ],
+      "$schema": "https://json-schema.org/draft/2020-12/schema"
+    },
+    "annotations": {
+      "title": "Read Site Export Image",
+      "readOnlyHint": true,
+      "destructiveHint": false,
+      "idempotentHint": false,
+      "openWorldHint": true
+    }
+  },
+  {
+    "name": "site_export_read",
+    "category": "web",
+    "title": "Read Site Export Page",
+    "description": "List a site export manifest or read complete JSON, HTML, or Markdown for one page in resumable UTF-8 windows. Uses owner-scoped durable content directly and does not load the final ZIP.",
+    "inputSchema": {
+      "type": "object",
+      "properties": {
+        "jobId": {
+          "type": "string",
+          "minLength": 1,
+          "description": "Site export job ID returned by extract_site or audit_site."
+        },
+        "pageId": {
+          "description": "Page ID returned by a manifest read. Omit to list the export manifest.",
+          "type": "string",
+          "pattern": "^[a-f0-9]{64}$"
+        },
+        "format": {
+          "default": "manifest",
+          "description": "manifest lists page/image IDs; JSON, HTML, and Markdown read one page representation.",
+          "type": "string",
+          "enum": [
+            "manifest",
+            "json",
+            "html",
+            "markdown"
+          ]
+        },
+        "offset": {
+          "default": 0,
+          "description": "UTF-8 byte offset. Continue from nextOffset until it is null.",
+          "type": "integer",
+          "minimum": 0,
+          "maximum": 9007199254740991
+        },
+        "maxBytes": {
+          "default": 64e3,
+          "description": "Maximum UTF-8 bytes returned in this window.",
+          "type": "integer",
+          "minimum": 1,
+          "maximum": 1e6
+        }
+      },
+      "required": [
+        "jobId"
+      ],
+      "$schema": "https://json-schema.org/draft/2020-12/schema"
+    },
+    "annotations": {
+      "title": "Read Site Export Page",
+      "readOnlyHint": true,
+      "destructiveHint": false,
+      "idempotentHint": false,
+      "openWorldHint": true
+    }
   }
 ];
 var MCP_TOOL_NAMES = MCP_TOOL_CATALOG.map((tool) => tool.name);
@@ -17812,21 +18107,68 @@ Vault deposit: ${page.memory.deposited ? `saved to ${page.memory.vault}` : "fail
       });
     });
   });
-  program.command("crawl <url>").description("Crawl and extract every page of a site").option("--max-pages <n>", "maximum pages to crawl", (v) => parseInt(v, 10)).option("--json", "print raw JSON").action(async (url, opts, cmd) => {
+  program.command("crawl <url>").description("Start a durable site export with complete per-page JSON, acquired HTML, and Markdown").option("--max-pages <n>", "maximum pages to crawl", (v) => parseInt(v, 10)).option("--idempotency-key <key>", "stable retry key; defaults to a new UUID for this intended crawl").option("--preserve-media", "download supported images into the export").option("--json", "print raw JSON").action(async (url, opts, cmd) => {
     await run(async () => {
       const result = await client(cmd).extractSite({
         url,
         maxPages: opts.maxPages ?? 100,
-        background: false,
+        background: true,
         downloadImages: false,
+        preserveMedia: Boolean(opts.preserveMedia),
         rotateProxies: false,
         rotateProxyEvery: 30,
-        browserFallback: false
-      });
+        browserFallback: false,
+        formats: ["json", "html", "markdown", "links"]
+      }, { idempotencyKey: opts.idempotencyKey ?? randomUUID() });
       printResult(result, Boolean(opts.json), (r) => {
-        const pages = r.pages ?? [];
-        console.log(`Crawled ${pages.length} page(s).`);
+        const job = r;
+        console.log(`Export ${job.status ?? "started"}: ${job.jobId ?? "(missing job id)"}`);
+        console.log(`Poll: mcpscraper export-status ${job.jobId ?? "<job-id>"}`);
+      });
+    });
+  });
+  program.command("export-status <job-id>").description("Poll a durable site export with progress, public errors, and artifact metadata").option("--json", "print raw JSON").action(async (jobId, opts, cmd) => {
+    await run(async () => {
+      const result = await client(cmd).getExtractSiteStatus(jobId);
+      printResult(result, Boolean(opts.json), (value) => {
+        const status = value;
+        console.log(`${status.status ?? "unknown"} \xB7 discovered ${status.discovered ?? 0} \xB7 attempted ${status.attempted ?? 0} \xB7 successful ${status.successful ?? 0} \xB7 failed ${status.failed ?? 0} \xB7 remaining ${status.remaining ?? 0}`);
+        if (status.error) console.log(status.error);
+        if (["complete", "partial", "failed"].includes(status.status ?? "")) console.log(`Manifest: mcpscraper export-read ${jobId}`);
+      });
+    });
+  });
+  program.command("export-read <job-id>").description("List an export manifest or read one complete JSON, HTML, or Markdown representation").option("--page-id <id>", "page ID returned by the manifest").option("--format <format>", "manifest, json, html, or markdown", "manifest").option("--offset <n>", "UTF-8 byte offset", (value) => parseInt(value, 10), 0).option("--max-bytes <n>", "maximum bytes in this window", (value) => parseInt(value, 10), 64e3).option("--output <path>", "write the returned text window to a file").option("--json", "print response metadata and text as JSON").action(async (jobId, opts, cmd) => {
+    await run(async () => {
+      if (!["manifest", "json", "html", "markdown"].includes(opts.format)) throw new Error("--format must be manifest, json, html, or markdown");
+      const result = await client(cmd).readExtractSiteExport({
+        jobId,
+        pageId: opts.pageId,
+        format: opts.format,
+        offset: opts.offset,
+        maxBytes: opts.maxBytes
       });
+      if (opts.output) writeFileSync(opts.output, result.text, { mode: 384 });
+      if (opts.json) console.log(JSON.stringify(result, null, 2));
+      else {
+        if (opts.output) console.log(`Saved ${Buffer.byteLength(result.text)} bytes to ${opts.output}`);
+        else process.stdout.write(result.text);
+        if (result.nextOffset != null) console.error(`
+Continue with --offset ${result.nextOffset}`);
+      }
+    });
+  });
+  program.command("export-image <job-id> <image-id>").description("Download one preserved site-export image by image ID").option("--output <path>", "output path; defaults under ~/Downloads/mcp-scraper").option("--json", "print image metadata after saving").action(async (jobId, imageId, opts, cmd) => {
+    await run(async () => {
+      const result = await client(cmd).readExtractSiteImage({ jobId, imageId });
+      const target = opts.output ?? join(homedir(), "Downloads", "mcp-scraper", `${imageId}.${mediaExtension(result.mimeType)}`);
+      mkdirSync(dirname(target), { recursive: true });
+      const bytes = decodeBase64(result.dataBase64);
+      writeFileSync(target, bytes, { mode: 384 });
+      const metadata = { ...result, dataBase64: void 0, savedPath: target };
+      if (opts.json) console.log(JSON.stringify(metadata, null, 2));
+      else console.log(`Saved ${bytes.length} bytes to ${target}
+Source: ${result.sourceUrl ?? "unknown"}`);
     });
   });
   program.command("map <url>").description("Discover all URLs on a site").option("--json", "print raw JSON").action(async (url, opts, cmd) => {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "mcpscraper-cli",
-  "version": "0.26.0",
+  "version": "0.28.0",
   "description": "Command-line interface for mcpscraper.dev and memory.mcpscraper.dev",
   "license": "MIT",
   "type": "module",
@@ -19,7 +19,7 @@
   },
   "dependencies": {
     "commander": "^12.0.0",
-    "mcpscraper-sdk": "^0.27.0"
+    "mcpscraper-sdk": "^0.29.0"
   },
   "devDependencies": {
     "@types/node": "^22.0.0"