mcp-scraper 0.80.0 → 0.83.0

package/dist/{site-extract-repository-VDONDDMR.js → site-extract-repository-Y5WQFSQW.js}

@@ -1,4 +1,5 @@
 import {
+  XRAY_ENTITLED_EXTRACT_BILLING_CLASS,
   abandonExtractSettlement,
   claimFailedExtractJobForRefinalize,
   completeExtractJob,
@@ -9,6 +10,7 @@ import {
   failExtractJob,
   failStaleRunningExtractJob,
   failUnfundedExtractJob,
+  finalizeXRayEntitledExtractJobCharge,
   finishExtractJob,
   getExtractJob,
   getExtractJobByIdempotencyKey,
@@ -28,11 +30,12 @@ import {
   setExtractJobTotal,
   settleExtractJob,
   terminalExtractJobStatus
-} from "./chunk-XBEUZJHS.js";
+} from "./chunk-HGJJQY24.js";
 import "./chunk-DNM65UCK.js";
-import "./chunk-PMONKQGJ.js";
+import "./chunk-4NZ6JFK5.js";
 import "./chunk-YXNDOQXN.js";
 export {
+  XRAY_ENTITLED_EXTRACT_BILLING_CLASS,
   abandonExtractSettlement,
   claimFailedExtractJobForRefinalize,
   completeExtractJob,
@@ -43,6 +46,7 @@ export {
   failExtractJob,
   failStaleRunningExtractJob,
   failUnfundedExtractJob,
+  finalizeXRayEntitledExtractJobCharge,
   finishExtractJob,
   getExtractJob,
   getExtractJobByIdempotencyKey,

package/dist/{worker-ICCZR443.js → worker-FM3ZRDY4.js}

@@ -9,7 +9,7 @@ import {
 import "./chunk-OZJMVCDK.js";
 import {
   harvest
-} from "./chunk-KNGTQD6U.js";
+} from "./chunk-NH4QAH3X.js";
 import {
   sanitizeAttempts,
   sanitizeHarvestResult
@@ -17,13 +17,13 @@ import {
 import {
   MC_COSTS,
   serpActualCostMc
-} from "./chunk-PMONKQGJ.js";
+} from "./chunk-4NZ6JFK5.js";
 import {
   browserServiceApiKey
 } from "./chunk-OM7HVEJ3.js";
 import {
   runWithCostContext
-} from "./chunk-5UAFBKJR.js";
+} from "./chunk-OPQIGAFB.js";
 import "./chunk-OWF2JJKN.js";
 import {
   claimPendingJob,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "mcp-scraper",
-  "version": "0.80.0",
+  "version": "0.83.0",
   "description": "MCP server for MCP Scraper web intelligence tools",
   "type": "module",
   "main": "./dist/index.cjs",
@@ -30,6 +30,8 @@
     "dist/*.d.cts",
     "dist/bin/*.js",
     "dist/editorial-reading-room/assets",
+    "public/agent-install.md",
+    "public/.well-known/xray-install.json",
     "README.md",
     "CHANGELOG.md"
   ],
@@ -57,6 +59,8 @@
     "mcp:measure-catalog": "node scripts/run-ts-sandbox-safe.mjs scripts/measure-mcp-catalog.ts",
     "mcp:lint-tools": "node scripts/run-ts-sandbox-safe.mjs scripts/lint-mcp-tool-craft.ts",
     "mcp:lint-tools:write": "node scripts/run-ts-sandbox-safe.mjs scripts/lint-mcp-tool-craft.ts --write",
+    "xray:install-contract": "node --import tsx scripts/generate-xray-install-contract.ts",
+    "xray:install-contract:check": "node --import tsx scripts/generate-xray-install-contract.ts --check",
     "api": "node scripts/run-ts-sandbox-safe.mjs bin/api-server.ts",
     "dev": "node scripts/run-ts-sandbox-safe.mjs src/cli.ts",
     "test": "vitest run tests/unit tests/contract",
@@ -97,7 +101,7 @@
     "assistant:deployment:preflight": "node --import tsx scripts/verify-personal-assistant-deployment-config.ts",
     "analytics:rollups": "node --import tsx scripts/run-analytics-rollups.ts",
     "test:mcp-output-contract": "vitest run --config vitest.mcp-output-contract.config.ts tests/unit/golden-payloads.test.ts tests/unit/mcp-formatter-structured.test.ts tests/unit/mcp-2026-stdio.test.ts",
-    "mcp:verify": "npm run build && npm run mcp:manifest && npm run build:rates && npm run mcp:measure-catalog && npm run mcp:lint-tools && npm run test:mcp-output-contract && npm run typecheck && npm test && npm run package:verify-entrypoints",
+    "mcp:verify": "npm run xray:install-contract:check && npm run build && npm run mcp:manifest && npm run build:rates && npm run mcp:measure-catalog && npm run mcp:lint-tools && npm run test:mcp-output-contract && npm run typecheck && npm test && npm run package:verify-entrypoints",
     "prepack": "npm run mcp:verify",
     "package:verify-entrypoints": "node scripts/verify-package-entrypoints.mjs",
     "product:install": "node scripts/run-product-command.mjs install",

package/public/.well-known/xray-install.json

@@ -0,0 +1,178 @@
+{
+  "schema": "https://mcpscraper.dev/schemas/xray-install-contract-v1.json",
+  "protocolVersion": "2.0.0",
+  "product": "Thorbit X-Ray",
+  "triggerPhrases": [
+    "Install X-Ray",
+    "Install X-Ray on this site and tell me what drives customers"
+  ],
+  "canonicalUrls": {
+    "markdown": "https://mcpscraper.dev/agent-install.md",
+    "json": "https://mcpscraper.dev/.well-known/xray-install.json",
+    "hostedMcp": "https://mcpscraper.dev/mcp"
+  },
+  "facade": {
+    "tool": "analytics_setup_site",
+    "initialCall": {
+      "startUrl": "<site URL>",
+      "goal": "Install X-Ray and tell me what drives customers"
+    },
+    "continuationRule": "Treat setupId plus revision as the durable continuation handle. Reuse both after approval, runtime publication, verification, retry, or return."
+  },
+  "topologies": {
+    "local_stdio": {
+      "label": "Local npm MCP process",
+      "authority": "The client starts MCP Scraper locally. After the one-time Pixel insertion, X-Ray setup publishes only owner-confirmed runtime configuration.",
+      "secretHandling": "Set MCP_SCRAPER_API_KEY in the local environment or client secret field. Never paste it into chat or commit it."
+    },
+    "hosted_remote": {
+      "label": "Hosted authenticated MCP",
+      "authority": "The hosted MCP inspects approved public pages and publishes owner-confirmed X-Ray runtime configuration. It cannot read or write a caller-local path.",
+      "secretHandling": "Use the MCP OAuth/connection flow. Never put a credential in a URL, prompt, or config shown in chat."
+    }
+  },
+  "hosts": [
+    {
+      "id": "codex",
+      "label": "Codex",
+      "supportedTopologies": [
+        "local_stdio",
+        "hosted_remote"
+      ],
+      "mcpConfigMode": "toml",
+      "workspaceRead": true,
+      "workspaceWrite": true,
+      "restartRequired": true,
+      "exactConfig": "[mcp_servers.mcp-scraper]\ncommand = \"npx\"\nargs = [\"-y\", \"--package\", \"mcp-scraper@latest\", \"mcp-scraper\"]\nenv_vars = [\"MCP_SCRAPER_API_KEY\"]",
+      "continuation": "Call analytics_setup_site with { \"startUrl\": \"<site URL>\", \"goal\": \"Install X-Ray and tell me what drives customers\" }; on every later call, reuse the returned setupId and revision.",
+      "rollback": "Remove the mcp_servers.mcp-scraper table, then restart Codex. Pixel removal is a separate owner-controlled site action."
+    },
+    {
+      "id": "claude-code",
+      "label": "Claude Code",
+      "supportedTopologies": [
+        "local_stdio",
+        "hosted_remote"
+      ],
+      "mcpConfigMode": "command",
+      "workspaceRead": true,
+      "workspaceWrite": true,
+      "restartRequired": true,
+      "exactConfig": "claude mcp add mcp-scraper --scope user --env MCP_SCRAPER_API_KEY=\"$MCP_SCRAPER_API_KEY\" -- npx -y --package mcp-scraper@latest mcp-scraper",
+      "continuation": "Call analytics_setup_site with { \"startUrl\": \"<site URL>\", \"goal\": \"Install X-Ray and tell me what drives customers\" }; on every later call, reuse the returned setupId and revision.",
+      "rollback": "Run: claude mcp remove mcp-scraper -s user. Pixel removal is a separate owner-controlled site action."
+    },
+    {
+      "id": "claude-desktop",
+      "label": "Claude Desktop",
+      "supportedTopologies": [
+        "local_stdio"
+      ],
+      "mcpConfigMode": "desktop_extension",
+      "workspaceRead": false,
+      "workspaceWrite": false,
+      "restartRequired": true,
+      "exactConfig": "Install https://mcpscraper.dev/downloads/mcp-scraper.mcpb and enter the API key only in Claude Desktop's protected configuration field.",
+      "continuation": "Call analytics_setup_site with { \"startUrl\": \"<site URL>\", \"goal\": \"Install X-Ray and tell me what drives customers\" }; on every later call, reuse the returned setupId and revision.",
+      "rollback": "Remove the MCP Scraper extension in Claude Desktop. Pixel removal is a separate owner-controlled site action."
+    },
+    {
+      "id": "cursor",
+      "label": "Cursor",
+      "supportedTopologies": [
+        "local_stdio",
+        "hosted_remote"
+      ],
+      "mcpConfigMode": "json",
+      "workspaceRead": true,
+      "workspaceWrite": true,
+      "restartRequired": true,
+      "exactConfig": "{\n  \"mcpServers\": {\n    \"mcp-scraper\": {\n      \"command\": \"npx\",\n      \"args\": [\n        \"-y\",\n        \"--package\",\n        \"mcp-scraper@latest\",\n        \"mcp-scraper\"\n      ],\n      \"env\": {\n        \"MCP_SCRAPER_API_KEY\": \"${env:MCP_SCRAPER_API_KEY}\"\n      }\n    }\n  }\n}",
+      "continuation": "Call analytics_setup_site with { \"startUrl\": \"<site URL>\", \"goal\": \"Install X-Ray and tell me what drives customers\" }; on every later call, reuse the returned setupId and revision.",
+      "rollback": "Remove mcp-scraper from .cursor/mcp.json (or the user MCP settings), then restart Cursor."
+    },
+    {
+      "id": "windsurf",
+      "label": "Windsurf",
+      "supportedTopologies": [
+        "local_stdio"
+      ],
+      "mcpConfigMode": "json",
+      "workspaceRead": true,
+      "workspaceWrite": true,
+      "restartRequired": true,
+      "exactConfig": "{\n  \"mcpServers\": {\n    \"mcp-scraper\": {\n      \"command\": \"npx\",\n      \"args\": [\n        \"-y\",\n        \"--package\",\n        \"mcp-scraper@latest\",\n        \"mcp-scraper\"\n      ],\n      \"env\": {\n        \"MCP_SCRAPER_API_KEY\": \"${env:MCP_SCRAPER_API_KEY}\"\n      }\n    }\n  }\n}",
+      "continuation": "Call analytics_setup_site with { \"startUrl\": \"<site URL>\", \"goal\": \"Install X-Ray and tell me what drives customers\" }; on every later call, reuse the returned setupId and revision.",
+      "rollback": "Remove mcp-scraper from the Windsurf MCP configuration, then restart Windsurf."
+    },
+    {
+      "id": "cline",
+      "label": "Cline",
+      "supportedTopologies": [
+        "local_stdio"
+      ],
+      "mcpConfigMode": "json",
+      "workspaceRead": true,
+      "workspaceWrite": true,
+      "restartRequired": true,
+      "exactConfig": "{\n  \"mcpServers\": {\n    \"mcp-scraper\": {\n      \"command\": \"npx\",\n      \"args\": [\n        \"-y\",\n        \"--package\",\n        \"mcp-scraper@latest\",\n        \"mcp-scraper\"\n      ],\n      \"env\": {\n        \"MCP_SCRAPER_API_KEY\": \"${env:MCP_SCRAPER_API_KEY}\"\n      }\n    }\n  }\n}",
+      "continuation": "Call analytics_setup_site with { \"startUrl\": \"<site URL>\", \"goal\": \"Install X-Ray and tell me what drives customers\" }; on every later call, reuse the returned setupId and revision.",
+      "rollback": "Remove mcp-scraper from Cline MCP Servers, then restart the extension host."
+    },
+    {
+      "id": "user-action-only",
+      "label": "User-action-only MCP host",
+      "supportedTopologies": [
+        "hosted_remote"
+      ],
+      "mcpConfigMode": "handoff_only",
+      "workspaceRead": false,
+      "workspaceWrite": false,
+      "restartRequired": false,
+      "exactConfig": "{\n  \"mcpServers\": {\n    \"mcp-scraper\": {\n      \"url\": \"https://mcpscraper.dev/mcp\"\n    }\n  }\n}",
+      "continuation": "Call analytics_setup_site with { \"startUrl\": \"<site URL>\", \"goal\": \"Install X-Ray and tell me what drives customers\" }; on every later call, reuse the returned setupId and revision.",
+      "rollback": "Disconnect MCP Scraper from the host. Pixel removal remains an owner-controlled site action."
+    }
+  ],
+  "tagBootstrap": {
+    "requirement": "one_global_pixel_once",
+    "ongoingConfiguration": "xray_runtime_manifest",
+    "sourceAccessRequiredAfterInstall": false,
+    "supportedInsertionPaths": [
+      "manual",
+      "tag_manager",
+      "cms_plugin",
+      "coding_host"
+    ]
+  },
+  "workflow": [
+    "Resolve Thorbit X-Ray entitlement and the target site without consuming MCP Scraper Credits.",
+    "Inspect the whole approved public site; enumerate distinct renderers, routes, and Pixel coverage.",
+    "Infer the business model and customer outcome, then propose one privacy-safe measurement plan.",
+    "If the Pixel is absent, return the canonical one-time tag and exact uncovered surfaces; do not claim ongoing control there.",
+    "Publish an approved measurement revision as a digest-bound runtime manifest; source or repository access is not required.",
+    "Verify route coverage, consent, PageViews, manifest convergence, bounded rule matches, and persisted definition/version evidence.",
+    "Require a separate exact-revision approval before any canary or real-visitor experiment activation.",
+    "Return first customer-journey answers when sufficient, or exact evidence-readiness thresholds when not.",
+    "On “improve”, propose one highest-value uncovered outcome, require approval when needed, and reverify."
+  ],
+  "safetyRules": [
+    "Never invent or hand-author tracking code; use the canonical product-authored Pixel snippet returned by analytics_setup_site.",
+    "Never ask for or print API keys, OAuth tokens, signed URLs, cookies, or other credentials.",
+    "Never call ad platforms directly or silently activate events, definitions, funnels, or delivery destinations.",
+    "Never treat a CTA click as a customer, sale, or revenue event without verified outcome evidence.",
+    "Never claim full-domain installation while an in-scope route or renderer is missing, excluded without approval, or unverifiable.",
+    "Never treat event-name presence as verification; require persisted definition ID and version evidence."
+  ],
+  "terminalStates": [
+    "complete",
+    "tag_required",
+    "input_required",
+    "approval_required",
+    "runtime_convergence_required",
+    "outcome_connection_required",
+    "not_ready",
+    "error"
+  ],
+  "protocolDigest": "sha256:8f2f59ecfa074c1b9a1aba042c43b1de7cd18ebb2354e1911ad8c8e4a936ea3e"
+}

package/public/agent-install.md

@@ -0,0 +1,194 @@
+# Install Thorbit X-Ray
+
+Protocol version: 2.0.0
+Protocol digest: sha256:8f2f59ecfa074c1b9a1aba042c43b1de7cd18ebb2354e1911ad8c8e4a936ea3e
+
+Say **“Install X-Ray”** or **“Install X-Ray on this site and tell me what drives customers.”**
+
+Install one Pixel globally once. X-Ray then inventories the approved public site, proposes one measurement plan, publishes digest-bound runtime configuration without GitHub, verifies live convergence, and returns customer-journey answers or exact evidence-readiness thresholds.
+
+## Exact continuation
+
+> Call analytics_setup_site with { "startUrl": "<site URL>", "goal": "Install X-Ray and tell me what drives customers" }; on every later call, reuse the returned setupId and revision.
+
+## Connection topologies
+
+- **Local stdio:** The client starts MCP Scraper locally. After the one-time Pixel insertion, X-Ray setup publishes only owner-confirmed runtime configuration. Set MCP_SCRAPER_API_KEY in the local environment or client secret field. Never paste it into chat or commit it.
+- **Hosted remote:** The hosted MCP inspects approved public pages and publishes owner-confirmed X-Ray runtime configuration. It cannot read or write a caller-local path. Use the MCP OAuth/connection flow. Never put a credential in a URL, prompt, or config shown in chat.
+
+## Host recipes
+
+### Codex (`codex`)
+
+- Topology: local_stdio, hosted_remote
+- Workspace read/write: yes/yes
+- Restart required: yes
+
+```text
+[mcp_servers.mcp-scraper]
+command = "npx"
+args = ["-y", "--package", "mcp-scraper@latest", "mcp-scraper"]
+env_vars = ["MCP_SCRAPER_API_KEY"]
+```
+
+Continuation: Call analytics_setup_site with { "startUrl": "<site URL>", "goal": "Install X-Ray and tell me what drives customers" }; on every later call, reuse the returned setupId and revision.
+
+Rollback: Remove the mcp_servers.mcp-scraper table, then restart Codex. Pixel removal is a separate owner-controlled site action.
+
+### Claude Code (`claude-code`)
+
+- Topology: local_stdio, hosted_remote
+- Workspace read/write: yes/yes
+- Restart required: yes
+
+```text
+claude mcp add mcp-scraper --scope user --env MCP_SCRAPER_API_KEY="$MCP_SCRAPER_API_KEY" -- npx -y --package mcp-scraper@latest mcp-scraper
+```
+
+Continuation: Call analytics_setup_site with { "startUrl": "<site URL>", "goal": "Install X-Ray and tell me what drives customers" }; on every later call, reuse the returned setupId and revision.
+
+Rollback: Run: claude mcp remove mcp-scraper -s user. Pixel removal is a separate owner-controlled site action.
+
+### Claude Desktop (`claude-desktop`)
+
+- Topology: local_stdio
+- Workspace read/write: no/no
+- Restart required: yes
+
+```text
+Install https://mcpscraper.dev/downloads/mcp-scraper.mcpb and enter the API key only in Claude Desktop's protected configuration field.
+```
+
+Continuation: Call analytics_setup_site with { "startUrl": "<site URL>", "goal": "Install X-Ray and tell me what drives customers" }; on every later call, reuse the returned setupId and revision.
+
+Rollback: Remove the MCP Scraper extension in Claude Desktop. Pixel removal is a separate owner-controlled site action.
+
+### Cursor (`cursor`)
+
+- Topology: local_stdio, hosted_remote
+- Workspace read/write: yes/yes
+- Restart required: yes
+
+```text
+{
+  "mcpServers": {
+    "mcp-scraper": {
+      "command": "npx",
+      "args": [
+        "-y",
+        "--package",
+        "mcp-scraper@latest",
+        "mcp-scraper"
+      ],
+      "env": {
+        "MCP_SCRAPER_API_KEY": "${env:MCP_SCRAPER_API_KEY}"
+      }
+    }
+  }
+}
+```
+
+Continuation: Call analytics_setup_site with { "startUrl": "<site URL>", "goal": "Install X-Ray and tell me what drives customers" }; on every later call, reuse the returned setupId and revision.
+
+Rollback: Remove mcp-scraper from .cursor/mcp.json (or the user MCP settings), then restart Cursor.
+
+### Windsurf (`windsurf`)
+
+- Topology: local_stdio
+- Workspace read/write: yes/yes
+- Restart required: yes
+
+```text
+{
+  "mcpServers": {
+    "mcp-scraper": {
+      "command": "npx",
+      "args": [
+        "-y",
+        "--package",
+        "mcp-scraper@latest",
+        "mcp-scraper"
+      ],
+      "env": {
+        "MCP_SCRAPER_API_KEY": "${env:MCP_SCRAPER_API_KEY}"
+      }
+    }
+  }
+}
+```
+
+Continuation: Call analytics_setup_site with { "startUrl": "<site URL>", "goal": "Install X-Ray and tell me what drives customers" }; on every later call, reuse the returned setupId and revision.
+
+Rollback: Remove mcp-scraper from the Windsurf MCP configuration, then restart Windsurf.
+
+### Cline (`cline`)
+
+- Topology: local_stdio
+- Workspace read/write: yes/yes
+- Restart required: yes
+
+```text
+{
+  "mcpServers": {
+    "mcp-scraper": {
+      "command": "npx",
+      "args": [
+        "-y",
+        "--package",
+        "mcp-scraper@latest",
+        "mcp-scraper"
+      ],
+      "env": {
+        "MCP_SCRAPER_API_KEY": "${env:MCP_SCRAPER_API_KEY}"
+      }
+    }
+  }
+}
+```
+
+Continuation: Call analytics_setup_site with { "startUrl": "<site URL>", "goal": "Install X-Ray and tell me what drives customers" }; on every later call, reuse the returned setupId and revision.
+
+Rollback: Remove mcp-scraper from Cline MCP Servers, then restart the extension host.
+
+### User-action-only MCP host (`user-action-only`)
+
+- Topology: hosted_remote
+- Workspace read/write: no/no
+- Restart required: no
+
+```text
+{
+  "mcpServers": {
+    "mcp-scraper": {
+      "url": "https://mcpscraper.dev/mcp"
+    }
+  }
+}
+```
+
+Continuation: Call analytics_setup_site with { "startUrl": "<site URL>", "goal": "Install X-Ray and tell me what drives customers" }; on every later call, reuse the returned setupId and revision.
+
+Rollback: Disconnect MCP Scraper from the host. Pixel removal remains an owner-controlled site action.
+
+## What X-Ray does
+
+1. Resolve Thorbit X-Ray entitlement and the target site without consuming MCP Scraper Credits.
+1. Inspect the whole approved public site; enumerate distinct renderers, routes, and Pixel coverage.
+1. Infer the business model and customer outcome, then propose one privacy-safe measurement plan.
+1. If the Pixel is absent, return the canonical one-time tag and exact uncovered surfaces; do not claim ongoing control there.
+1. Publish an approved measurement revision as a digest-bound runtime manifest; source or repository access is not required.
+1. Verify route coverage, consent, PageViews, manifest convergence, bounded rule matches, and persisted definition/version evidence.
+1. Require a separate exact-revision approval before any canary or real-visitor experiment activation.
+1. Return first customer-journey answers when sufficient, or exact evidence-readiness thresholds when not.
+1. On “improve”, propose one highest-value uncovered outcome, require approval when needed, and reverify.
+
+## Safety contract
+
+- Never invent or hand-author tracking code; use the canonical product-authored Pixel snippet returned by analytics_setup_site.
+- Never ask for or print API keys, OAuth tokens, signed URLs, cookies, or other credentials.
+- Never call ad platforms directly or silently activate events, definitions, funnels, or delivery destinations.
+- Never treat a CTA click as a customer, sale, or revenue event without verified outcome evidence.
+- Never claim full-domain installation while an in-scope route or renderer is missing, excluded without approval, or unverifiable.
+- Never treat event-name presence as verification; require persisted definition ID and version evidence.
+
+Machine-readable contract: https://mcpscraper.dev/.well-known/xray-install.json