mcp-scraper 0.81.0 → 0.83.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (26) hide show
  1. package/CHANGELOG.md +45 -1
  2. package/README.md +2 -2
  3. package/dist/{analytics-repository-YOY5KMOD.js → analytics-repository-GA6P5S5J.js} +1 -1
  4. package/dist/bin/api-server.js +3 -3
  5. package/dist/bin/mcp-scraper-cli.js +61 -46
  6. package/dist/bin/mcp-scraper-core.js +6 -6
  7. package/dist/bin/mcp-scraper-install.js +2 -2
  8. package/dist/bin/mcp-stdio-server.js +6 -6
  9. package/dist/{chunk-PMONKQGJ.js → chunk-4NZ6JFK5.js} +2 -2
  10. package/dist/{chunk-RSXDHQBA.js → chunk-62ROBG4P.js} +1 -1
  11. package/dist/{chunk-VEQPA65D.js → chunk-ACDNW7TG.js} +1 -1
  12. package/dist/{chunk-VXLU74YZ.js → chunk-BGD7U3AE.js} +207 -1
  13. package/dist/{chunk-HEUMDX4I.js → chunk-CNAJWRJT.js} +131 -1
  14. package/dist/{chunk-XBEUZJHS.js → chunk-HGJJQY24.js} +49 -5
  15. package/dist/{chunk-Z63DQAO7.js → chunk-J2CRSYL3.js} +4 -3
  16. package/dist/{chunk-M2A7YSJV.js → chunk-UXVT7FWW.js} +2 -2
  17. package/dist/{chunk-XWXCLKXF.js → chunk-W54QRM7I.js} +3107 -2210
  18. package/dist/{chunk-VWC7XATD.js → chunk-ZHPGPVJJ.js} +1 -1
  19. package/dist/{extract-bundle-232SA7MI.js → extract-bundle-HTWWJD6N.js} +2 -2
  20. package/dist/{gmail-service-2A3ZZT3M.js → gmail-service-VXE7OYNA.js} +2 -2
  21. package/dist/{server-QRYEMF4H.js → server-XBUOLGZ3.js} +2117 -791
  22. package/dist/{site-extract-repository-VDONDDMR.js → site-extract-repository-Y5WQFSQW.js} +6 -2
  23. package/dist/{worker-2MY77WYY.js → worker-FM3ZRDY4.js} +1 -1
  24. package/package.json +6 -2
  25. package/public/.well-known/xray-install.json +178 -0
  26. package/public/agent-install.md +194 -0
package/CHANGELOG.md CHANGED
@@ -4,6 +4,47 @@ All notable changes to MCP Scraper are documented here. The format is based on [
4
4
 
5
5
  ## [Unreleased]
6
6
 
7
+ ## [0.83.1] - 2026-09-01
8
+
9
+ ### Fixed
10
+
11
+ - Allowed the X-Ray Pixel to authenticate normal same-origin runtime-manifest and health requests from their approved `Referer` hostname when browsers omit the `Origin` header, while preserving approved-domain enforcement and failing closed when neither header identifies an HTTP origin.
12
+
13
+ ## [0.83.0] - 2026-09-01
14
+
15
+ ### Added
16
+
17
+ - Added a digest-bound X-Ray runtime manifest that remotely publishes versioned event rules through the globally installed Pixel, reports bounded selector health, observes SPA and dynamic DOM changes, and preserves consent and privacy boundaries without requiring site-source access.
18
+ - Added pre-registered X-Ray experiments with deterministic assignment, separate exact-revision approval, canary and production states, five allowlisted presentation operations, exposure evidence, fixed-horizon evaluation, sample-ratio checks, and pause/kill rollback controls.
19
+ - Added `analytics_create_experiment`, `analytics_set_experiment_state`, and `analytics_get_experiment`, expanding the governed unified MCP catalog to 363 tools.
20
+
21
+ ### Changed
22
+
23
+ - Replaced X-Ray's repository, branch, writer, patch, and draft-PR setup model with one-time global tag verification followed by X-Ray-owned measurement-manifest publication. GitHub remains an optional general integration and is no longer part of X-Ray setup.
24
+ - Advanced the canonical X-Ray install protocol to 2.0.0 and aligned its MCP Resource, Prompt, npm CLI, public Markdown, well-known JSON, REST contract, and dashboard projections around the tag-only control plane.
25
+
26
+ ### Migration
27
+
28
+ - Advance the analytics schema to `2026-09-01.1` before enabling runtime manifests or experiments. Existing active event definitions remain compatible and measurement publication does not activate a visitor experiment.
29
+
30
+ ## [0.82.0] - 2026-08-31
31
+
32
+ ### Added
33
+
34
+ - Added `analytics_setup_site` with explicit prepare, apply, and verify operations, durable owner-bound revisions and receipts, whole-site Pixel and CTA discovery, eight-dimensional readiness, and truthful first-customer answers at zero MCP Scraper Credit charge under an active X-Ray entitlement.
35
+ - Added a versioned agent-install protocol across public Markdown/JSON, npm CLI output, an MCP Resource, an owner-isolated setup ResourceTemplate, and the `setup_xray_site` Prompt for Codex, Claude, Cursor, Windsurf, and Cline.
36
+ - Added a trusted internal `xray_entitlement` extraction billing class that persists zero-charge jobs without wallet holds, debit keys, or ledger settlement.
37
+
38
+ ### Changed
39
+
40
+ - Expanded the unified MCP catalog from 356 to 360 tools with operation-specific setup contracts, generated REST parity, private discovery caching where supported, and a measured reviewed catalog budget.
41
+ - Added privacy-safe semantic CTA metadata for placement, component, action, destination, and plan while excluding labels, form values, URLs and query strings, email addresses, phone-like values, and arbitrary datasets.
42
+
43
+ ### Fixed
44
+
45
+ - Installed exactly one first-party Pixel across both the Next.js marketing application and all eight static/API blog documents, and added reusable semantic CTA tagging to shared marketing components.
46
+ - Removed the duplicate initial PageView caused by pre-granted consent while retaining one PageView for each actual SPA navigation or blocked-to-granted consent transition.
47
+
7
48
  ## [0.81.0] - 2026-08-31
8
49
 
9
50
  ### Added
@@ -1544,7 +1585,10 @@ All notable changes to MCP Scraper are documented here. The format is based on [
1544
1585
  - Write actions remain unavailable until the account owner explicitly enables them.
1545
1586
  - Provider-specific connection data is normalized into one agent-facing contract.
1546
1587
 
1547
- [Unreleased]: https://github.com/VilovietaSEO/mcp-scraper/compare/v0.81.0...HEAD
1588
+ [Unreleased]: https://github.com/VilovietaSEO/mcp-scraper/compare/v0.83.1...HEAD
1589
+ [0.83.1]: https://github.com/VilovietaSEO/mcp-scraper/compare/v0.83.0...v0.83.1
1590
+ [0.83.0]: https://github.com/VilovietaSEO/mcp-scraper/compare/v0.82.0...v0.83.0
1591
+ [0.82.0]: https://github.com/VilovietaSEO/mcp-scraper/compare/v0.81.0...v0.82.0
1548
1592
  [0.81.0]: https://github.com/VilovietaSEO/mcp-scraper/compare/v0.80.0...v0.81.0
1549
1593
  [0.80.0]: https://github.com/VilovietaSEO/mcp-scraper/compare/v0.79.4...v0.80.0
1550
1594
  [0.79.4]: https://github.com/VilovietaSEO/mcp-scraper/compare/v0.79.3...v0.79.4
package/README.md CHANGED
@@ -175,7 +175,7 @@ Build the branded one-click bundle:
175
175
  npm run build:mcpb
176
176
  ```
177
177
 
178
- The generated bundle is written to `build/mcpb/mcp-scraper-<version>.mcpb` and copied to `public/downloads/` for the hosted download. The current public bundle is `https://mcpscraper.dev/downloads/mcp-scraper.mcpb` (`0.81.0`, SHA-256 `866faf8d18568514b29f407c6789bc779e895c9780cfcee27eb958cb079cf244`). Install it by opening or dragging it into Claude Desktop. Claude displays the `MCP Scraper` install card, icon, API-key configuration field, and manually curated current-release message from the bundle manifest.
178
+ The generated bundle is written to `build/mcpb/mcp-scraper-<version>.mcpb` and copied to `public/downloads/` for the hosted download. The current public bundle is `https://mcpscraper.dev/downloads/mcp-scraper.mcpb` (`0.83.1`, SHA-256 `57e9dfb6d83aa729cf90ca4654f72a4c8b16f219cbdbde737f72f23f36f3b46c`). Install it by opening or dragging it into Claude Desktop. Claude displays the `MCP Scraper` install card, icon, API-key configuration field, and manually curated current-release message from the bundle manifest.
179
179
 
180
180
  The MCPB install exposes every tool — web-intelligence plus all `browser_*` tools — through the one `mcp-scraper` server.
181
181
 
@@ -337,7 +337,7 @@ The `mcp-scraper` server (and the MCPB bundle, which runs it) exposes both secti
337
337
 
338
338
  All MCP tools return `structuredContent` with the IDs, URLs, CSV paths, transcripts, browser session handles, replay paths, artifacts, recipe fields, or blueprint fields needed by the next step, plus readable text content for compatibility. Runtime `tools/list` omits output schemas so strict clients can register the complete catalog; the generated developer manifest retains every canonical output schema for validation and typed SDK generation. All tools carry MCP annotations; file-writing tools such as replay downloads and annotations state their filesystem side effects.
339
339
 
340
- The canonical tool inventory is generated at `docs/mcp-tool-manifest.generated.json`. The unified server exposes 356 tools: 255 scraper, browser, workflow, billing, connected-service, and personal-assistant tools plus 101 durable-memory tools. The scraper-side inventory includes complete Gmail selection, message, attachment, export, bulk-action, and Memory-import workflows; governed personal-assistant commands, messaging, approvals, grants, number setup, and execution readback; durable PAA starts and status; rendered site-content similarity; governed Local Sourcebook and Transparent Commons workflows; direct site-export reads; Editorial Reading Room and News Publisher templates; and production X-Ray analytics, attribution, CRM policy and receipt, campaign, export, and scheduled-report tools. Successful evidence-compiled Local Sourcebook revisions publish automatically to their canonical `localsourcebook.com` category profile and review URLs; administrator controls handle exceptional rejection or unpublishing. Release verification compares the exact local and hosted tool-name sets, not only the count.
340
+ The canonical tool inventory is generated at `docs/mcp-tool-manifest.generated.json`. The unified server exposes 363 tools: 262 scraper, browser, workflow, billing, connected-service, and personal-assistant tools plus 101 durable-memory tools. The scraper-side inventory includes complete Gmail selection, message, attachment, export, bulk-action, and Memory-import workflows; governed personal-assistant commands, messaging, approvals, grants, number setup, and execution readback; durable PAA starts and status; rendered site-content similarity; governed Local Sourcebook and Transparent Commons workflows; direct site-export reads; Editorial Reading Room and News Publisher templates; and production X-Ray setup, analytics, attribution, CRM policy and receipt, campaign, export, and scheduled-report tools. Successful evidence-compiled Local Sourcebook revisions publish automatically to their canonical `localsourcebook.com` category profile and review URLs; administrator controls handle exceptional rejection or unpublishing. Release verification compares the exact local and hosted tool-name sets, not only the count.
341
341
 
342
342
  For contract parity, stdio and MCPB memory calls invoke the matching public tool on the hosted MCP Scraper `/mcp` endpoint. The hosted aggregate runtime owns MCP Scraper-specific billing, scheduling, credential, and in-process cutover policy; its internal `/memory/mcp-call` bridge is a fallback to the standalone memory service, not the public stdio execution path. Direct `mcp-memory` OAuth and stdio clients continue to use `memory.mcpscraper.dev` and must be verified as a separate dependent release surface.
343
343
 
@@ -85,7 +85,7 @@ import {
85
85
  updateAnalyticsPixel,
86
86
  upsertAnalyticsAdSpend,
87
87
  upsertAnalyticsHostGroup
88
- } from "./chunk-HEUMDX4I.js";
88
+ } from "./chunk-CNAJWRJT.js";
89
89
  import "./chunk-C5Z4OFKW.js";
90
90
  import "./chunk-YXNDOQXN.js";
91
91
  export {
@@ -17,8 +17,8 @@ loadDotEnv();
17
17
  async function main() {
18
18
  const [{ serve }, { app, personalAssistantProductionStartup }, { startWorker }, { migrate }] = await Promise.all([
19
19
  import("@hono/node-server"),
20
- import("../server-QRYEMF4H.js"),
21
- import("../worker-2MY77WYY.js"),
20
+ import("../server-XBUOLGZ3.js"),
21
+ import("../worker-FM3ZRDY4.js"),
22
22
  import("../db-Z34LPZNR.js")
23
23
  ]);
24
24
  const PORT = parseInt(process.env.PORT ?? "3001");
@@ -26,7 +26,7 @@ async function main() {
26
26
  await personalAssistantProductionStartup;
27
27
  await migrate();
28
28
  if (process.env.ANALYTICS_DATABASE_URL) {
29
- const { migrateAnalytics } = await import("../analytics-repository-YOY5KMOD.js");
29
+ const { migrateAnalytics } = await import("../analytics-repository-GA6P5S5J.js");
30
30
  await migrateAnalytics();
31
31
  }
32
32
  startWorker();
@@ -6,14 +6,20 @@ import {
6
6
  openWorkflowReport,
7
7
  runWorkflow,
8
8
  workflowOutputBaseDir
9
- } from "../chunk-VEQPA65D.js";
9
+ } from "../chunk-ACDNW7TG.js";
10
10
  import "../chunk-RJ7JVYKU.js";
11
11
  import "../chunk-OWF2JJKN.js";
12
- import "../chunk-VXLU74YZ.js";
12
+ import {
13
+ XRAY_INSTALL_CONTRACT,
14
+ XRAY_INSTALL_PROTOCOL_DIGEST,
15
+ XRAY_INSTALL_PROTOCOL_VERSION,
16
+ renderXraySetupPrompt,
17
+ xrayHostRecipe
18
+ } from "../chunk-BGD7U3AE.js";
13
19
  import "../chunk-GGZEC22A.js";
14
20
  import {
15
21
  PACKAGE_VERSION
16
- } from "../chunk-RSXDHQBA.js";
22
+ } from "../chunk-62ROBG4P.js";
17
23
 
18
24
  // src/cli/human-cli.ts
19
25
  import { Command } from "commander";
@@ -38,26 +44,6 @@ function envConfig(options) {
38
44
  if (options.browserProfileSaveChanges === true) env.BROWSER_AGENT_PROFILE_SAVE_CHANGES = "true";
39
45
  return env;
40
46
  }
41
- function tomlInlineTable(value) {
42
- const entries = Object.entries(value).map(([key, item]) => `${key} = ${JSON.stringify(item)}`);
43
- return `{ ${entries.join(", ")} }`;
44
- }
45
- function renderCodexConfig(options = {}) {
46
- return [
47
- "[mcp_servers.mcp-scraper]",
48
- 'command = "npx"',
49
- `args = ${JSON.stringify(combinedNpxArgs(options))}`,
50
- `env = ${tomlInlineTable(envConfig(options))}`
51
- ].join("\n");
52
- }
53
- function renderClaudeCommand(options = {}) {
54
- const env = Object.entries(envConfig(options)).map(([key, value]) => ` --env ${key}=${value}`);
55
- return [
56
- "claude mcp add mcp-scraper --scope user",
57
- ...env,
58
- ` -- npx ${combinedNpxArgs(options).join(" ")}`
59
- ].join(" \\\n");
60
- }
61
47
  function claudeMcpRemoveArgs() {
62
48
  return ["mcp", "remove", "mcp-scraper", "-s", "user"];
63
49
  }
@@ -101,28 +87,26 @@ function claudeMcpAddArgs(options = {}) {
101
87
  }
102
88
  function normalizeAgentHost(host) {
103
89
  if (host === "claude-code") return "claude";
104
- if (host === "claude" || host === "codex" || host === "claude-desktop") return host;
105
- throw new Error('Unknown host "' + host + '". Use: codex, claude, claude-code, or claude-desktop');
90
+ if (host === "claude" || XRAY_INSTALL_CONTRACT.hosts.some((candidate) => candidate.id === host)) return host;
91
+ throw new Error('Unknown host "' + host + '". Use: codex, claude, claude-code, claude-desktop, cursor, windsurf, cline, or user-action-only');
106
92
  }
107
- function renderClaudeDesktopConfig(options = {}) {
108
- return JSON.stringify({
109
- mcpServers: {
110
- "mcp-scraper": {
111
- command: "npx",
112
- args: combinedNpxArgs(options),
113
- env: envConfig(options)
114
- }
115
- }
116
- }, null, 2);
93
+ function agentHostCapability(host) {
94
+ return xrayHostRecipe(host === "claude" ? "claude-code" : host);
117
95
  }
118
- function renderAgentInstall(host, options = {}) {
96
+ function renderAgentInstall(host, _options = {}) {
119
97
  const normalizedHost = normalizeAgentHost(host);
98
+ const recipe = agentHostCapability(normalizedHost);
120
99
  const restart = "Restart the MCP client so it starts a fresh npx process.";
121
- const applyCommand = `MCP_SCRAPER_API_KEY=${apiKeyValue(options)} npx -y -p mcp-scraper@latest mcp-scraper-cli agent install claude --apply`;
100
+ const applyCommand = 'MCP_SCRAPER_API_KEY="$MCP_SCRAPER_API_KEY" npx -y -p mcp-scraper@latest mcp-scraper-cli agent install claude --apply';
101
+ const protocol = `X-Ray install protocol: ${XRAY_INSTALL_PROTOCOL_VERSION} (${XRAY_INSTALL_PROTOCOL_DIGEST})`;
122
102
  if (normalizedHost === "codex") {
123
103
  return [
124
104
  "# Codex MCP config",
125
- renderCodexConfig(options),
105
+ protocol,
106
+ recipe.exactConfig,
107
+ "",
108
+ `Continuation: ${recipe.continuation}`,
109
+ `Rollback: ${recipe.rollback}`,
126
110
  "",
127
111
  restart
128
112
  ].join("\n");
@@ -130,20 +114,38 @@ function renderAgentInstall(host, options = {}) {
130
114
  if (normalizedHost === "claude") {
131
115
  return [
132
116
  "# Claude Code command",
133
- renderClaudeCommand(options),
117
+ protocol,
118
+ recipe.exactConfig,
134
119
  "",
135
120
  "# One-command Claude Code setup",
136
121
  applyCommand,
137
122
  "",
123
+ `Continuation: ${recipe.continuation}`,
124
+ `Rollback: ${recipe.rollback}`,
125
+ "",
126
+ restart
127
+ ].join("\n");
128
+ }
129
+ if (normalizedHost === "claude-desktop") {
130
+ return [
131
+ "# Claude Desktop config",
132
+ protocol,
133
+ recipe.exactConfig,
134
+ "",
135
+ "Desktop Extension: https://mcpscraper.dev/downloads/mcp-scraper.mcpb",
136
+ `Continuation: ${recipe.continuation}`,
137
+ `Rollback: ${recipe.rollback}`,
138
138
  restart
139
139
  ].join("\n");
140
140
  }
141
141
  return [
142
- "# Claude Desktop config",
143
- renderClaudeDesktopConfig(options),
142
+ `# ${recipe.label} MCP config`,
143
+ protocol,
144
+ recipe.exactConfig,
144
145
  "",
145
- "Desktop Extension: https://mcpscraper.dev/downloads/mcp-scraper.mcpb",
146
- restart
146
+ `Continuation: ${recipe.continuation}`,
147
+ `Rollback: ${recipe.rollback}`,
148
+ ...recipe.restartRequired ? ["", restart] : []
147
149
  ].join("\n");
148
150
  }
149
151
 
@@ -288,6 +290,7 @@ function renderDoctor(output) {
288
290
 
289
291
  // src/cli/prompts.ts
290
292
  var AGENT_PROMPTS = {
293
+ "xray-setup": renderXraySetupPrompt(),
291
294
  "agent-packet": [
292
295
  "# MCP Scraper Agent Packet Prompt",
293
296
  "",
@@ -561,7 +564,7 @@ function buildHumanCli() {
561
564
  ].join("\n"), false);
562
565
  });
563
566
  const agent = program.command("agent").description("Generate AI-agent install configs and workflow prompts.");
564
- agent.command("install <host>").description("Print or apply install/config instructions for codex, claude/claude-code, or claude-desktop.").option("--api-key <key>", "API key to place in generated config").option("--package <spec>", "npm package spec", "mcp-scraper@latest").option("--browser-profile <name>", "Default saved browser profile for browser_open sessions.").option("--save-browser-profile-changes", "Persist cookies and browser storage back to the named browser profile when sessions close").option("--apply", "Apply the config to this client when supported. For Claude Code, this upserts the user-scope mcp-scraper server.").option("--json", "Print machine-readable JSON").action(async (hostInput, opts) => {
567
+ agent.command("install <host>").description("Print or apply install/config instructions for Codex, Claude, Cursor, Windsurf, Cline, or a handoff-only MCP host.").option("--api-key <key>", "API key to place in generated config").option("--package <spec>", "npm package spec", "mcp-scraper@latest").option("--browser-profile <name>", "Default saved browser profile for browser_open sessions.").option("--save-browser-profile-changes", "Persist cookies and browser storage back to the named browser profile when sessions close").option("--apply", "Apply the config to this client when supported. For Claude Code, this upserts the user-scope mcp-scraper server.").option("--json", "Print machine-readable JSON").action(async (hostInput, opts) => {
565
568
  const host = normalizeAgentHost(hostInput);
566
569
  const apiKey = opts.apiKey ?? process.env.MCP_SCRAPER_API_KEY;
567
570
  if (opts.apply) {
@@ -631,7 +634,12 @@ ${result.list}` : ""
631
634
  browserProfileName: opts.browserProfile,
632
635
  browserProfileSaveChanges: opts.saveBrowserProfileChanges
633
636
  });
634
- writeOutput(opts.json ? { host, text } : text, opts.json);
637
+ writeOutput(opts.json ? {
638
+ host,
639
+ protocolVersion: XRAY_INSTALL_PROTOCOL_VERSION,
640
+ protocolDigest: XRAY_INSTALL_PROTOCOL_DIGEST,
641
+ text
642
+ } : text, opts.json);
635
643
  });
636
644
  agent.command("prompt [name]").description("Print an agent prompt template.").option("--json", "Print machine-readable JSON").action((name, opts) => {
637
645
  if (!name) {
@@ -639,7 +647,14 @@ ${result.list}` : ""
639
647
  return;
640
648
  }
641
649
  const text = renderPrompt(name);
642
- writeOutput(opts.json ? { name, text } : text, opts.json);
650
+ writeOutput(opts.json ? {
651
+ name,
652
+ ...name === "xray-setup" ? {
653
+ protocolVersion: XRAY_INSTALL_PROTOCOL_VERSION,
654
+ protocolDigest: XRAY_INSTALL_PROTOCOL_DIGEST
655
+ } : {},
656
+ text
657
+ } : text, opts.json);
643
658
  });
644
659
  const workflow = program.command("workflow").description("Run named SEO workflows.");
645
660
  workflow.command("list").description("List available workflows.").option("--json", "Print machine-readable JSON").action((opts) => {
@@ -1,19 +1,19 @@
1
1
  #!/usr/bin/env node
2
2
  import {
3
3
  runMcpScraperStdio
4
- } from "../chunk-Z63DQAO7.js";
5
- import "../chunk-XWXCLKXF.js";
4
+ } from "../chunk-J2CRSYL3.js";
5
+ import "../chunk-W54QRM7I.js";
6
6
  import "../chunk-PGJQDMC2.js";
7
7
  import "../chunk-DNM65UCK.js";
8
8
  import "../chunk-T3MZISOF.js";
9
9
  import "../chunk-6HAV7LCE.js";
10
- import "../chunk-PMONKQGJ.js";
10
+ import "../chunk-4NZ6JFK5.js";
11
11
  import "../chunk-OM7HVEJ3.js";
12
12
  import "../chunk-OPQIGAFB.js";
13
- import "../chunk-M2A7YSJV.js";
14
- import "../chunk-VXLU74YZ.js";
13
+ import "../chunk-UXVT7FWW.js";
14
+ import "../chunk-BGD7U3AE.js";
15
15
  import "../chunk-GGZEC22A.js";
16
- import "../chunk-RSXDHQBA.js";
16
+ import "../chunk-62ROBG4P.js";
17
17
  import "../chunk-YXNDOQXN.js";
18
18
 
19
19
  // src/mcp/thorbit-restricted-tool-surface.ts
@@ -1,10 +1,10 @@
1
1
  #!/usr/bin/env node
2
2
  import {
3
3
  renderInstallTerminal
4
- } from "../chunk-M2A7YSJV.js";
4
+ } from "../chunk-UXVT7FWW.js";
5
5
  import {
6
6
  PACKAGE_VERSION
7
- } from "../chunk-RSXDHQBA.js";
7
+ } from "../chunk-62ROBG4P.js";
8
8
 
9
9
  // bin/mcp-scraper-install.ts
10
10
  var noColor = process.argv.includes("--no-color") || process.env.NO_COLOR !== void 0 || process.env.FORCE_COLOR === "0" || !process.stdout.isTTY;
@@ -1,19 +1,19 @@
1
1
  #!/usr/bin/env node
2
2
  import {
3
3
  runMcpScraperStdio
4
- } from "../chunk-Z63DQAO7.js";
5
- import "../chunk-XWXCLKXF.js";
4
+ } from "../chunk-J2CRSYL3.js";
5
+ import "../chunk-W54QRM7I.js";
6
6
  import "../chunk-PGJQDMC2.js";
7
7
  import "../chunk-DNM65UCK.js";
8
8
  import "../chunk-T3MZISOF.js";
9
9
  import "../chunk-6HAV7LCE.js";
10
- import "../chunk-PMONKQGJ.js";
10
+ import "../chunk-4NZ6JFK5.js";
11
11
  import "../chunk-OM7HVEJ3.js";
12
12
  import "../chunk-OPQIGAFB.js";
13
- import "../chunk-M2A7YSJV.js";
14
- import "../chunk-VXLU74YZ.js";
13
+ import "../chunk-UXVT7FWW.js";
14
+ import "../chunk-BGD7U3AE.js";
15
15
  import "../chunk-GGZEC22A.js";
16
- import "../chunk-RSXDHQBA.js";
16
+ import "../chunk-62ROBG4P.js";
17
17
  import "../chunk-YXNDOQXN.js";
18
18
 
19
19
  // bin/mcp-stdio-server.ts
@@ -213,8 +213,8 @@ var PUBLIC_PRODUCT_CONTRACT = {
213
213
  "freeSignupCredits": 0
214
214
  },
215
215
  "inventory": {
216
- "totalTools": 356,
217
- "scraperTools": 255,
216
+ "totalTools": 363,
217
+ "scraperTools": 262,
218
218
  "memoryTools": 101
219
219
  },
220
220
  "concurrencyPack": {
@@ -1,5 +1,5 @@
1
1
  // src/version.ts
2
- var PACKAGE_VERSION = "0.81.0";
2
+ var PACKAGE_VERSION = "0.83.1";
3
3
 
4
4
  export {
5
5
  PACKAGE_VERSION
@@ -11,7 +11,7 @@ import {
11
11
  extractContactEvidence,
12
12
  extractSerpEmailEvidence,
13
13
  selectPrimaryEmail
14
- } from "./chunk-VXLU74YZ.js";
14
+ } from "./chunk-BGD7U3AE.js";
15
15
 
16
16
  // src/lib/slugify.ts
17
17
  function slugify(s) {
@@ -145,6 +145,204 @@ function extractContactEvidence(html, pageUrl) {
145
145
  };
146
146
  }
147
147
 
148
+ // src/mcp/xray-install-contract.ts
149
+ import { createHash } from "crypto";
150
+ var XRAY_INSTALL_PROTOCOL_VERSION = "2.0.0";
151
+ var XRAY_SETUP_TOOL = "analytics_setup_site";
152
+ var XRAY_TRIGGER_PHRASE = "Install X-Ray";
153
+ var XRAY_CANONICAL_GOAL = "Install X-Ray and tell me what drives customers";
154
+ var CONTINUATION = `Call ${XRAY_SETUP_TOOL} with { "startUrl": "<site URL>", "goal": "${XRAY_CANONICAL_GOAL}" }; on every later call, reuse the returned setupId and revision.`;
155
+ var LOCAL_JSON = JSON.stringify({
156
+ mcpServers: {
157
+ "mcp-scraper": {
158
+ command: "npx",
159
+ args: ["-y", "--package", "mcp-scraper@latest", "mcp-scraper"],
160
+ env: { MCP_SCRAPER_API_KEY: "${env:MCP_SCRAPER_API_KEY}" }
161
+ }
162
+ }
163
+ }, null, 2);
164
+ var HOSTED_JSON = JSON.stringify({
165
+ mcpServers: {
166
+ "mcp-scraper": { url: "https://mcpscraper.dev/mcp" }
167
+ }
168
+ }, null, 2);
169
+ var XRAY_INSTALL_CONTRACT_PAYLOAD = {
170
+ schema: "https://mcpscraper.dev/schemas/xray-install-contract-v1.json",
171
+ protocolVersion: XRAY_INSTALL_PROTOCOL_VERSION,
172
+ product: "Thorbit X-Ray",
173
+ triggerPhrases: [XRAY_TRIGGER_PHRASE, "Install X-Ray on this site and tell me what drives customers"],
174
+ canonicalUrls: {
175
+ markdown: "https://mcpscraper.dev/agent-install.md",
176
+ json: "https://mcpscraper.dev/.well-known/xray-install.json",
177
+ hostedMcp: "https://mcpscraper.dev/mcp"
178
+ },
179
+ facade: {
180
+ tool: XRAY_SETUP_TOOL,
181
+ initialCall: { startUrl: "<site URL>", goal: XRAY_CANONICAL_GOAL },
182
+ continuationRule: "Treat setupId plus revision as the durable continuation handle. Reuse both after approval, runtime publication, verification, retry, or return."
183
+ },
184
+ topologies: {
185
+ local_stdio: {
186
+ label: "Local npm MCP process",
187
+ authority: "The client starts MCP Scraper locally. After the one-time Pixel insertion, X-Ray setup publishes only owner-confirmed runtime configuration.",
188
+ secretHandling: "Set MCP_SCRAPER_API_KEY in the local environment or client secret field. Never paste it into chat or commit it."
189
+ },
190
+ hosted_remote: {
191
+ label: "Hosted authenticated MCP",
192
+ authority: "The hosted MCP inspects approved public pages and publishes owner-confirmed X-Ray runtime configuration. It cannot read or write a caller-local path.",
193
+ secretHandling: "Use the MCP OAuth/connection flow. Never put a credential in a URL, prompt, or config shown in chat."
194
+ }
195
+ },
196
+ hosts: [
197
+ {
198
+ id: "codex",
199
+ label: "Codex",
200
+ supportedTopologies: ["local_stdio", "hosted_remote"],
201
+ mcpConfigMode: "toml",
202
+ workspaceRead: true,
203
+ workspaceWrite: true,
204
+ restartRequired: true,
205
+ exactConfig: '[mcp_servers.mcp-scraper]\ncommand = "npx"\nargs = ["-y", "--package", "mcp-scraper@latest", "mcp-scraper"]\nenv_vars = ["MCP_SCRAPER_API_KEY"]',
206
+ continuation: CONTINUATION,
207
+ rollback: "Remove the mcp_servers.mcp-scraper table, then restart Codex. Pixel removal is a separate owner-controlled site action."
208
+ },
209
+ {
210
+ id: "claude-code",
211
+ label: "Claude Code",
212
+ supportedTopologies: ["local_stdio", "hosted_remote"],
213
+ mcpConfigMode: "command",
214
+ workspaceRead: true,
215
+ workspaceWrite: true,
216
+ restartRequired: true,
217
+ exactConfig: 'claude mcp add mcp-scraper --scope user --env MCP_SCRAPER_API_KEY="$MCP_SCRAPER_API_KEY" -- npx -y --package mcp-scraper@latest mcp-scraper',
218
+ continuation: CONTINUATION,
219
+ rollback: "Run: claude mcp remove mcp-scraper -s user. Pixel removal is a separate owner-controlled site action."
220
+ },
221
+ {
222
+ id: "claude-desktop",
223
+ label: "Claude Desktop",
224
+ supportedTopologies: ["local_stdio"],
225
+ mcpConfigMode: "desktop_extension",
226
+ workspaceRead: false,
227
+ workspaceWrite: false,
228
+ restartRequired: true,
229
+ exactConfig: "Install https://mcpscraper.dev/downloads/mcp-scraper.mcpb and enter the API key only in Claude Desktop's protected configuration field.",
230
+ continuation: CONTINUATION,
231
+ rollback: "Remove the MCP Scraper extension in Claude Desktop. Pixel removal is a separate owner-controlled site action."
232
+ },
233
+ {
234
+ id: "cursor",
235
+ label: "Cursor",
236
+ supportedTopologies: ["local_stdio", "hosted_remote"],
237
+ mcpConfigMode: "json",
238
+ workspaceRead: true,
239
+ workspaceWrite: true,
240
+ restartRequired: true,
241
+ exactConfig: LOCAL_JSON,
242
+ continuation: CONTINUATION,
243
+ rollback: "Remove mcp-scraper from .cursor/mcp.json (or the user MCP settings), then restart Cursor."
244
+ },
245
+ {
246
+ id: "windsurf",
247
+ label: "Windsurf",
248
+ supportedTopologies: ["local_stdio"],
249
+ mcpConfigMode: "json",
250
+ workspaceRead: true,
251
+ workspaceWrite: true,
252
+ restartRequired: true,
253
+ exactConfig: LOCAL_JSON,
254
+ continuation: CONTINUATION,
255
+ rollback: "Remove mcp-scraper from the Windsurf MCP configuration, then restart Windsurf."
256
+ },
257
+ {
258
+ id: "cline",
259
+ label: "Cline",
260
+ supportedTopologies: ["local_stdio"],
261
+ mcpConfigMode: "json",
262
+ workspaceRead: true,
263
+ workspaceWrite: true,
264
+ restartRequired: true,
265
+ exactConfig: LOCAL_JSON,
266
+ continuation: CONTINUATION,
267
+ rollback: "Remove mcp-scraper from Cline MCP Servers, then restart the extension host."
268
+ },
269
+ {
270
+ id: "user-action-only",
271
+ label: "User-action-only MCP host",
272
+ supportedTopologies: ["hosted_remote"],
273
+ mcpConfigMode: "handoff_only",
274
+ workspaceRead: false,
275
+ workspaceWrite: false,
276
+ restartRequired: false,
277
+ exactConfig: HOSTED_JSON,
278
+ continuation: CONTINUATION,
279
+ rollback: "Disconnect MCP Scraper from the host. Pixel removal remains an owner-controlled site action."
280
+ }
281
+ ],
282
+ tagBootstrap: {
283
+ requirement: "one_global_pixel_once",
284
+ ongoingConfiguration: "xray_runtime_manifest",
285
+ sourceAccessRequiredAfterInstall: false,
286
+ supportedInsertionPaths: ["manual", "tag_manager", "cms_plugin", "coding_host"]
287
+ },
288
+ workflow: [
289
+ "Resolve Thorbit X-Ray entitlement and the target site without consuming MCP Scraper Credits.",
290
+ "Inspect the whole approved public site; enumerate distinct renderers, routes, and Pixel coverage.",
291
+ "Infer the business model and customer outcome, then propose one privacy-safe measurement plan.",
292
+ "If the Pixel is absent, return the canonical one-time tag and exact uncovered surfaces; do not claim ongoing control there.",
293
+ "Publish an approved measurement revision as a digest-bound runtime manifest; source or repository access is not required.",
294
+ "Verify route coverage, consent, PageViews, manifest convergence, bounded rule matches, and persisted definition/version evidence.",
295
+ "Require a separate exact-revision approval before any canary or real-visitor experiment activation.",
296
+ "Return first customer-journey answers when sufficient, or exact evidence-readiness thresholds when not.",
297
+ "On \u201Cimprove\u201D, propose one highest-value uncovered outcome, require approval when needed, and reverify."
298
+ ],
299
+ safetyRules: [
300
+ "Never invent or hand-author tracking code; use the canonical product-authored Pixel snippet returned by analytics_setup_site.",
301
+ "Never ask for or print API keys, OAuth tokens, signed URLs, cookies, or other credentials.",
302
+ "Never call ad platforms directly or silently activate events, definitions, funnels, or delivery destinations.",
303
+ "Never treat a CTA click as a customer, sale, or revenue event without verified outcome evidence.",
304
+ "Never claim full-domain installation while an in-scope route or renderer is missing, excluded without approval, or unverifiable.",
305
+ "Never treat event-name presence as verification; require persisted definition ID and version evidence."
306
+ ],
307
+ terminalStates: ["complete", "tag_required", "input_required", "approval_required", "runtime_convergence_required", "outcome_connection_required", "not_ready", "error"]
308
+ };
309
+ function canonicalJson(value) {
310
+ if (Array.isArray(value)) return `[${value.map(canonicalJson).join(",")}]`;
311
+ if (value && typeof value === "object") {
312
+ return `{${Object.entries(value).sort(([left], [right]) => left.localeCompare(right)).map(([key, item]) => `${JSON.stringify(key)}:${canonicalJson(item)}`).join(",")}}`;
313
+ }
314
+ return JSON.stringify(value);
315
+ }
316
+ function xrayInstallDigestPayload() {
317
+ return canonicalJson(XRAY_INSTALL_CONTRACT_PAYLOAD);
318
+ }
319
+ var XRAY_INSTALL_PROTOCOL_DIGEST = `sha256:${createHash("sha256").update(xrayInstallDigestPayload(), "utf8").digest("hex")}`;
320
+ var XRAY_INSTALL_CONTRACT = {
321
+ ...XRAY_INSTALL_CONTRACT_PAYLOAD,
322
+ protocolDigest: XRAY_INSTALL_PROTOCOL_DIGEST
323
+ };
324
+ function xrayHostRecipe(host) {
325
+ const recipe = XRAY_INSTALL_CONTRACT.hosts.find((candidate) => candidate.id === host);
326
+ if (!recipe) throw new Error(`Unknown X-Ray host: ${host}`);
327
+ return recipe;
328
+ }
329
+ function renderXraySetupPrompt() {
330
+ return [
331
+ "# Install X-Ray",
332
+ "",
333
+ `Protocol: ${XRAY_INSTALL_PROTOCOL_VERSION} (${XRAY_INSTALL_PROTOCOL_DIGEST})`,
334
+ "",
335
+ `Use ${XRAY_SETUP_TOOL} as the single conversational setup facade. Ask for the target site URL if it is missing, then call it with the goal \u201C${XRAY_CANONICAL_GOAL}\u201D.`,
336
+ "",
337
+ "Continue the same setup with its setupId and revision. Confirm one global Pixel across all public rendering surfaces; present one measurement plan; publish it as a digest-bound runtime manifest after exact-revision approval; verify manifest and rule versions; then return first customer-journey answers or explicit readiness thresholds. GitHub or repository access is not required after the tag is installed.",
338
+ "",
339
+ "Do not request pasted credentials, invent tracking code, silently activate measurement, call ad platforms directly, or claim customers/revenue from CTA clicks."
340
+ ].join("\n");
341
+ }
342
+ function xrayInstallJson(contract = XRAY_INSTALL_CONTRACT) {
343
+ return JSON.parse(JSON.stringify(contract));
344
+ }
345
+
148
346
  // src/leads/contact-enrichment.ts
149
347
  var GENERIC_EMAIL_PREFIXES = /* @__PURE__ */ new Set([
150
348
  "admin",
@@ -289,5 +487,13 @@ export {
289
487
  selectPrimaryEmail,
290
488
  contactAttemptUrls,
291
489
  buildSerpEmailQuery,
292
- extractSerpEmailEvidence
490
+ extractSerpEmailEvidence,
491
+ XRAY_INSTALL_PROTOCOL_VERSION,
492
+ XRAY_SETUP_TOOL,
493
+ XRAY_CANONICAL_GOAL,
494
+ XRAY_INSTALL_PROTOCOL_DIGEST,
495
+ XRAY_INSTALL_CONTRACT,
496
+ xrayHostRecipe,
497
+ renderXraySetupPrompt,
498
+ xrayInstallJson
293
499
  };