mcp-scraper 0.81.0 → 0.83.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +45 -1
- package/README.md +2 -2
- package/dist/{analytics-repository-YOY5KMOD.js → analytics-repository-GA6P5S5J.js} +1 -1
- package/dist/bin/api-server.js +3 -3
- package/dist/bin/mcp-scraper-cli.js +61 -46
- package/dist/bin/mcp-scraper-core.js +6 -6
- package/dist/bin/mcp-scraper-install.js +2 -2
- package/dist/bin/mcp-stdio-server.js +6 -6
- package/dist/{chunk-PMONKQGJ.js → chunk-4NZ6JFK5.js} +2 -2
- package/dist/{chunk-RSXDHQBA.js → chunk-62ROBG4P.js} +1 -1
- package/dist/{chunk-VEQPA65D.js → chunk-ACDNW7TG.js} +1 -1
- package/dist/{chunk-VXLU74YZ.js → chunk-BGD7U3AE.js} +207 -1
- package/dist/{chunk-HEUMDX4I.js → chunk-CNAJWRJT.js} +131 -1
- package/dist/{chunk-XBEUZJHS.js → chunk-HGJJQY24.js} +49 -5
- package/dist/{chunk-Z63DQAO7.js → chunk-J2CRSYL3.js} +4 -3
- package/dist/{chunk-M2A7YSJV.js → chunk-UXVT7FWW.js} +2 -2
- package/dist/{chunk-XWXCLKXF.js → chunk-W54QRM7I.js} +3107 -2210
- package/dist/{chunk-VWC7XATD.js → chunk-ZHPGPVJJ.js} +1 -1
- package/dist/{extract-bundle-232SA7MI.js → extract-bundle-HTWWJD6N.js} +2 -2
- package/dist/{gmail-service-2A3ZZT3M.js → gmail-service-VXE7OYNA.js} +2 -2
- package/dist/{server-QRYEMF4H.js → server-XBUOLGZ3.js} +2117 -791
- package/dist/{site-extract-repository-VDONDDMR.js → site-extract-repository-Y5WQFSQW.js} +6 -2
- package/dist/{worker-2MY77WYY.js → worker-FM3ZRDY4.js} +1 -1
- package/package.json +6 -2
- package/public/.well-known/xray-install.json +178 -0
- package/public/agent-install.md +194 -0
package/CHANGELOG.md
CHANGED
|
@@ -4,6 +4,47 @@ All notable changes to MCP Scraper are documented here. The format is based on [
|
|
|
4
4
|
|
|
5
5
|
## [Unreleased]
|
|
6
6
|
|
|
7
|
+
## [0.83.1] - 2026-09-01
|
|
8
|
+
|
|
9
|
+
### Fixed
|
|
10
|
+
|
|
11
|
+
- Allowed the X-Ray Pixel to authenticate normal same-origin runtime-manifest and health requests from their approved `Referer` hostname when browsers omit the `Origin` header, while preserving approved-domain enforcement and failing closed when neither header identifies an HTTP origin.
|
|
12
|
+
|
|
13
|
+
## [0.83.0] - 2026-09-01
|
|
14
|
+
|
|
15
|
+
### Added
|
|
16
|
+
|
|
17
|
+
- Added a digest-bound X-Ray runtime manifest that remotely publishes versioned event rules through the globally installed Pixel, reports bounded selector health, observes SPA and dynamic DOM changes, and preserves consent and privacy boundaries without requiring site-source access.
|
|
18
|
+
- Added pre-registered X-Ray experiments with deterministic assignment, separate exact-revision approval, canary and production states, five allowlisted presentation operations, exposure evidence, fixed-horizon evaluation, sample-ratio checks, and pause/kill rollback controls.
|
|
19
|
+
- Added `analytics_create_experiment`, `analytics_set_experiment_state`, and `analytics_get_experiment`, expanding the governed unified MCP catalog to 363 tools.
|
|
20
|
+
|
|
21
|
+
### Changed
|
|
22
|
+
|
|
23
|
+
- Replaced X-Ray's repository, branch, writer, patch, and draft-PR setup model with one-time global tag verification followed by X-Ray-owned measurement-manifest publication. GitHub remains an optional general integration and is no longer part of X-Ray setup.
|
|
24
|
+
- Advanced the canonical X-Ray install protocol to 2.0.0 and aligned its MCP Resource, Prompt, npm CLI, public Markdown, well-known JSON, REST contract, and dashboard projections around the tag-only control plane.
|
|
25
|
+
|
|
26
|
+
### Migration
|
|
27
|
+
|
|
28
|
+
- Advance the analytics schema to `2026-09-01.1` before enabling runtime manifests or experiments. Existing active event definitions remain compatible and measurement publication does not activate a visitor experiment.
|
|
29
|
+
|
|
30
|
+
## [0.82.0] - 2026-08-31
|
|
31
|
+
|
|
32
|
+
### Added
|
|
33
|
+
|
|
34
|
+
- Added `analytics_setup_site` with explicit prepare, apply, and verify operations, durable owner-bound revisions and receipts, whole-site Pixel and CTA discovery, eight-dimensional readiness, and truthful first-customer answers at zero MCP Scraper Credit charge under an active X-Ray entitlement.
|
|
35
|
+
- Added a versioned agent-install protocol across public Markdown/JSON, npm CLI output, an MCP Resource, an owner-isolated setup ResourceTemplate, and the `setup_xray_site` Prompt for Codex, Claude, Cursor, Windsurf, and Cline.
|
|
36
|
+
- Added a trusted internal `xray_entitlement` extraction billing class that persists zero-charge jobs without wallet holds, debit keys, or ledger settlement.
|
|
37
|
+
|
|
38
|
+
### Changed
|
|
39
|
+
|
|
40
|
+
- Expanded the unified MCP catalog from 356 to 360 tools with operation-specific setup contracts, generated REST parity, private discovery caching where supported, and a measured reviewed catalog budget.
|
|
41
|
+
- Added privacy-safe semantic CTA metadata for placement, component, action, destination, and plan while excluding labels, form values, URLs and query strings, email addresses, phone-like values, and arbitrary datasets.
|
|
42
|
+
|
|
43
|
+
### Fixed
|
|
44
|
+
|
|
45
|
+
- Installed exactly one first-party Pixel across both the Next.js marketing application and all eight static/API blog documents, and added reusable semantic CTA tagging to shared marketing components.
|
|
46
|
+
- Removed the duplicate initial PageView caused by pre-granted consent while retaining one PageView for each actual SPA navigation or blocked-to-granted consent transition.
|
|
47
|
+
|
|
7
48
|
## [0.81.0] - 2026-08-31
|
|
8
49
|
|
|
9
50
|
### Added
|
|
@@ -1544,7 +1585,10 @@ All notable changes to MCP Scraper are documented here. The format is based on [
|
|
|
1544
1585
|
- Write actions remain unavailable until the account owner explicitly enables them.
|
|
1545
1586
|
- Provider-specific connection data is normalized into one agent-facing contract.
|
|
1546
1587
|
|
|
1547
|
-
[Unreleased]: https://github.com/VilovietaSEO/mcp-scraper/compare/v0.
|
|
1588
|
+
[Unreleased]: https://github.com/VilovietaSEO/mcp-scraper/compare/v0.83.1...HEAD
|
|
1589
|
+
[0.83.1]: https://github.com/VilovietaSEO/mcp-scraper/compare/v0.83.0...v0.83.1
|
|
1590
|
+
[0.83.0]: https://github.com/VilovietaSEO/mcp-scraper/compare/v0.82.0...v0.83.0
|
|
1591
|
+
[0.82.0]: https://github.com/VilovietaSEO/mcp-scraper/compare/v0.81.0...v0.82.0
|
|
1548
1592
|
[0.81.0]: https://github.com/VilovietaSEO/mcp-scraper/compare/v0.80.0...v0.81.0
|
|
1549
1593
|
[0.80.0]: https://github.com/VilovietaSEO/mcp-scraper/compare/v0.79.4...v0.80.0
|
|
1550
1594
|
[0.79.4]: https://github.com/VilovietaSEO/mcp-scraper/compare/v0.79.3...v0.79.4
|
package/README.md
CHANGED
|
@@ -175,7 +175,7 @@ Build the branded one-click bundle:
|
|
|
175
175
|
npm run build:mcpb
|
|
176
176
|
```
|
|
177
177
|
|
|
178
|
-
The generated bundle is written to `build/mcpb/mcp-scraper-<version>.mcpb` and copied to `public/downloads/` for the hosted download. The current public bundle is `https://mcpscraper.dev/downloads/mcp-scraper.mcpb` (`0.
|
|
178
|
+
The generated bundle is written to `build/mcpb/mcp-scraper-<version>.mcpb` and copied to `public/downloads/` for the hosted download. The current public bundle is `https://mcpscraper.dev/downloads/mcp-scraper.mcpb` (`0.83.1`, SHA-256 `57e9dfb6d83aa729cf90ca4654f72a4c8b16f219cbdbde737f72f23f36f3b46c`). Install it by opening or dragging it into Claude Desktop. Claude displays the `MCP Scraper` install card, icon, API-key configuration field, and manually curated current-release message from the bundle manifest.
|
|
179
179
|
|
|
180
180
|
The MCPB install exposes every tool — web-intelligence plus all `browser_*` tools — through the one `mcp-scraper` server.
|
|
181
181
|
|
|
@@ -337,7 +337,7 @@ The `mcp-scraper` server (and the MCPB bundle, which runs it) exposes both secti
|
|
|
337
337
|
|
|
338
338
|
All MCP tools return `structuredContent` with the IDs, URLs, CSV paths, transcripts, browser session handles, replay paths, artifacts, recipe fields, or blueprint fields needed by the next step, plus readable text content for compatibility. Runtime `tools/list` omits output schemas so strict clients can register the complete catalog; the generated developer manifest retains every canonical output schema for validation and typed SDK generation. All tools carry MCP annotations; file-writing tools such as replay downloads and annotations state their filesystem side effects.
|
|
339
339
|
|
|
340
|
-
The canonical tool inventory is generated at `docs/mcp-tool-manifest.generated.json`. The unified server exposes
|
|
340
|
+
The canonical tool inventory is generated at `docs/mcp-tool-manifest.generated.json`. The unified server exposes 363 tools: 262 scraper, browser, workflow, billing, connected-service, and personal-assistant tools plus 101 durable-memory tools. The scraper-side inventory includes complete Gmail selection, message, attachment, export, bulk-action, and Memory-import workflows; governed personal-assistant commands, messaging, approvals, grants, number setup, and execution readback; durable PAA starts and status; rendered site-content similarity; governed Local Sourcebook and Transparent Commons workflows; direct site-export reads; Editorial Reading Room and News Publisher templates; and production X-Ray setup, analytics, attribution, CRM policy and receipt, campaign, export, and scheduled-report tools. Successful evidence-compiled Local Sourcebook revisions publish automatically to their canonical `localsourcebook.com` category profile and review URLs; administrator controls handle exceptional rejection or unpublishing. Release verification compares the exact local and hosted tool-name sets, not only the count.
|
|
341
341
|
|
|
342
342
|
For contract parity, stdio and MCPB memory calls invoke the matching public tool on the hosted MCP Scraper `/mcp` endpoint. The hosted aggregate runtime owns MCP Scraper-specific billing, scheduling, credential, and in-process cutover policy; its internal `/memory/mcp-call` bridge is a fallback to the standalone memory service, not the public stdio execution path. Direct `mcp-memory` OAuth and stdio clients continue to use `memory.mcpscraper.dev` and must be verified as a separate dependent release surface.
|
|
343
343
|
|
package/dist/bin/api-server.js
CHANGED
|
@@ -17,8 +17,8 @@ loadDotEnv();
|
|
|
17
17
|
async function main() {
|
|
18
18
|
const [{ serve }, { app, personalAssistantProductionStartup }, { startWorker }, { migrate }] = await Promise.all([
|
|
19
19
|
import("@hono/node-server"),
|
|
20
|
-
import("../server-
|
|
21
|
-
import("../worker-
|
|
20
|
+
import("../server-XBUOLGZ3.js"),
|
|
21
|
+
import("../worker-FM3ZRDY4.js"),
|
|
22
22
|
import("../db-Z34LPZNR.js")
|
|
23
23
|
]);
|
|
24
24
|
const PORT = parseInt(process.env.PORT ?? "3001");
|
|
@@ -26,7 +26,7 @@ async function main() {
|
|
|
26
26
|
await personalAssistantProductionStartup;
|
|
27
27
|
await migrate();
|
|
28
28
|
if (process.env.ANALYTICS_DATABASE_URL) {
|
|
29
|
-
const { migrateAnalytics } = await import("../analytics-repository-
|
|
29
|
+
const { migrateAnalytics } = await import("../analytics-repository-GA6P5S5J.js");
|
|
30
30
|
await migrateAnalytics();
|
|
31
31
|
}
|
|
32
32
|
startWorker();
|
|
@@ -6,14 +6,20 @@ import {
|
|
|
6
6
|
openWorkflowReport,
|
|
7
7
|
runWorkflow,
|
|
8
8
|
workflowOutputBaseDir
|
|
9
|
-
} from "../chunk-
|
|
9
|
+
} from "../chunk-ACDNW7TG.js";
|
|
10
10
|
import "../chunk-RJ7JVYKU.js";
|
|
11
11
|
import "../chunk-OWF2JJKN.js";
|
|
12
|
-
import
|
|
12
|
+
import {
|
|
13
|
+
XRAY_INSTALL_CONTRACT,
|
|
14
|
+
XRAY_INSTALL_PROTOCOL_DIGEST,
|
|
15
|
+
XRAY_INSTALL_PROTOCOL_VERSION,
|
|
16
|
+
renderXraySetupPrompt,
|
|
17
|
+
xrayHostRecipe
|
|
18
|
+
} from "../chunk-BGD7U3AE.js";
|
|
13
19
|
import "../chunk-GGZEC22A.js";
|
|
14
20
|
import {
|
|
15
21
|
PACKAGE_VERSION
|
|
16
|
-
} from "../chunk-
|
|
22
|
+
} from "../chunk-62ROBG4P.js";
|
|
17
23
|
|
|
18
24
|
// src/cli/human-cli.ts
|
|
19
25
|
import { Command } from "commander";
|
|
@@ -38,26 +44,6 @@ function envConfig(options) {
|
|
|
38
44
|
if (options.browserProfileSaveChanges === true) env.BROWSER_AGENT_PROFILE_SAVE_CHANGES = "true";
|
|
39
45
|
return env;
|
|
40
46
|
}
|
|
41
|
-
function tomlInlineTable(value) {
|
|
42
|
-
const entries = Object.entries(value).map(([key, item]) => `${key} = ${JSON.stringify(item)}`);
|
|
43
|
-
return `{ ${entries.join(", ")} }`;
|
|
44
|
-
}
|
|
45
|
-
function renderCodexConfig(options = {}) {
|
|
46
|
-
return [
|
|
47
|
-
"[mcp_servers.mcp-scraper]",
|
|
48
|
-
'command = "npx"',
|
|
49
|
-
`args = ${JSON.stringify(combinedNpxArgs(options))}`,
|
|
50
|
-
`env = ${tomlInlineTable(envConfig(options))}`
|
|
51
|
-
].join("\n");
|
|
52
|
-
}
|
|
53
|
-
function renderClaudeCommand(options = {}) {
|
|
54
|
-
const env = Object.entries(envConfig(options)).map(([key, value]) => ` --env ${key}=${value}`);
|
|
55
|
-
return [
|
|
56
|
-
"claude mcp add mcp-scraper --scope user",
|
|
57
|
-
...env,
|
|
58
|
-
` -- npx ${combinedNpxArgs(options).join(" ")}`
|
|
59
|
-
].join(" \\\n");
|
|
60
|
-
}
|
|
61
47
|
function claudeMcpRemoveArgs() {
|
|
62
48
|
return ["mcp", "remove", "mcp-scraper", "-s", "user"];
|
|
63
49
|
}
|
|
@@ -101,28 +87,26 @@ function claudeMcpAddArgs(options = {}) {
|
|
|
101
87
|
}
|
|
102
88
|
function normalizeAgentHost(host) {
|
|
103
89
|
if (host === "claude-code") return "claude";
|
|
104
|
-
if (host === "claude" ||
|
|
105
|
-
throw new Error('Unknown host "' + host + '". Use: codex, claude, claude-code, or
|
|
90
|
+
if (host === "claude" || XRAY_INSTALL_CONTRACT.hosts.some((candidate) => candidate.id === host)) return host;
|
|
91
|
+
throw new Error('Unknown host "' + host + '". Use: codex, claude, claude-code, claude-desktop, cursor, windsurf, cline, or user-action-only');
|
|
106
92
|
}
|
|
107
|
-
function
|
|
108
|
-
return
|
|
109
|
-
mcpServers: {
|
|
110
|
-
"mcp-scraper": {
|
|
111
|
-
command: "npx",
|
|
112
|
-
args: combinedNpxArgs(options),
|
|
113
|
-
env: envConfig(options)
|
|
114
|
-
}
|
|
115
|
-
}
|
|
116
|
-
}, null, 2);
|
|
93
|
+
function agentHostCapability(host) {
|
|
94
|
+
return xrayHostRecipe(host === "claude" ? "claude-code" : host);
|
|
117
95
|
}
|
|
118
|
-
function renderAgentInstall(host,
|
|
96
|
+
function renderAgentInstall(host, _options = {}) {
|
|
119
97
|
const normalizedHost = normalizeAgentHost(host);
|
|
98
|
+
const recipe = agentHostCapability(normalizedHost);
|
|
120
99
|
const restart = "Restart the MCP client so it starts a fresh npx process.";
|
|
121
|
-
const applyCommand =
|
|
100
|
+
const applyCommand = 'MCP_SCRAPER_API_KEY="$MCP_SCRAPER_API_KEY" npx -y -p mcp-scraper@latest mcp-scraper-cli agent install claude --apply';
|
|
101
|
+
const protocol = `X-Ray install protocol: ${XRAY_INSTALL_PROTOCOL_VERSION} (${XRAY_INSTALL_PROTOCOL_DIGEST})`;
|
|
122
102
|
if (normalizedHost === "codex") {
|
|
123
103
|
return [
|
|
124
104
|
"# Codex MCP config",
|
|
125
|
-
|
|
105
|
+
protocol,
|
|
106
|
+
recipe.exactConfig,
|
|
107
|
+
"",
|
|
108
|
+
`Continuation: ${recipe.continuation}`,
|
|
109
|
+
`Rollback: ${recipe.rollback}`,
|
|
126
110
|
"",
|
|
127
111
|
restart
|
|
128
112
|
].join("\n");
|
|
@@ -130,20 +114,38 @@ function renderAgentInstall(host, options = {}) {
|
|
|
130
114
|
if (normalizedHost === "claude") {
|
|
131
115
|
return [
|
|
132
116
|
"# Claude Code command",
|
|
133
|
-
|
|
117
|
+
protocol,
|
|
118
|
+
recipe.exactConfig,
|
|
134
119
|
"",
|
|
135
120
|
"# One-command Claude Code setup",
|
|
136
121
|
applyCommand,
|
|
137
122
|
"",
|
|
123
|
+
`Continuation: ${recipe.continuation}`,
|
|
124
|
+
`Rollback: ${recipe.rollback}`,
|
|
125
|
+
"",
|
|
126
|
+
restart
|
|
127
|
+
].join("\n");
|
|
128
|
+
}
|
|
129
|
+
if (normalizedHost === "claude-desktop") {
|
|
130
|
+
return [
|
|
131
|
+
"# Claude Desktop config",
|
|
132
|
+
protocol,
|
|
133
|
+
recipe.exactConfig,
|
|
134
|
+
"",
|
|
135
|
+
"Desktop Extension: https://mcpscraper.dev/downloads/mcp-scraper.mcpb",
|
|
136
|
+
`Continuation: ${recipe.continuation}`,
|
|
137
|
+
`Rollback: ${recipe.rollback}`,
|
|
138
138
|
restart
|
|
139
139
|
].join("\n");
|
|
140
140
|
}
|
|
141
141
|
return [
|
|
142
|
-
|
|
143
|
-
|
|
142
|
+
`# ${recipe.label} MCP config`,
|
|
143
|
+
protocol,
|
|
144
|
+
recipe.exactConfig,
|
|
144
145
|
"",
|
|
145
|
-
|
|
146
|
-
|
|
146
|
+
`Continuation: ${recipe.continuation}`,
|
|
147
|
+
`Rollback: ${recipe.rollback}`,
|
|
148
|
+
...recipe.restartRequired ? ["", restart] : []
|
|
147
149
|
].join("\n");
|
|
148
150
|
}
|
|
149
151
|
|
|
@@ -288,6 +290,7 @@ function renderDoctor(output) {
|
|
|
288
290
|
|
|
289
291
|
// src/cli/prompts.ts
|
|
290
292
|
var AGENT_PROMPTS = {
|
|
293
|
+
"xray-setup": renderXraySetupPrompt(),
|
|
291
294
|
"agent-packet": [
|
|
292
295
|
"# MCP Scraper Agent Packet Prompt",
|
|
293
296
|
"",
|
|
@@ -561,7 +564,7 @@ function buildHumanCli() {
|
|
|
561
564
|
].join("\n"), false);
|
|
562
565
|
});
|
|
563
566
|
const agent = program.command("agent").description("Generate AI-agent install configs and workflow prompts.");
|
|
564
|
-
agent.command("install <host>").description("Print or apply install/config instructions for
|
|
567
|
+
agent.command("install <host>").description("Print or apply install/config instructions for Codex, Claude, Cursor, Windsurf, Cline, or a handoff-only MCP host.").option("--api-key <key>", "API key to place in generated config").option("--package <spec>", "npm package spec", "mcp-scraper@latest").option("--browser-profile <name>", "Default saved browser profile for browser_open sessions.").option("--save-browser-profile-changes", "Persist cookies and browser storage back to the named browser profile when sessions close").option("--apply", "Apply the config to this client when supported. For Claude Code, this upserts the user-scope mcp-scraper server.").option("--json", "Print machine-readable JSON").action(async (hostInput, opts) => {
|
|
565
568
|
const host = normalizeAgentHost(hostInput);
|
|
566
569
|
const apiKey = opts.apiKey ?? process.env.MCP_SCRAPER_API_KEY;
|
|
567
570
|
if (opts.apply) {
|
|
@@ -631,7 +634,12 @@ ${result.list}` : ""
|
|
|
631
634
|
browserProfileName: opts.browserProfile,
|
|
632
635
|
browserProfileSaveChanges: opts.saveBrowserProfileChanges
|
|
633
636
|
});
|
|
634
|
-
writeOutput(opts.json ? {
|
|
637
|
+
writeOutput(opts.json ? {
|
|
638
|
+
host,
|
|
639
|
+
protocolVersion: XRAY_INSTALL_PROTOCOL_VERSION,
|
|
640
|
+
protocolDigest: XRAY_INSTALL_PROTOCOL_DIGEST,
|
|
641
|
+
text
|
|
642
|
+
} : text, opts.json);
|
|
635
643
|
});
|
|
636
644
|
agent.command("prompt [name]").description("Print an agent prompt template.").option("--json", "Print machine-readable JSON").action((name, opts) => {
|
|
637
645
|
if (!name) {
|
|
@@ -639,7 +647,14 @@ ${result.list}` : ""
|
|
|
639
647
|
return;
|
|
640
648
|
}
|
|
641
649
|
const text = renderPrompt(name);
|
|
642
|
-
writeOutput(opts.json ? {
|
|
650
|
+
writeOutput(opts.json ? {
|
|
651
|
+
name,
|
|
652
|
+
...name === "xray-setup" ? {
|
|
653
|
+
protocolVersion: XRAY_INSTALL_PROTOCOL_VERSION,
|
|
654
|
+
protocolDigest: XRAY_INSTALL_PROTOCOL_DIGEST
|
|
655
|
+
} : {},
|
|
656
|
+
text
|
|
657
|
+
} : text, opts.json);
|
|
643
658
|
});
|
|
644
659
|
const workflow = program.command("workflow").description("Run named SEO workflows.");
|
|
645
660
|
workflow.command("list").description("List available workflows.").option("--json", "Print machine-readable JSON").action((opts) => {
|
|
@@ -1,19 +1,19 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
2
|
import {
|
|
3
3
|
runMcpScraperStdio
|
|
4
|
-
} from "../chunk-
|
|
5
|
-
import "../chunk-
|
|
4
|
+
} from "../chunk-J2CRSYL3.js";
|
|
5
|
+
import "../chunk-W54QRM7I.js";
|
|
6
6
|
import "../chunk-PGJQDMC2.js";
|
|
7
7
|
import "../chunk-DNM65UCK.js";
|
|
8
8
|
import "../chunk-T3MZISOF.js";
|
|
9
9
|
import "../chunk-6HAV7LCE.js";
|
|
10
|
-
import "../chunk-
|
|
10
|
+
import "../chunk-4NZ6JFK5.js";
|
|
11
11
|
import "../chunk-OM7HVEJ3.js";
|
|
12
12
|
import "../chunk-OPQIGAFB.js";
|
|
13
|
-
import "../chunk-
|
|
14
|
-
import "../chunk-
|
|
13
|
+
import "../chunk-UXVT7FWW.js";
|
|
14
|
+
import "../chunk-BGD7U3AE.js";
|
|
15
15
|
import "../chunk-GGZEC22A.js";
|
|
16
|
-
import "../chunk-
|
|
16
|
+
import "../chunk-62ROBG4P.js";
|
|
17
17
|
import "../chunk-YXNDOQXN.js";
|
|
18
18
|
|
|
19
19
|
// src/mcp/thorbit-restricted-tool-surface.ts
|
|
@@ -1,10 +1,10 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
2
|
import {
|
|
3
3
|
renderInstallTerminal
|
|
4
|
-
} from "../chunk-
|
|
4
|
+
} from "../chunk-UXVT7FWW.js";
|
|
5
5
|
import {
|
|
6
6
|
PACKAGE_VERSION
|
|
7
|
-
} from "../chunk-
|
|
7
|
+
} from "../chunk-62ROBG4P.js";
|
|
8
8
|
|
|
9
9
|
// bin/mcp-scraper-install.ts
|
|
10
10
|
var noColor = process.argv.includes("--no-color") || process.env.NO_COLOR !== void 0 || process.env.FORCE_COLOR === "0" || !process.stdout.isTTY;
|
|
@@ -1,19 +1,19 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
2
|
import {
|
|
3
3
|
runMcpScraperStdio
|
|
4
|
-
} from "../chunk-
|
|
5
|
-
import "../chunk-
|
|
4
|
+
} from "../chunk-J2CRSYL3.js";
|
|
5
|
+
import "../chunk-W54QRM7I.js";
|
|
6
6
|
import "../chunk-PGJQDMC2.js";
|
|
7
7
|
import "../chunk-DNM65UCK.js";
|
|
8
8
|
import "../chunk-T3MZISOF.js";
|
|
9
9
|
import "../chunk-6HAV7LCE.js";
|
|
10
|
-
import "../chunk-
|
|
10
|
+
import "../chunk-4NZ6JFK5.js";
|
|
11
11
|
import "../chunk-OM7HVEJ3.js";
|
|
12
12
|
import "../chunk-OPQIGAFB.js";
|
|
13
|
-
import "../chunk-
|
|
14
|
-
import "../chunk-
|
|
13
|
+
import "../chunk-UXVT7FWW.js";
|
|
14
|
+
import "../chunk-BGD7U3AE.js";
|
|
15
15
|
import "../chunk-GGZEC22A.js";
|
|
16
|
-
import "../chunk-
|
|
16
|
+
import "../chunk-62ROBG4P.js";
|
|
17
17
|
import "../chunk-YXNDOQXN.js";
|
|
18
18
|
|
|
19
19
|
// bin/mcp-stdio-server.ts
|
|
@@ -145,6 +145,204 @@ function extractContactEvidence(html, pageUrl) {
|
|
|
145
145
|
};
|
|
146
146
|
}
|
|
147
147
|
|
|
148
|
+
// src/mcp/xray-install-contract.ts
|
|
149
|
+
import { createHash } from "crypto";
|
|
150
|
+
var XRAY_INSTALL_PROTOCOL_VERSION = "2.0.0";
|
|
151
|
+
var XRAY_SETUP_TOOL = "analytics_setup_site";
|
|
152
|
+
var XRAY_TRIGGER_PHRASE = "Install X-Ray";
|
|
153
|
+
var XRAY_CANONICAL_GOAL = "Install X-Ray and tell me what drives customers";
|
|
154
|
+
var CONTINUATION = `Call ${XRAY_SETUP_TOOL} with { "startUrl": "<site URL>", "goal": "${XRAY_CANONICAL_GOAL}" }; on every later call, reuse the returned setupId and revision.`;
|
|
155
|
+
var LOCAL_JSON = JSON.stringify({
|
|
156
|
+
mcpServers: {
|
|
157
|
+
"mcp-scraper": {
|
|
158
|
+
command: "npx",
|
|
159
|
+
args: ["-y", "--package", "mcp-scraper@latest", "mcp-scraper"],
|
|
160
|
+
env: { MCP_SCRAPER_API_KEY: "${env:MCP_SCRAPER_API_KEY}" }
|
|
161
|
+
}
|
|
162
|
+
}
|
|
163
|
+
}, null, 2);
|
|
164
|
+
var HOSTED_JSON = JSON.stringify({
|
|
165
|
+
mcpServers: {
|
|
166
|
+
"mcp-scraper": { url: "https://mcpscraper.dev/mcp" }
|
|
167
|
+
}
|
|
168
|
+
}, null, 2);
|
|
169
|
+
var XRAY_INSTALL_CONTRACT_PAYLOAD = {
|
|
170
|
+
schema: "https://mcpscraper.dev/schemas/xray-install-contract-v1.json",
|
|
171
|
+
protocolVersion: XRAY_INSTALL_PROTOCOL_VERSION,
|
|
172
|
+
product: "Thorbit X-Ray",
|
|
173
|
+
triggerPhrases: [XRAY_TRIGGER_PHRASE, "Install X-Ray on this site and tell me what drives customers"],
|
|
174
|
+
canonicalUrls: {
|
|
175
|
+
markdown: "https://mcpscraper.dev/agent-install.md",
|
|
176
|
+
json: "https://mcpscraper.dev/.well-known/xray-install.json",
|
|
177
|
+
hostedMcp: "https://mcpscraper.dev/mcp"
|
|
178
|
+
},
|
|
179
|
+
facade: {
|
|
180
|
+
tool: XRAY_SETUP_TOOL,
|
|
181
|
+
initialCall: { startUrl: "<site URL>", goal: XRAY_CANONICAL_GOAL },
|
|
182
|
+
continuationRule: "Treat setupId plus revision as the durable continuation handle. Reuse both after approval, runtime publication, verification, retry, or return."
|
|
183
|
+
},
|
|
184
|
+
topologies: {
|
|
185
|
+
local_stdio: {
|
|
186
|
+
label: "Local npm MCP process",
|
|
187
|
+
authority: "The client starts MCP Scraper locally. After the one-time Pixel insertion, X-Ray setup publishes only owner-confirmed runtime configuration.",
|
|
188
|
+
secretHandling: "Set MCP_SCRAPER_API_KEY in the local environment or client secret field. Never paste it into chat or commit it."
|
|
189
|
+
},
|
|
190
|
+
hosted_remote: {
|
|
191
|
+
label: "Hosted authenticated MCP",
|
|
192
|
+
authority: "The hosted MCP inspects approved public pages and publishes owner-confirmed X-Ray runtime configuration. It cannot read or write a caller-local path.",
|
|
193
|
+
secretHandling: "Use the MCP OAuth/connection flow. Never put a credential in a URL, prompt, or config shown in chat."
|
|
194
|
+
}
|
|
195
|
+
},
|
|
196
|
+
hosts: [
|
|
197
|
+
{
|
|
198
|
+
id: "codex",
|
|
199
|
+
label: "Codex",
|
|
200
|
+
supportedTopologies: ["local_stdio", "hosted_remote"],
|
|
201
|
+
mcpConfigMode: "toml",
|
|
202
|
+
workspaceRead: true,
|
|
203
|
+
workspaceWrite: true,
|
|
204
|
+
restartRequired: true,
|
|
205
|
+
exactConfig: '[mcp_servers.mcp-scraper]\ncommand = "npx"\nargs = ["-y", "--package", "mcp-scraper@latest", "mcp-scraper"]\nenv_vars = ["MCP_SCRAPER_API_KEY"]',
|
|
206
|
+
continuation: CONTINUATION,
|
|
207
|
+
rollback: "Remove the mcp_servers.mcp-scraper table, then restart Codex. Pixel removal is a separate owner-controlled site action."
|
|
208
|
+
},
|
|
209
|
+
{
|
|
210
|
+
id: "claude-code",
|
|
211
|
+
label: "Claude Code",
|
|
212
|
+
supportedTopologies: ["local_stdio", "hosted_remote"],
|
|
213
|
+
mcpConfigMode: "command",
|
|
214
|
+
workspaceRead: true,
|
|
215
|
+
workspaceWrite: true,
|
|
216
|
+
restartRequired: true,
|
|
217
|
+
exactConfig: 'claude mcp add mcp-scraper --scope user --env MCP_SCRAPER_API_KEY="$MCP_SCRAPER_API_KEY" -- npx -y --package mcp-scraper@latest mcp-scraper',
|
|
218
|
+
continuation: CONTINUATION,
|
|
219
|
+
rollback: "Run: claude mcp remove mcp-scraper -s user. Pixel removal is a separate owner-controlled site action."
|
|
220
|
+
},
|
|
221
|
+
{
|
|
222
|
+
id: "claude-desktop",
|
|
223
|
+
label: "Claude Desktop",
|
|
224
|
+
supportedTopologies: ["local_stdio"],
|
|
225
|
+
mcpConfigMode: "desktop_extension",
|
|
226
|
+
workspaceRead: false,
|
|
227
|
+
workspaceWrite: false,
|
|
228
|
+
restartRequired: true,
|
|
229
|
+
exactConfig: "Install https://mcpscraper.dev/downloads/mcp-scraper.mcpb and enter the API key only in Claude Desktop's protected configuration field.",
|
|
230
|
+
continuation: CONTINUATION,
|
|
231
|
+
rollback: "Remove the MCP Scraper extension in Claude Desktop. Pixel removal is a separate owner-controlled site action."
|
|
232
|
+
},
|
|
233
|
+
{
|
|
234
|
+
id: "cursor",
|
|
235
|
+
label: "Cursor",
|
|
236
|
+
supportedTopologies: ["local_stdio", "hosted_remote"],
|
|
237
|
+
mcpConfigMode: "json",
|
|
238
|
+
workspaceRead: true,
|
|
239
|
+
workspaceWrite: true,
|
|
240
|
+
restartRequired: true,
|
|
241
|
+
exactConfig: LOCAL_JSON,
|
|
242
|
+
continuation: CONTINUATION,
|
|
243
|
+
rollback: "Remove mcp-scraper from .cursor/mcp.json (or the user MCP settings), then restart Cursor."
|
|
244
|
+
},
|
|
245
|
+
{
|
|
246
|
+
id: "windsurf",
|
|
247
|
+
label: "Windsurf",
|
|
248
|
+
supportedTopologies: ["local_stdio"],
|
|
249
|
+
mcpConfigMode: "json",
|
|
250
|
+
workspaceRead: true,
|
|
251
|
+
workspaceWrite: true,
|
|
252
|
+
restartRequired: true,
|
|
253
|
+
exactConfig: LOCAL_JSON,
|
|
254
|
+
continuation: CONTINUATION,
|
|
255
|
+
rollback: "Remove mcp-scraper from the Windsurf MCP configuration, then restart Windsurf."
|
|
256
|
+
},
|
|
257
|
+
{
|
|
258
|
+
id: "cline",
|
|
259
|
+
label: "Cline",
|
|
260
|
+
supportedTopologies: ["local_stdio"],
|
|
261
|
+
mcpConfigMode: "json",
|
|
262
|
+
workspaceRead: true,
|
|
263
|
+
workspaceWrite: true,
|
|
264
|
+
restartRequired: true,
|
|
265
|
+
exactConfig: LOCAL_JSON,
|
|
266
|
+
continuation: CONTINUATION,
|
|
267
|
+
rollback: "Remove mcp-scraper from Cline MCP Servers, then restart the extension host."
|
|
268
|
+
},
|
|
269
|
+
{
|
|
270
|
+
id: "user-action-only",
|
|
271
|
+
label: "User-action-only MCP host",
|
|
272
|
+
supportedTopologies: ["hosted_remote"],
|
|
273
|
+
mcpConfigMode: "handoff_only",
|
|
274
|
+
workspaceRead: false,
|
|
275
|
+
workspaceWrite: false,
|
|
276
|
+
restartRequired: false,
|
|
277
|
+
exactConfig: HOSTED_JSON,
|
|
278
|
+
continuation: CONTINUATION,
|
|
279
|
+
rollback: "Disconnect MCP Scraper from the host. Pixel removal remains an owner-controlled site action."
|
|
280
|
+
}
|
|
281
|
+
],
|
|
282
|
+
tagBootstrap: {
|
|
283
|
+
requirement: "one_global_pixel_once",
|
|
284
|
+
ongoingConfiguration: "xray_runtime_manifest",
|
|
285
|
+
sourceAccessRequiredAfterInstall: false,
|
|
286
|
+
supportedInsertionPaths: ["manual", "tag_manager", "cms_plugin", "coding_host"]
|
|
287
|
+
},
|
|
288
|
+
workflow: [
|
|
289
|
+
"Resolve Thorbit X-Ray entitlement and the target site without consuming MCP Scraper Credits.",
|
|
290
|
+
"Inspect the whole approved public site; enumerate distinct renderers, routes, and Pixel coverage.",
|
|
291
|
+
"Infer the business model and customer outcome, then propose one privacy-safe measurement plan.",
|
|
292
|
+
"If the Pixel is absent, return the canonical one-time tag and exact uncovered surfaces; do not claim ongoing control there.",
|
|
293
|
+
"Publish an approved measurement revision as a digest-bound runtime manifest; source or repository access is not required.",
|
|
294
|
+
"Verify route coverage, consent, PageViews, manifest convergence, bounded rule matches, and persisted definition/version evidence.",
|
|
295
|
+
"Require a separate exact-revision approval before any canary or real-visitor experiment activation.",
|
|
296
|
+
"Return first customer-journey answers when sufficient, or exact evidence-readiness thresholds when not.",
|
|
297
|
+
"On \u201Cimprove\u201D, propose one highest-value uncovered outcome, require approval when needed, and reverify."
|
|
298
|
+
],
|
|
299
|
+
safetyRules: [
|
|
300
|
+
"Never invent or hand-author tracking code; use the canonical product-authored Pixel snippet returned by analytics_setup_site.",
|
|
301
|
+
"Never ask for or print API keys, OAuth tokens, signed URLs, cookies, or other credentials.",
|
|
302
|
+
"Never call ad platforms directly or silently activate events, definitions, funnels, or delivery destinations.",
|
|
303
|
+
"Never treat a CTA click as a customer, sale, or revenue event without verified outcome evidence.",
|
|
304
|
+
"Never claim full-domain installation while an in-scope route or renderer is missing, excluded without approval, or unverifiable.",
|
|
305
|
+
"Never treat event-name presence as verification; require persisted definition ID and version evidence."
|
|
306
|
+
],
|
|
307
|
+
terminalStates: ["complete", "tag_required", "input_required", "approval_required", "runtime_convergence_required", "outcome_connection_required", "not_ready", "error"]
|
|
308
|
+
};
|
|
309
|
+
function canonicalJson(value) {
|
|
310
|
+
if (Array.isArray(value)) return `[${value.map(canonicalJson).join(",")}]`;
|
|
311
|
+
if (value && typeof value === "object") {
|
|
312
|
+
return `{${Object.entries(value).sort(([left], [right]) => left.localeCompare(right)).map(([key, item]) => `${JSON.stringify(key)}:${canonicalJson(item)}`).join(",")}}`;
|
|
313
|
+
}
|
|
314
|
+
return JSON.stringify(value);
|
|
315
|
+
}
|
|
316
|
+
function xrayInstallDigestPayload() {
|
|
317
|
+
return canonicalJson(XRAY_INSTALL_CONTRACT_PAYLOAD);
|
|
318
|
+
}
|
|
319
|
+
var XRAY_INSTALL_PROTOCOL_DIGEST = `sha256:${createHash("sha256").update(xrayInstallDigestPayload(), "utf8").digest("hex")}`;
|
|
320
|
+
var XRAY_INSTALL_CONTRACT = {
|
|
321
|
+
...XRAY_INSTALL_CONTRACT_PAYLOAD,
|
|
322
|
+
protocolDigest: XRAY_INSTALL_PROTOCOL_DIGEST
|
|
323
|
+
};
|
|
324
|
+
function xrayHostRecipe(host) {
|
|
325
|
+
const recipe = XRAY_INSTALL_CONTRACT.hosts.find((candidate) => candidate.id === host);
|
|
326
|
+
if (!recipe) throw new Error(`Unknown X-Ray host: ${host}`);
|
|
327
|
+
return recipe;
|
|
328
|
+
}
|
|
329
|
+
function renderXraySetupPrompt() {
|
|
330
|
+
return [
|
|
331
|
+
"# Install X-Ray",
|
|
332
|
+
"",
|
|
333
|
+
`Protocol: ${XRAY_INSTALL_PROTOCOL_VERSION} (${XRAY_INSTALL_PROTOCOL_DIGEST})`,
|
|
334
|
+
"",
|
|
335
|
+
`Use ${XRAY_SETUP_TOOL} as the single conversational setup facade. Ask for the target site URL if it is missing, then call it with the goal \u201C${XRAY_CANONICAL_GOAL}\u201D.`,
|
|
336
|
+
"",
|
|
337
|
+
"Continue the same setup with its setupId and revision. Confirm one global Pixel across all public rendering surfaces; present one measurement plan; publish it as a digest-bound runtime manifest after exact-revision approval; verify manifest and rule versions; then return first customer-journey answers or explicit readiness thresholds. GitHub or repository access is not required after the tag is installed.",
|
|
338
|
+
"",
|
|
339
|
+
"Do not request pasted credentials, invent tracking code, silently activate measurement, call ad platforms directly, or claim customers/revenue from CTA clicks."
|
|
340
|
+
].join("\n");
|
|
341
|
+
}
|
|
342
|
+
function xrayInstallJson(contract = XRAY_INSTALL_CONTRACT) {
|
|
343
|
+
return JSON.parse(JSON.stringify(contract));
|
|
344
|
+
}
|
|
345
|
+
|
|
148
346
|
// src/leads/contact-enrichment.ts
|
|
149
347
|
var GENERIC_EMAIL_PREFIXES = /* @__PURE__ */ new Set([
|
|
150
348
|
"admin",
|
|
@@ -289,5 +487,13 @@ export {
|
|
|
289
487
|
selectPrimaryEmail,
|
|
290
488
|
contactAttemptUrls,
|
|
291
489
|
buildSerpEmailQuery,
|
|
292
|
-
extractSerpEmailEvidence
|
|
490
|
+
extractSerpEmailEvidence,
|
|
491
|
+
XRAY_INSTALL_PROTOCOL_VERSION,
|
|
492
|
+
XRAY_SETUP_TOOL,
|
|
493
|
+
XRAY_CANONICAL_GOAL,
|
|
494
|
+
XRAY_INSTALL_PROTOCOL_DIGEST,
|
|
495
|
+
XRAY_INSTALL_CONTRACT,
|
|
496
|
+
xrayHostRecipe,
|
|
497
|
+
renderXraySetupPrompt,
|
|
498
|
+
xrayInstallJson
|
|
293
499
|
};
|