mcp-scraper 0.88.1 → 0.89.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (87) hide show
  1. package/CHANGELOG.md +43 -15
  2. package/README.md +6 -5
  3. package/THIRD_PARTY_NOTICES.html +203 -0
  4. package/dist/analytics-repository-2BMT5JNE.js +1 -0
  5. package/dist/bin/api-server.js +2 -41
  6. package/dist/bin/mcp-scraper-cli.js +39 -756
  7. package/dist/bin/mcp-scraper-core.js +1 -60
  8. package/dist/bin/mcp-scraper-install.js +2 -25
  9. package/dist/bin/mcp-stdio-server.js +1 -19
  10. package/dist/bin/paa-harvest.js +1 -41
  11. package/dist/chunk-3GP5CYZX.js +1 -0
  12. package/dist/chunk-4AI7DOS7.js +59 -0
  13. package/dist/chunk-4FROKQJN.js +1 -0
  14. package/dist/chunk-7YGVI5J4.js +21710 -0
  15. package/dist/chunk-CCYWSJNG.js +16 -0
  16. package/dist/chunk-CFI6CXIV.js +182 -0
  17. package/dist/chunk-E2WRWV3A.js +1 -0
  18. package/dist/chunk-GMWKPIYX.js +1172 -0
  19. package/dist/chunk-HDPYG3XV.js +102 -0
  20. package/dist/chunk-HE45FFBU.js +1 -0
  21. package/dist/chunk-HUV2WTRW.js +1 -0
  22. package/dist/chunk-KJQXUZ4Y.js +4 -0
  23. package/dist/chunk-L4CGLFPU.js +4 -0
  24. package/dist/chunk-M22MM4N4.js +84 -0
  25. package/dist/chunk-MASR22K4.js +73 -0
  26. package/dist/chunk-MZN4U5BL.js +1 -0
  27. package/dist/chunk-PUHFVA7P.js +1280 -0
  28. package/dist/chunk-QPWPR5XG.js +10 -0
  29. package/dist/chunk-TMB56NCA.js +1 -0
  30. package/dist/chunk-TXENITMS.js +20 -0
  31. package/dist/chunk-W2BVJ7S2.js +13 -0
  32. package/dist/chunk-WO3N5FH2.js +5 -0
  33. package/dist/chunk-WSCGYRWA.js +2595 -0
  34. package/dist/chunk-X54CQLK2.js +1 -0
  35. package/dist/chunk-XLWNEVUZ.js +27 -0
  36. package/dist/chunk-XPZVJIZ2.js +100 -0
  37. package/dist/chunk-YQZGZBB4.js +1 -0
  38. package/dist/chunk-Z2QGQJS2.js +1 -0
  39. package/dist/db-F2MX63GI.js +1 -0
  40. package/dist/extract-bundle-SNUIHM3J.js +26 -0
  41. package/dist/gmail-service-BZ3H75XC.js +1 -0
  42. package/dist/index.cjs +21750 -6045
  43. package/dist/index.d.cts +14 -14
  44. package/dist/index.d.ts +14 -14
  45. package/dist/index.js +18 -315
  46. package/dist/lead-list-enrichment-repository-S2H3U7T7.js +1 -0
  47. package/dist/location-data-repository-OTWHWMV6.js +1 -0
  48. package/dist/server-RFR2A5UJ.js +7303 -0
  49. package/dist/site-extract-repository-SE776XDC.js +1 -0
  50. package/dist/worker-XUDSM3AL.js +1 -0
  51. package/package.json +17 -124
  52. package/dist/analytics-repository-GGJJCVVP.js +0 -194
  53. package/dist/chunk-4QMUF6XM.js +0 -1013
  54. package/dist/chunk-65OWZPWE.js +0 -224
  55. package/dist/chunk-6HAV7LCE.js +0 -265
  56. package/dist/chunk-ABF2CGOZ.js +0 -113
  57. package/dist/chunk-C5Z4OFKW.js +0 -404
  58. package/dist/chunk-DNM65UCK.js +0 -299
  59. package/dist/chunk-EG2PDKD7.js +0 -33134
  60. package/dist/chunk-EQGTEHLZ.js +0 -592
  61. package/dist/chunk-F5GQJWZU.js +0 -732
  62. package/dist/chunk-GGZEC22A.js +0 -215
  63. package/dist/chunk-GXBZXWXB.js +0 -184
  64. package/dist/chunk-HIMFLGLV.js +0 -6
  65. package/dist/chunk-IHXAXYIS.js +0 -843
  66. package/dist/chunk-K3Z5AQYE.js +0 -683
  67. package/dist/chunk-MZDNZQWT.js +0 -2078
  68. package/dist/chunk-NVUKO5NN.js +0 -256
  69. package/dist/chunk-OM7HVEJ3.js +0 -26
  70. package/dist/chunk-OPQIGAFB.js +0 -286
  71. package/dist/chunk-OZJMVCDK.js +0 -16
  72. package/dist/chunk-P7FWOMU7.js +0 -505
  73. package/dist/chunk-PGJQDMC2.js +0 -383
  74. package/dist/chunk-RJ7JVYKU.js +0 -68
  75. package/dist/chunk-S24LFPL7.js +0 -5262
  76. package/dist/chunk-T3MZISOF.js +0 -240
  77. package/dist/chunk-UZPTGUDV.js +0 -1915
  78. package/dist/chunk-X623GTBV.js +0 -8290
  79. package/dist/chunk-YXNDOQXN.js +0 -4018
  80. package/dist/db-Z34LPZNR.js +0 -284
  81. package/dist/extract-bundle-565SBZCR.js +0 -1003
  82. package/dist/gmail-service-E6ALS7JG.js +0 -25
  83. package/dist/lead-list-enrichment-repository-36RPVV6N.js +0 -67
  84. package/dist/location-data-repository-WPRG62GE.js +0 -34
  85. package/dist/server-JCJML2EW.js +0 -86602
  86. package/dist/site-extract-repository-VYFZASPU.js +0 -69
  87. package/dist/worker-LDCAULWL.js +0 -146
@@ -1,224 +0,0 @@
1
- import {
2
- HttpMcpToolExecutor,
3
- MEMORY_TOOL_REGISTRY,
4
- SERVER_INSTRUCTIONS,
5
- ScheduledResultsMcpExecutor,
6
- hashOwnerId,
7
- installInputFieldDescriptions,
8
- permitsLocalNetworkAccess,
9
- registerBrowserAgentMcpTools,
10
- registerMemoryMcpTools,
11
- registerPaaExtractorMcpTools,
12
- registerScheduledResultsMcpTools,
13
- registerSerpIntelligenceCaptureTools,
14
- resolveDeploymentProfile
15
- } from "./chunk-EG2PDKD7.js";
16
- import {
17
- renderInstallTerminal
18
- } from "./chunk-ABF2CGOZ.js";
19
- import {
20
- PACKAGE_VERSION
21
- } from "./chunk-HIMFLGLV.js";
22
-
23
- // src/mcp/stdio-runtime.ts
24
- import { readFileSync } from "fs";
25
- import { homedir } from "os";
26
- import { join } from "path";
27
- import { serveStdio } from "@modelcontextprotocol/server/stdio";
28
- import { McpServer } from "@modelcontextprotocol/server";
29
-
30
- // src/mcp/hosted-memory-mcp-tool-executor.ts
31
- var publicToolNameByUpstreamName = new Map(
32
- MEMORY_TOOL_REGISTRY.map((schema) => [schema.upstreamName, schema.id])
33
- );
34
- function parseJsonRpcEnvelope(text) {
35
- const payloads = text.split("\n").filter((line) => line.startsWith("data:")).map((line) => line.slice(5).trim()).filter(Boolean);
36
- if (!payloads.length) return JSON.parse(text);
37
- for (const payload of payloads) {
38
- const parsed = JSON.parse(payload);
39
- if (parsed.result || parsed.error) return parsed;
40
- }
41
- throw new Error("hosted MCP returned no JSON-RPC result");
42
- }
43
- function errorResult(message) {
44
- return {
45
- content: [{ type: "text", text: JSON.stringify({ ok: false, error: message }) }],
46
- isError: true
47
- };
48
- }
49
- var HostedMemoryMcpToolExecutor = class {
50
- baseUrl;
51
- apiKey;
52
- constructor(baseUrl, apiKey) {
53
- this.baseUrl = baseUrl.replace(/\/$/, "");
54
- this.apiKey = apiKey;
55
- }
56
- async callMemoryTool(upstreamName, args) {
57
- const publicToolName = publicToolNameByUpstreamName.get(upstreamName);
58
- if (!publicToolName) return errorResult(`unknown memory tool: ${upstreamName}`);
59
- try {
60
- const res = await fetch(`${this.baseUrl}/mcp`, {
61
- method: "POST",
62
- headers: {
63
- "content-type": "application/json",
64
- accept: "application/json, text/event-stream",
65
- "x-api-key": this.apiKey
66
- },
67
- body: JSON.stringify({
68
- jsonrpc: "2.0",
69
- id: `memory:${publicToolName}`,
70
- method: "tools/call",
71
- params: { name: publicToolName, arguments: args }
72
- })
73
- });
74
- const text = await res.text();
75
- if (!res.ok) return errorResult(`hosted memory ${publicToolName} failed (HTTP ${res.status})`);
76
- const envelope = parseJsonRpcEnvelope(text);
77
- if (envelope.error) {
78
- return errorResult(envelope.error.message ?? `hosted memory ${publicToolName} failed`);
79
- }
80
- return envelope.result ?? errorResult(`hosted memory ${publicToolName} returned no result`);
81
- } catch (err) {
82
- return errorResult(err instanceof Error ? err.message : `hosted memory ${publicToolName} call failed`);
83
- }
84
- }
85
- };
86
-
87
- // src/mcp/exact-tool-registration.ts
88
- function installExactToolRegistrationGuard(server, requiredToolNames) {
89
- if (requiredToolNames.length === 0) {
90
- throw new Error("Restricted MCP tool allowlist must contain at least one tool name");
91
- }
92
- const required = /* @__PURE__ */ new Set();
93
- for (const rawName of requiredToolNames) {
94
- if (typeof rawName !== "string" || rawName.trim() !== rawName || rawName.length === 0) {
95
- throw new Error("Restricted MCP tool allowlist contains a malformed tool name");
96
- }
97
- if (required.has(rawName)) {
98
- throw new Error(`Restricted MCP tool allowlist contains duplicate tool name: ${rawName}`);
99
- }
100
- required.add(rawName);
101
- }
102
- const registered = /* @__PURE__ */ new Set();
103
- const mutableServer = server;
104
- const registerTool = mutableServer.registerTool.bind(server);
105
- mutableServer.registerTool = (name, config, callback) => {
106
- if (!required.has(name)) return void 0;
107
- registered.add(name);
108
- return registerTool(name, config, callback);
109
- };
110
- return {
111
- assertComplete() {
112
- const missing = [...required].filter((name) => !registered.has(name));
113
- if (missing.length > 0) {
114
- throw new Error(`Restricted MCP tool allowlist contains unknown or unavailable tool names: ${missing.join(", ")}`);
115
- }
116
- },
117
- registeredToolNames() {
118
- return [...registered];
119
- }
120
- };
121
- }
122
-
123
- // src/mcp/stdio-runtime.ts
124
- var ALL_STDIO_TOOLSETS = /* @__PURE__ */ new Set([
125
- "paa",
126
- "serp",
127
- "browser-agent",
128
- "scheduled-results",
129
- "memory"
130
- ]);
131
- function readApiKeyFile() {
132
- const explicitPath = process.env.MCP_SCRAPER_KEY_PATH?.trim();
133
- const fileNames = [explicitPath, join(homedir(), ".mcp-scraper-key")].filter(Boolean);
134
- for (const fileName of fileNames) {
135
- try {
136
- const value = readFileSync(fileName, "utf8").trim();
137
- if (value) return value;
138
- } catch {
139
- }
140
- }
141
- return void 0;
142
- }
143
- function buildStdioServer(requiredApiKey, options = {}) {
144
- const toolsets = options.toolsets ?? ALL_STDIO_TOOLSETS;
145
- const baseUrl = process.env.MCP_SCRAPER_BASE_URL?.trim() || process.env.MCP_BASE_URL?.trim() || "https://mcpscraper.dev";
146
- const deploymentProfile = resolveDeploymentProfile();
147
- const localNetworkAccess = permitsLocalNetworkAccess({
148
- deploymentProfile,
149
- transportProfile: "stdio",
150
- baseUrl,
151
- explicitlyEnabled: process.env.MCP_SCRAPER_ALLOW_PRIVATE_NETWORK === "1"
152
- });
153
- const consoleBaseUrl = process.env.BROWSER_AGENT_CONSOLE_URL?.trim() || baseUrl;
154
- const server = new McpServer({ name: "mcp-scraper", version: PACKAGE_VERSION }, {
155
- instructions: SERVER_INSTRUCTIONS,
156
- cacheHints: {
157
- "server/discover": { ttlMs: 3e5, cacheScope: "private" },
158
- "tools/list": { ttlMs: 3e5, cacheScope: "private" },
159
- "prompts/list": { ttlMs: 3e5, cacheScope: "private" },
160
- "resources/list": { ttlMs: 3e5, cacheScope: "private" },
161
- "resources/templates/list": { ttlMs: 3e5, cacheScope: "private" },
162
- "resources/read": { ttlMs: 6e4, cacheScope: "private" }
163
- }
164
- });
165
- installInputFieldDescriptions(server);
166
- const registrationGuard = options.allowedToolNames ? installExactToolRegistrationGuard(server, options.allowedToolNames) : void 0;
167
- const httpExecutor = options.httpExecutor ?? new HttpMcpToolExecutor(baseUrl, requiredApiKey, { localNetworkAccess });
168
- if (toolsets.has("paa")) {
169
- registerPaaExtractorMcpTools(server, httpExecutor, {
170
- ownerId: hashOwnerId(requiredApiKey),
171
- deploymentProfile,
172
- transportProfile: "stdio",
173
- baseUrl,
174
- localNetworkAccess,
175
- taskHandleSecret: requiredApiKey
176
- });
177
- }
178
- if (toolsets.has("serp")) {
179
- registerSerpIntelligenceCaptureTools(server, httpExecutor, {
180
- exposeDevelopmentDiagnostics: deploymentProfile === "development" || deploymentProfile === "test"
181
- });
182
- }
183
- if (toolsets.has("browser-agent")) {
184
- registerBrowserAgentMcpTools(server, { baseUrl, apiKey: requiredApiKey, consoleBaseUrl });
185
- }
186
- if (toolsets.has("scheduled-results")) {
187
- registerScheduledResultsMcpTools(server, new ScheduledResultsMcpExecutor(baseUrl, requiredApiKey));
188
- }
189
- if (toolsets.has("memory")) {
190
- registerMemoryMcpTools(server, new HostedMemoryMcpToolExecutor(baseUrl, requiredApiKey));
191
- }
192
- registrationGuard?.assertComplete();
193
- return server;
194
- }
195
- function runMcpScraperStdio(options = {}) {
196
- const forceStdio = process.argv.includes("--stdio") || process.env.MCP_SCRAPER_FORCE_STDIO === "1";
197
- const interactiveTerminal = Boolean(process.stdin.isTTY && process.stdout.isTTY);
198
- const wantsHelp = process.argv.includes("--help") || process.argv.includes("-h");
199
- if (!forceStdio && (interactiveTerminal || wantsHelp)) {
200
- const noColor = process.argv.includes("--no-color") || process.env.NO_COLOR !== void 0 || process.env.FORCE_COLOR === "0" || !process.stdout.isTTY;
201
- process.stdout.write(renderInstallTerminal({
202
- version: PACKAGE_VERSION,
203
- color: !noColor,
204
- apiKeyConfigured: Boolean(process.env.MCP_SCRAPER_API_KEY?.trim())
205
- }));
206
- process.exit(0);
207
- }
208
- const apiKey = (process.env.MCP_SCRAPER_API_KEY ?? process.env.MCP_SCRAPER_KEY ?? process.env.MCP_API_KEY ?? readApiKeyFile())?.trim();
209
- if (!apiKey) {
210
- process.stderr.write("MCP_SCRAPER_API_KEY env var or ~/.mcp-scraper-key is required\n");
211
- process.exit(1);
212
- }
213
- serveStdio(() => buildStdioServer(apiKey, options), {
214
- legacy: "serve",
215
- onerror(error) {
216
- process.stderr.write(`${error.message}
217
- `);
218
- }
219
- });
220
- }
221
-
222
- export {
223
- runMcpScraperStdio
224
- };
@@ -1,265 +0,0 @@
1
- import {
2
- publicErrorMessage,
3
- sanitizeVendorName
4
- } from "./chunk-YXNDOQXN.js";
5
-
6
- // src/api/outbound-sanitize.ts
7
- var KEY_RENAMES = {
8
- kernel: "browserRuntime"
9
- };
10
- var ERROR_VALUE_KEYS = /error|message/i;
11
- var PRIVATE_UPSTREAM_FAILURE = /(?:spending|billing|organization|account).{0,80}(?:cap|limit|blocked|disabled)|(?:quota|capacity).{0,40}(?:reached|exceeded)|https?:\/\/\S*(?:dashboard|onkernel)/i;
12
- var PRIVATE_RUNTIME_FAILURE = /\b(?:ENOENT|ENOTDIR|EACCES|EPERM|EISDIR|EMFILE|ENFILE)\b|\b(?:mkdir|open|stat|lstat|readFile|writeFile)\s+['"]?(?:\/|[A-Za-z]:\\)|(?:^|[\s'"])(?:\/Users|\/home|\/tmp|\/var|\/private|\/opt|\/srv|\/root)\//i;
13
- var INTERNAL_DIAGNOSTIC_KEYS = /* @__PURE__ */ new Set([
14
- "kernel_session_id",
15
- "browser_session_id",
16
- "kernelSessionId",
17
- "browserSessionId",
18
- "sessionId",
19
- "kernel_delete_started",
20
- "session_cleanup_started",
21
- "kernelDeleteStarted",
22
- "sessionCleanupStarted",
23
- "kernel_delete_succeeded",
24
- "session_cleanup_succeeded",
25
- "kernelDeleteSucceeded",
26
- "sessionCleanupSucceeded",
27
- "kernel_delete_error",
28
- "session_cleanup_error",
29
- "kernelDeleteError",
30
- "sessionCleanupError",
31
- "kernelProxyId",
32
- "proxyId",
33
- "requestedProxyIdSuffix",
34
- "retrievedProxyIdSuffix",
35
- "provider",
36
- "providerName",
37
- "service",
38
- "serviceName",
39
- "vendor",
40
- "vendorName",
41
- "organization",
42
- "organizationId",
43
- "accountId",
44
- // Provider attempt telemetry is projected field-by-field by sanitizeAttempts.
45
- // Keeping these keys private here prevents an unreviewed nested object from
46
- // bypassing that projection through the generic diagnostics sanitizer.
47
- "browser_provider",
48
- "provider_session_id",
49
- "provider_session_observed_at",
50
- "provider_lookup_status",
51
- "provider_status",
52
- "provider_error_code",
53
- "provider_error_message",
54
- "provider_reason_source",
55
- "provider_duration_ms",
56
- "provider_bandwidth_bytes",
57
- "provider_captcha",
58
- "provider_checked_at",
59
- "provider_detail",
60
- "provider_detail_json",
61
- "provider_disconnect_observed",
62
- "provider_disconnect_message",
63
- "browserProvider",
64
- "providerSessionId",
65
- "providerLookupStatus",
66
- "providerStatus",
67
- "providerErrorCode",
68
- "providerErrorMessage",
69
- "providerReasonSource",
70
- "providerDurationMs",
71
- "providerBandwidthBytes",
72
- "providerCaptcha",
73
- "providerCheckedAt",
74
- "providerDetail",
75
- "providerDisconnectObserved",
76
- "providerDisconnectMessage"
77
- ]);
78
- var VENDOR_URL_KEYS = /* @__PURE__ */ new Set([
79
- "hosted_url",
80
- "hostedUrl",
81
- "live_view_url",
82
- "liveViewUrl",
83
- "cdp_ws_url",
84
- "cdpWsUrl",
85
- "browser_live_view_url"
86
- ]);
87
- var VENDOR_URL_RE = /\b(?:wss?|https?):\/\/[^\s"'<>]*\bonkernel\.com[^\s"'<>]*/gi;
88
- function redactVendorUrls(value) {
89
- return value.replace(VENDOR_URL_RE, "[browser-service]");
90
- }
91
- var PUBLIC_BROWSER_PROVIDERS = /* @__PURE__ */ new Set(["bright_data_browser_api", "kernel", "local"]);
92
- var PUBLIC_PROVIDER_LOOKUP_STATUSES = /* @__PURE__ */ new Set([
93
- "pending",
94
- "complete",
95
- "not_found",
96
- "unavailable_credentials",
97
- "temporary_error",
98
- "invalid_payload",
99
- "not_applicable"
100
- ]);
101
- var PUBLIC_PROVIDER_STATUSES = /* @__PURE__ */ new Set([
102
- "active",
103
- "running",
104
- "complete",
105
- "completed",
106
- "failed",
107
- "closed",
108
- "stopped",
109
- "terminated",
110
- "timed_out"
111
- ]);
112
- var PUBLIC_PROVIDER_REASON_SOURCES = /* @__PURE__ */ new Set([
113
- "provider_session_api",
114
- "client_runtime",
115
- "extractor",
116
- "unknown"
117
- ]);
118
- var PUBLIC_PROVIDER_ERROR_CODES = /* @__PURE__ */ new Set([
119
- "browser_disconnected",
120
- "worker_disconnect",
121
- "job_killed",
122
- "cdp_cmd_timeout",
123
- "network_inactivity_timeout",
124
- "session_timeout",
125
- "client_timeout",
126
- "navigate_domains_limit",
127
- "cdp_error",
128
- "browser_session_interrupted",
129
- "provider_detail_pending",
130
- "provider_lookup_unavailable",
131
- "provider_reported_no_error",
132
- "provider_session_not_found",
133
- "provider_invalid_json",
134
- "provider_invalid_payload",
135
- "provider_lookup_timeout",
136
- "provider_lookup_network_error",
137
- "unknown"
138
- ]);
139
- var PROVIDER_TELEMETRY_KEYS = /* @__PURE__ */ new Set([
140
- "browser_provider",
141
- "provider_session_id",
142
- "provider_lookup_status",
143
- "provider_status",
144
- "provider_error_code",
145
- "provider_error_message",
146
- "provider_reason_source",
147
- "provider_duration_ms",
148
- "provider_bandwidth_bytes",
149
- "provider_captcha",
150
- "provider_checked_at",
151
- "provider_detail",
152
- "provider_detail_json",
153
- "provider_disconnect_observed",
154
- "provider_disconnect_message"
155
- ]);
156
- var MAX_PROVIDER_SESSION_ID_LENGTH = 512;
157
- var MAX_PROVIDER_MESSAGE_LENGTH = 320;
158
- function allowlistedString(value, values) {
159
- return typeof value === "string" && values.has(value) ? value : null;
160
- }
161
- function publicProviderErrorCode(value) {
162
- if (typeof value !== "string") return null;
163
- if (PUBLIC_PROVIDER_ERROR_CODES.has(value)) return value;
164
- return /^provider_http_(?:429|5\d\d)$/.test(value) ? value : null;
165
- }
166
- function publicProviderSessionId(value) {
167
- if (typeof value !== "string") return null;
168
- if (value.length < 1 || value.length > MAX_PROVIDER_SESSION_ID_LENGTH) return null;
169
- if (/^(?:authorization|bearer|password|secret|token|api[_-]?key|account)/i.test(value)) return null;
170
- return /^[A-Za-z0-9._:-]+$/.test(value) && !value.includes("://") ? value : null;
171
- }
172
- function publicNonNegativeInteger(value) {
173
- return typeof value === "number" && Number.isSafeInteger(value) && value >= 0 ? value : null;
174
- }
175
- function publicIsoTimestamp(value) {
176
- if (typeof value !== "string" || value.length > 40) return null;
177
- const parsed = Date.parse(value);
178
- return Number.isFinite(parsed) && new Date(parsed).toISOString() === value ? value : null;
179
- }
180
- function publicProviderMessage(value) {
181
- if (typeof value !== "string") return null;
182
- const normalized = value.replace(/[\u0000-\u001f\u007f]+/g, " ").replace(/\s+/g, " ").trim();
183
- if (!normalized || normalized.startsWith("{") || normalized.startsWith("[")) return null;
184
- const redacted = normalized.replace(/\b(?:wss?|https?):\/\/\S+/gi, "[redacted endpoint]").replace(/\b(?:authorization|proxy-authorization)\s*:\s*(?:Bearer\s+)?\S+/gi, "[redacted credential]").replace(/\bBearer\s+\S+/gi, "[redacted credential]").replace(/\b(?:api[_-]?key|password|secret|token|account[_-]?id)\s*[=:]\s*[^\s,;]+/gi, "[redacted credential]").replace(/\b(?:account|organization)\s+(?:id\s+)?[A-Za-z0-9_-]{4,}/gi, "[redacted account]");
185
- return redactVendorUrls(redacted).slice(0, MAX_PROVIDER_MESSAGE_LENGTH);
186
- }
187
- function publicNullableBoolean(value) {
188
- return typeof value === "boolean" ? value : null;
189
- }
190
- function projectAttemptProviderTelemetry(attempt) {
191
- return {
192
- browser_provider: allowlistedString(attempt.browser_provider, PUBLIC_BROWSER_PROVIDERS),
193
- provider_session_id: publicProviderSessionId(attempt.provider_session_id),
194
- provider_session_observed_at: publicIsoTimestamp(attempt.provider_session_observed_at),
195
- provider_lookup_status: allowlistedString(attempt.provider_lookup_status, PUBLIC_PROVIDER_LOOKUP_STATUSES),
196
- provider_status: allowlistedString(attempt.provider_status, PUBLIC_PROVIDER_STATUSES),
197
- provider_error_code: publicProviderErrorCode(attempt.provider_error_code),
198
- provider_error_message: publicProviderMessage(attempt.provider_error_message),
199
- provider_reason_source: allowlistedString(attempt.provider_reason_source, PUBLIC_PROVIDER_REASON_SOURCES),
200
- provider_duration_ms: publicNonNegativeInteger(attempt.provider_duration_ms),
201
- provider_bandwidth_bytes: publicNonNegativeInteger(attempt.provider_bandwidth_bytes),
202
- provider_captcha: publicNullableBoolean(attempt.provider_captcha),
203
- provider_checked_at: publicIsoTimestamp(attempt.provider_checked_at),
204
- provider_disconnect_observed: publicNullableBoolean(attempt.provider_disconnect_observed),
205
- provider_disconnect_message: publicProviderMessage(attempt.provider_disconnect_message)
206
- };
207
- }
208
- function sanitizeOutboundDiagnostics(value, parentKey = "") {
209
- if (typeof value === "string") {
210
- let out = redactVendorUrls(value);
211
- if (ERROR_VALUE_KEYS.test(parentKey) && (PRIVATE_UPSTREAM_FAILURE.test(value) || PRIVATE_RUNTIME_FAILURE.test(value))) {
212
- out = publicErrorMessage("service_unavailable");
213
- } else if (ERROR_VALUE_KEYS.test(parentKey)) {
214
- out = sanitizeVendorName(out);
215
- }
216
- return out;
217
- }
218
- if (Array.isArray(value)) return value.map((v) => sanitizeOutboundDiagnostics(v, parentKey));
219
- if (value !== null && typeof value === "object") {
220
- const out = {};
221
- for (const [key, val] of Object.entries(value)) {
222
- if (INTERNAL_DIAGNOSTIC_KEYS.has(key)) continue;
223
- const renamed = KEY_RENAMES[key] ?? key;
224
- if (VENDOR_URL_KEYS.has(key)) {
225
- out[renamed] = null;
226
- continue;
227
- }
228
- out[renamed] = sanitizeOutboundDiagnostics(val, key);
229
- }
230
- return out;
231
- }
232
- return value;
233
- }
234
- function sanitizeAttempts(attempts) {
235
- return attempts.map((attempt) => {
236
- if (attempt === null || typeof attempt !== "object" || Array.isArray(attempt)) {
237
- return sanitizeOutboundDiagnostics(attempt);
238
- }
239
- const row = attempt;
240
- const sanitized = sanitizeOutboundDiagnostics(row);
241
- const hasProviderTelemetry = Object.keys(row).some((key) => PROVIDER_TELEMETRY_KEYS.has(key));
242
- if (!hasProviderTelemetry) return sanitized;
243
- return {
244
- ...sanitized,
245
- ...projectAttemptProviderTelemetry(row)
246
- };
247
- });
248
- }
249
- function sanitizeHarvestResult(result) {
250
- const diagnostics = result?.diagnostics;
251
- if (!diagnostics?.debug) return result;
252
- return {
253
- ...result,
254
- diagnostics: {
255
- ...diagnostics,
256
- debug: sanitizeOutboundDiagnostics(diagnostics.debug)
257
- }
258
- };
259
- }
260
-
261
- export {
262
- sanitizeOutboundDiagnostics,
263
- sanitizeAttempts,
264
- sanitizeHarvestResult
265
- };
@@ -1,113 +0,0 @@
1
- // release-message.json
2
- var release_message_default = {
3
- message: "PAA harvests can now request two organic result pages before expanding questions on the original first page. Use pages: 2 in MCP, SDKs, or the dashboard and check the reported capture status; an unavailable second page preserves first-page evidence."
4
- };
5
-
6
- // src/install-terminal.ts
7
- var CODES = {
8
- reset: "\x1B[0m",
9
- cyan: "\x1B[36m",
10
- lime: "\x1B[32m",
11
- amber: "\x1B[33m",
12
- red: "\x1B[31m",
13
- muted: "\x1B[90m",
14
- bold: "\x1B[1m"
15
- };
16
- function colorize(value, tone, enabled) {
17
- if (!enabled) return value;
18
- return `${CODES[tone]}${value}${CODES.reset}`;
19
- }
20
- function toolRow(label, tools, enabled) {
21
- const padded = label.padEnd(9, " ");
22
- return ` ${colorize(padded, "muted", enabled)} ${tools.join(colorize(" . ", "muted", enabled))}`;
23
- }
24
- function renderInstallTerminal(options) {
25
- const color = options.color ?? true;
26
- const apiKeyValue = options.apiKeyConfigured ? "$MCP_SCRAPER_API_KEY" : "sk_live_your_key";
27
- const ascii = String.raw`
28
- __ __ ____ ____
29
- | \/ |/ ___| _ \
30
- | |\/| | | | |_) |
31
- | | | | |___| __/
32
- |_| |_|\____|_|
33
-
34
- ____ ____ ____ _ ____ _____ ____
35
- / ___| / ___| _ \ / \ | _ \| ____| _ \
36
- \___ \| | | |_) | / _ \ | |_) | _| | |_) |
37
- ___) | |___| _ < / ___ \| __/| |___| _ <
38
- |____/ \____|_| \_\/_/ \_\_| |_____|_| \_\
39
- `;
40
- const claudeCommand = [
41
- `MCP_SCRAPER_API_KEY=${apiKeyValue} npx -y -p mcp-scraper@latest \\`,
42
- " mcp-scraper-cli agent install claude --apply"
43
- ].join("\n");
44
- const codexConfig = [
45
- "[mcp_servers.mcp-scraper]",
46
- 'command = "npx"',
47
- 'args = ["-y", "-p", "mcp-scraper@latest", "mcp-scraper"]',
48
- `env = { MCP_SCRAPER_API_KEY = "${apiKeyValue}" }`
49
- ].join("\n");
50
- return [
51
- colorize(`mcp-scraper v${options.version}`, "bold", color),
52
- colorize("> mcp-scraper-install", "muted", color),
53
- colorize(ascii, "amber", color),
54
- `${colorize("MCP Scraper Agent", "cyan", color)} . v${options.version} . mcpscraper.dev`,
55
- "1/1 install surfaces ready",
56
- colorize(`Newest in v${options.version}: ${release_message_default.message}`, "lime", color),
57
- "",
58
- `${colorize("Tools", "cyan", color)} ${colorize("(375 MCP tools)", "muted", color)}`,
59
- toolRow("search", ["harvest_paa", "search_serp", "maps_search", "maps_place_intel"], color),
60
- toolRow("extract", ["extract_url", "map_site_urls", "extract_site", "audit_site", "directory_workflow"], color),
61
- toolRow("build", ["create_editorial_reading_room", "rank_tracker_workflow", "portable HTML"], color),
62
- toolRow("media", ["youtube_harvest", "youtube_transcribe", "facebook_ad_search", "facebook_page_intel", "facebook_ad_transcribe", "facebook_video_transcribe", "instagram_profile_content", "instagram_media_download", "reddit_thread"], color),
63
- toolRow("browser", ["serp_identity_create", "serp_identity_list", "browser_open", "browser_profile_connect", "browser_profile_list", "browser_close", "browser_screenshot", "browser_read", "browser_locate", "browser_replay_mark", "browser_replay_annotate"], color),
64
- toolRow("connect", ["list_service_connections", "describe_service_connection_tool", "import_service_connection_to_memory", "export_connected_service_data", "renew_connected_data_download", "read_service_connection", "call_service_connection_action"], color),
65
- toolRow("commons", ["commons_search_entities", "commons_get_entity_linkset", "commons_prepare_entity", "commons_submit_entity", "commons_prepare_publication", "commons_claim_publication", "commons_publish_editorial", "commons_get_publication"], color),
66
- toolRow("account", ["credits_info", "reports", "MCP resources"], color),
67
- toolRow("memory", ["memory-put", "memory-get", "memory-search", "list-vaults", "record-fact", "list-scheduled-actions"], color),
68
- `${colorize("Workflows", "cyan", color)} ${colorize("(MCP + CLI + API)", "muted", color)}`,
69
- toolRow("route", ["workflow_list", "workflow_suggest", "workflow_run", "workflow_step", "workflow_status", "workflow_artifact_read"], color),
70
- toolRow("seo", ["directory", "agent-packet", "competitive audit", "map/serp comparison", "PAA/AIO briefs", "scheduled runs"], color),
71
- "",
72
- colorize("Usage tips:", "amber", color),
73
- "Run mcp-scraper-install for this visible card. Run mcp-scraper-cli for setup utilities and subcommands.",
74
- "Run mcp-scraper in a human terminal to print this card; MCP clients get the same command as a silent stdio server.",
75
- "Explicit card command: npx -y -p mcp-scraper@latest mcp-scraper-install",
76
- "Hosted browser sessions use direct/no-proxy egress by default.",
77
- "Customer auth setup: run browser_profile_connect, send the watch_url, let the user sign in, then call browser_profile_list until AUTHENTICATED.",
78
- "Connected account ranges: call export_connected_service_data once. It handles Gmail, Calendar, Zoom, and Resend pagination; do not loop read_service_connection over individual records.",
79
- "Connected account RAG: call import_service_connection_to_memory for one bounded approved read. It writes a redacted, untrusted snapshot to a stable Memory path and embeds it for search.",
80
- "Stack logins / reconnect: run browser_profile_connect again with the same profile name and another domain to add accounts or refresh a login.",
81
- "Start with workflow_suggest for broad jobs like market analysis, ICP research, CRO audits, brand briefs, content gaps, and AI visibility.",
82
- "For MCP clients, use mcp-scraper so one install can mix SERP, Maps, browser, reports, and saved MCP resources.",
83
- "If you hit the concurrency limit, add 2 browsers for $5/month with mcp-scraper-cli billing concurrency checkout.",
84
- "",
85
- `${colorize("Ready.", "lime", color)} Install the combined MCP server with one command:`,
86
- "",
87
- colorize("Setup doctor", "amber", color),
88
- "npx -y -p mcp-scraper@latest mcp-scraper-cli doctor",
89
- "",
90
- colorize("Hosted profile setup", "amber", color),
91
- 'In your MCP client, call browser_profile_connect with email="seo@example.com" and domain="chatgpt.com".',
92
- "Give the returned watch_url to the user. After they sign in, call browser_profile_list, then browser_open with the returned profile. Add more logins by calling browser_profile_connect again with the same profile and a new domain.",
93
- "",
94
- colorize("Claude Code one-command setup", "amber", color),
95
- claudeCommand,
96
- "Then fully exit Claude Code and open a new Claude terminal. Check with: claude mcp list",
97
- "",
98
- colorize("Codex config", "amber", color),
99
- codexConfig,
100
- "",
101
- colorize("Claude Desktop Extension", "amber", color),
102
- "Download: https://mcpscraper.dev/downloads/mcp-scraper.mcpb",
103
- "",
104
- colorize("Safety note:", "muted", color),
105
- "mcp-scraper prints this card only when stdin/stdout are an interactive TTY. In MCP clients it writes only JSON-RPC to stdout.",
106
- "Use --stdio or MCP_SCRAPER_FORCE_STDIO=1 to force server mode from a terminal.",
107
- ""
108
- ].join("\n");
109
- }
110
-
111
- export {
112
- renderInstallTerminal
113
- };