mcp-scraper 0.88.2 → 0.89.0

package/dist/chunk-6HAV7LCE.js

@@ -1,265 +0,0 @@
-import {
-  publicErrorMessage,
-  sanitizeVendorName
-} from "./chunk-YXNDOQXN.js";
-
-// src/api/outbound-sanitize.ts
-var KEY_RENAMES = {
-  kernel: "browserRuntime"
-};
-var ERROR_VALUE_KEYS = /error|message/i;
-var PRIVATE_UPSTREAM_FAILURE = /(?:spending|billing|organization|account).{0,80}(?:cap|limit|blocked|disabled)|(?:quota|capacity).{0,40}(?:reached|exceeded)|https?:\/\/\S*(?:dashboard|onkernel)/i;
-var PRIVATE_RUNTIME_FAILURE = /\b(?:ENOENT|ENOTDIR|EACCES|EPERM|EISDIR|EMFILE|ENFILE)\b|\b(?:mkdir|open|stat|lstat|readFile|writeFile)\s+['"]?(?:\/|[A-Za-z]:\\)|(?:^|[\s'"])(?:\/Users|\/home|\/tmp|\/var|\/private|\/opt|\/srv|\/root)\//i;
-var INTERNAL_DIAGNOSTIC_KEYS = /* @__PURE__ */ new Set([
-  "kernel_session_id",
-  "browser_session_id",
-  "kernelSessionId",
-  "browserSessionId",
-  "sessionId",
-  "kernel_delete_started",
-  "session_cleanup_started",
-  "kernelDeleteStarted",
-  "sessionCleanupStarted",
-  "kernel_delete_succeeded",
-  "session_cleanup_succeeded",
-  "kernelDeleteSucceeded",
-  "sessionCleanupSucceeded",
-  "kernel_delete_error",
-  "session_cleanup_error",
-  "kernelDeleteError",
-  "sessionCleanupError",
-  "kernelProxyId",
-  "proxyId",
-  "requestedProxyIdSuffix",
-  "retrievedProxyIdSuffix",
-  "provider",
-  "providerName",
-  "service",
-  "serviceName",
-  "vendor",
-  "vendorName",
-  "organization",
-  "organizationId",
-  "accountId",
-  // Provider attempt telemetry is projected field-by-field by sanitizeAttempts.
-  // Keeping these keys private here prevents an unreviewed nested object from
-  // bypassing that projection through the generic diagnostics sanitizer.
-  "browser_provider",
-  "provider_session_id",
-  "provider_session_observed_at",
-  "provider_lookup_status",
-  "provider_status",
-  "provider_error_code",
-  "provider_error_message",
-  "provider_reason_source",
-  "provider_duration_ms",
-  "provider_bandwidth_bytes",
-  "provider_captcha",
-  "provider_checked_at",
-  "provider_detail",
-  "provider_detail_json",
-  "provider_disconnect_observed",
-  "provider_disconnect_message",
-  "browserProvider",
-  "providerSessionId",
-  "providerLookupStatus",
-  "providerStatus",
-  "providerErrorCode",
-  "providerErrorMessage",
-  "providerReasonSource",
-  "providerDurationMs",
-  "providerBandwidthBytes",
-  "providerCaptcha",
-  "providerCheckedAt",
-  "providerDetail",
-  "providerDisconnectObserved",
-  "providerDisconnectMessage"
-]);
-var VENDOR_URL_KEYS = /* @__PURE__ */ new Set([
-  "hosted_url",
-  "hostedUrl",
-  "live_view_url",
-  "liveViewUrl",
-  "cdp_ws_url",
-  "cdpWsUrl",
-  "browser_live_view_url"
-]);
-var VENDOR_URL_RE = /\b(?:wss?|https?):\/\/[^\s"'<>]*\bonkernel\.com[^\s"'<>]*/gi;
-function redactVendorUrls(value) {
-  return value.replace(VENDOR_URL_RE, "[browser-service]");
-}
-var PUBLIC_BROWSER_PROVIDERS = /* @__PURE__ */ new Set(["bright_data_browser_api", "kernel", "local"]);
-var PUBLIC_PROVIDER_LOOKUP_STATUSES = /* @__PURE__ */ new Set([
-  "pending",
-  "complete",
-  "not_found",
-  "unavailable_credentials",
-  "temporary_error",
-  "invalid_payload",
-  "not_applicable"
-]);
-var PUBLIC_PROVIDER_STATUSES = /* @__PURE__ */ new Set([
-  "active",
-  "running",
-  "complete",
-  "completed",
-  "failed",
-  "closed",
-  "stopped",
-  "terminated",
-  "timed_out"
-]);
-var PUBLIC_PROVIDER_REASON_SOURCES = /* @__PURE__ */ new Set([
-  "provider_session_api",
-  "client_runtime",
-  "extractor",
-  "unknown"
-]);
-var PUBLIC_PROVIDER_ERROR_CODES = /* @__PURE__ */ new Set([
-  "browser_disconnected",
-  "worker_disconnect",
-  "job_killed",
-  "cdp_cmd_timeout",
-  "network_inactivity_timeout",
-  "session_timeout",
-  "client_timeout",
-  "navigate_domains_limit",
-  "cdp_error",
-  "browser_session_interrupted",
-  "provider_detail_pending",
-  "provider_lookup_unavailable",
-  "provider_reported_no_error",
-  "provider_session_not_found",
-  "provider_invalid_json",
-  "provider_invalid_payload",
-  "provider_lookup_timeout",
-  "provider_lookup_network_error",
-  "unknown"
-]);
-var PROVIDER_TELEMETRY_KEYS = /* @__PURE__ */ new Set([
-  "browser_provider",
-  "provider_session_id",
-  "provider_lookup_status",
-  "provider_status",
-  "provider_error_code",
-  "provider_error_message",
-  "provider_reason_source",
-  "provider_duration_ms",
-  "provider_bandwidth_bytes",
-  "provider_captcha",
-  "provider_checked_at",
-  "provider_detail",
-  "provider_detail_json",
-  "provider_disconnect_observed",
-  "provider_disconnect_message"
-]);
-var MAX_PROVIDER_SESSION_ID_LENGTH = 512;
-var MAX_PROVIDER_MESSAGE_LENGTH = 320;
-function allowlistedString(value, values) {
-  return typeof value === "string" && values.has(value) ? value : null;
-}
-function publicProviderErrorCode(value) {
-  if (typeof value !== "string") return null;
-  if (PUBLIC_PROVIDER_ERROR_CODES.has(value)) return value;
-  return /^provider_http_(?:429|5\d\d)$/.test(value) ? value : null;
-}
-function publicProviderSessionId(value) {
-  if (typeof value !== "string") return null;
-  if (value.length < 1 || value.length > MAX_PROVIDER_SESSION_ID_LENGTH) return null;
-  if (/^(?:authorization|bearer|password|secret|token|api[_-]?key|account)/i.test(value)) return null;
-  return /^[A-Za-z0-9._:-]+$/.test(value) && !value.includes("://") ? value : null;
-}
-function publicNonNegativeInteger(value) {
-  return typeof value === "number" && Number.isSafeInteger(value) && value >= 0 ? value : null;
-}
-function publicIsoTimestamp(value) {
-  if (typeof value !== "string" || value.length > 40) return null;
-  const parsed = Date.parse(value);
-  return Number.isFinite(parsed) && new Date(parsed).toISOString() === value ? value : null;
-}
-function publicProviderMessage(value) {
-  if (typeof value !== "string") return null;
-  const normalized = value.replace(/[\u0000-\u001f\u007f]+/g, " ").replace(/\s+/g, " ").trim();
-  if (!normalized || normalized.startsWith("{") || normalized.startsWith("[")) return null;
-  const redacted = normalized.replace(/\b(?:wss?|https?):\/\/\S+/gi, "[redacted endpoint]").replace(/\b(?:authorization|proxy-authorization)\s*:\s*(?:Bearer\s+)?\S+/gi, "[redacted credential]").replace(/\bBearer\s+\S+/gi, "[redacted credential]").replace(/\b(?:api[_-]?key|password|secret|token|account[_-]?id)\s*[=:]\s*[^\s,;]+/gi, "[redacted credential]").replace(/\b(?:account|organization)\s+(?:id\s+)?[A-Za-z0-9_-]{4,}/gi, "[redacted account]");
-  return redactVendorUrls(redacted).slice(0, MAX_PROVIDER_MESSAGE_LENGTH);
-}
-function publicNullableBoolean(value) {
-  return typeof value === "boolean" ? value : null;
-}
-function projectAttemptProviderTelemetry(attempt) {
-  return {
-    browser_provider: allowlistedString(attempt.browser_provider, PUBLIC_BROWSER_PROVIDERS),
-    provider_session_id: publicProviderSessionId(attempt.provider_session_id),
-    provider_session_observed_at: publicIsoTimestamp(attempt.provider_session_observed_at),
-    provider_lookup_status: allowlistedString(attempt.provider_lookup_status, PUBLIC_PROVIDER_LOOKUP_STATUSES),
-    provider_status: allowlistedString(attempt.provider_status, PUBLIC_PROVIDER_STATUSES),
-    provider_error_code: publicProviderErrorCode(attempt.provider_error_code),
-    provider_error_message: publicProviderMessage(attempt.provider_error_message),
-    provider_reason_source: allowlistedString(attempt.provider_reason_source, PUBLIC_PROVIDER_REASON_SOURCES),
-    provider_duration_ms: publicNonNegativeInteger(attempt.provider_duration_ms),
-    provider_bandwidth_bytes: publicNonNegativeInteger(attempt.provider_bandwidth_bytes),
-    provider_captcha: publicNullableBoolean(attempt.provider_captcha),
-    provider_checked_at: publicIsoTimestamp(attempt.provider_checked_at),
-    provider_disconnect_observed: publicNullableBoolean(attempt.provider_disconnect_observed),
-    provider_disconnect_message: publicProviderMessage(attempt.provider_disconnect_message)
-  };
-}
-function sanitizeOutboundDiagnostics(value, parentKey = "") {
-  if (typeof value === "string") {
-    let out = redactVendorUrls(value);
-    if (ERROR_VALUE_KEYS.test(parentKey) && (PRIVATE_UPSTREAM_FAILURE.test(value) || PRIVATE_RUNTIME_FAILURE.test(value))) {
-      out = publicErrorMessage("service_unavailable");
-    } else if (ERROR_VALUE_KEYS.test(parentKey)) {
-      out = sanitizeVendorName(out);
-    }
-    return out;
-  }
-  if (Array.isArray(value)) return value.map((v) => sanitizeOutboundDiagnostics(v, parentKey));
-  if (value !== null && typeof value === "object") {
-    const out = {};
-    for (const [key, val] of Object.entries(value)) {
-      if (INTERNAL_DIAGNOSTIC_KEYS.has(key)) continue;
-      const renamed = KEY_RENAMES[key] ?? key;
-      if (VENDOR_URL_KEYS.has(key)) {
-        out[renamed] = null;
-        continue;
-      }
-      out[renamed] = sanitizeOutboundDiagnostics(val, key);
-    }
-    return out;
-  }
-  return value;
-}
-function sanitizeAttempts(attempts) {
-  return attempts.map((attempt) => {
-    if (attempt === null || typeof attempt !== "object" || Array.isArray(attempt)) {
-      return sanitizeOutboundDiagnostics(attempt);
-    }
-    const row = attempt;
-    const sanitized = sanitizeOutboundDiagnostics(row);
-    const hasProviderTelemetry = Object.keys(row).some((key) => PROVIDER_TELEMETRY_KEYS.has(key));
-    if (!hasProviderTelemetry) return sanitized;
-    return {
-      ...sanitized,
-      ...projectAttemptProviderTelemetry(row)
-    };
-  });
-}
-function sanitizeHarvestResult(result) {
-  const diagnostics = result?.diagnostics;
-  if (!diagnostics?.debug) return result;
-  return {
-    ...result,
-    diagnostics: {
-      ...diagnostics,
-      debug: sanitizeOutboundDiagnostics(diagnostics.debug)
-    }
-  };
-}
-
-export {
-  sanitizeOutboundDiagnostics,
-  sanitizeAttempts,
-  sanitizeHarvestResult
-};

package/dist/chunk-ABF2CGOZ.js

@@ -1,113 +0,0 @@
-// release-message.json
-var release_message_default = {
-  message: "PAA harvests can now request two organic result pages before expanding questions on the original first page. Use pages: 2 in MCP, SDKs, or the dashboard and check the reported capture status; an unavailable second page preserves first-page evidence."
-};
-
-// src/install-terminal.ts
-var CODES = {
-  reset: "\x1B[0m",
-  cyan: "\x1B[36m",
-  lime: "\x1B[32m",
-  amber: "\x1B[33m",
-  red: "\x1B[31m",
-  muted: "\x1B[90m",
-  bold: "\x1B[1m"
-};
-function colorize(value, tone, enabled) {
-  if (!enabled) return value;
-  return `${CODES[tone]}${value}${CODES.reset}`;
-}
-function toolRow(label, tools, enabled) {
-  const padded = label.padEnd(9, " ");
-  return `  ${colorize(padded, "muted", enabled)} ${tools.join(colorize("  .  ", "muted", enabled))}`;
-}
-function renderInstallTerminal(options) {
-  const color = options.color ?? true;
-  const apiKeyValue = options.apiKeyConfigured ? "$MCP_SCRAPER_API_KEY" : "sk_live_your_key";
-  const ascii = String.raw`
- __  __  ____ ____ 
-|  \/  |/ ___|  _ \
-| |\/| | |   | |_) |
-| |  | | |___|  __/
-|_|  |_|\____|_|
-
- ____   ____ ____      _    ____  _____ ____
-/ ___| / ___|  _ \    / \  |  _ \| ____|  _ \
-\___ \| |   | |_) |  / _ \ | |_) |  _| | |_) |
- ___) | |___|  _ <  / ___ \|  __/| |___|  _ <
-|____/ \____|_| \_\/_/   \_\_|   |_____|_| \_\
-`;
-  const claudeCommand = [
-    `MCP_SCRAPER_API_KEY=${apiKeyValue} npx -y -p mcp-scraper@latest \\`,
-    "  mcp-scraper-cli agent install claude --apply"
-  ].join("\n");
-  const codexConfig = [
-    "[mcp_servers.mcp-scraper]",
-    'command = "npx"',
-    'args = ["-y", "-p", "mcp-scraper@latest", "mcp-scraper"]',
-    `env = { MCP_SCRAPER_API_KEY = "${apiKeyValue}" }`
-  ].join("\n");
-  return [
-    colorize(`mcp-scraper v${options.version}`, "bold", color),
-    colorize("> mcp-scraper-install", "muted", color),
-    colorize(ascii, "amber", color),
-    `${colorize("MCP Scraper Agent", "cyan", color)}  .  v${options.version}  .  mcpscraper.dev`,
-    "1/1 install surfaces ready",
-    colorize(`Newest in v${options.version}: ${release_message_default.message}`, "lime", color),
-    "",
-    `${colorize("Tools", "cyan", color)}  ${colorize("(375 MCP tools)", "muted", color)}`,
-    toolRow("search", ["harvest_paa", "search_serp", "maps_search", "maps_place_intel"], color),
-    toolRow("extract", ["extract_url", "map_site_urls", "extract_site", "audit_site", "directory_workflow"], color),
-    toolRow("build", ["create_editorial_reading_room", "rank_tracker_workflow", "portable HTML"], color),
-    toolRow("media", ["youtube_harvest", "youtube_transcribe", "facebook_ad_search", "facebook_page_intel", "facebook_ad_transcribe", "facebook_video_transcribe", "instagram_profile_content", "instagram_media_download", "reddit_thread"], color),
-    toolRow("browser", ["serp_identity_create", "serp_identity_list", "browser_open", "browser_profile_connect", "browser_profile_list", "browser_close", "browser_screenshot", "browser_read", "browser_locate", "browser_replay_mark", "browser_replay_annotate"], color),
-    toolRow("connect", ["list_service_connections", "describe_service_connection_tool", "import_service_connection_to_memory", "export_connected_service_data", "renew_connected_data_download", "read_service_connection", "call_service_connection_action"], color),
-    toolRow("commons", ["commons_search_entities", "commons_get_entity_linkset", "commons_prepare_entity", "commons_submit_entity", "commons_prepare_publication", "commons_claim_publication", "commons_publish_editorial", "commons_get_publication"], color),
-    toolRow("account", ["credits_info", "reports", "MCP resources"], color),
-    toolRow("memory", ["memory-put", "memory-get", "memory-search", "list-vaults", "record-fact", "list-scheduled-actions"], color),
-    `${colorize("Workflows", "cyan", color)}  ${colorize("(MCP + CLI + API)", "muted", color)}`,
-    toolRow("route", ["workflow_list", "workflow_suggest", "workflow_run", "workflow_step", "workflow_status", "workflow_artifact_read"], color),
-    toolRow("seo", ["directory", "agent-packet", "competitive audit", "map/serp comparison", "PAA/AIO briefs", "scheduled runs"], color),
-    "",
-    colorize("Usage tips:", "amber", color),
-    "Run mcp-scraper-install for this visible card. Run mcp-scraper-cli for setup utilities and subcommands.",
-    "Run mcp-scraper in a human terminal to print this card; MCP clients get the same command as a silent stdio server.",
-    "Explicit card command: npx -y -p mcp-scraper@latest mcp-scraper-install",
-    "Hosted browser sessions use direct/no-proxy egress by default.",
-    "Customer auth setup: run browser_profile_connect, send the watch_url, let the user sign in, then call browser_profile_list until AUTHENTICATED.",
-    "Connected account ranges: call export_connected_service_data once. It handles Gmail, Calendar, Zoom, and Resend pagination; do not loop read_service_connection over individual records.",
-    "Connected account RAG: call import_service_connection_to_memory for one bounded approved read. It writes a redacted, untrusted snapshot to a stable Memory path and embeds it for search.",
-    "Stack logins / reconnect: run browser_profile_connect again with the same profile name and another domain to add accounts or refresh a login.",
-    "Start with workflow_suggest for broad jobs like market analysis, ICP research, CRO audits, brand briefs, content gaps, and AI visibility.",
-    "For MCP clients, use mcp-scraper so one install can mix SERP, Maps, browser, reports, and saved MCP resources.",
-    "If you hit the concurrency limit, add 2 browsers for $5/month with mcp-scraper-cli billing concurrency checkout.",
-    "",
-    `${colorize("Ready.", "lime", color)} Install the combined MCP server with one command:`,
-    "",
-    colorize("Setup doctor", "amber", color),
-    "npx -y -p mcp-scraper@latest mcp-scraper-cli doctor",
-    "",
-    colorize("Hosted profile setup", "amber", color),
-    'In your MCP client, call browser_profile_connect with email="seo@example.com" and domain="chatgpt.com".',
-    "Give the returned watch_url to the user. After they sign in, call browser_profile_list, then browser_open with the returned profile. Add more logins by calling browser_profile_connect again with the same profile and a new domain.",
-    "",
-    colorize("Claude Code one-command setup", "amber", color),
-    claudeCommand,
-    "Then fully exit Claude Code and open a new Claude terminal. Check with: claude mcp list",
-    "",
-    colorize("Codex config", "amber", color),
-    codexConfig,
-    "",
-    colorize("Claude Desktop Extension", "amber", color),
-    "Download: https://mcpscraper.dev/downloads/mcp-scraper.mcpb",
-    "",
-    colorize("Safety note:", "muted", color),
-    "mcp-scraper prints this card only when stdin/stdout are an interactive TTY. In MCP clients it writes only JSON-RPC to stdout.",
-    "Use --stdio or MCP_SCRAPER_FORCE_STDIO=1 to force server mode from a terminal.",
-    ""
-  ].join("\n");
-}
-
-export {
-  renderInstallTerminal
-};