mcp-scraper 0.69.0 → 0.72.0

package/dist/chunk-7RQULQF3.js

@@ -0,0 +1,263 @@
+import {
+  publicErrorMessage,
+  sanitizeVendorName
+} from "./chunk-WEFPBAAG.js";
+
+// src/api/outbound-sanitize.ts
+var KEY_RENAMES = {
+  kernel: "browserRuntime"
+};
+var ERROR_VALUE_KEYS = /error|message/i;
+var PRIVATE_UPSTREAM_FAILURE = /(?:spending|billing|organization|account).{0,80}(?:cap|limit|blocked|disabled)|(?:quota|capacity).{0,40}(?:reached|exceeded)|https?:\/\/\S*(?:dashboard|onkernel)/i;
+var PRIVATE_RUNTIME_FAILURE = /\b(?:ENOENT|ENOTDIR|EACCES|EPERM|EISDIR|EMFILE|ENFILE)\b|\b(?:mkdir|open|stat|lstat|readFile|writeFile)\s+['"]?(?:\/|[A-Za-z]:\\)|(?:^|[\s'"])(?:\/Users|\/home|\/tmp|\/var|\/private|\/opt|\/srv|\/root)\//i;
+var INTERNAL_DIAGNOSTIC_KEYS = /* @__PURE__ */ new Set([
+  "kernel_session_id",
+  "browser_session_id",
+  "kernelSessionId",
+  "browserSessionId",
+  "sessionId",
+  "kernel_delete_started",
+  "session_cleanup_started",
+  "kernelDeleteStarted",
+  "sessionCleanupStarted",
+  "kernel_delete_succeeded",
+  "session_cleanup_succeeded",
+  "kernelDeleteSucceeded",
+  "sessionCleanupSucceeded",
+  "kernel_delete_error",
+  "session_cleanup_error",
+  "kernelDeleteError",
+  "sessionCleanupError",
+  "kernelProxyId",
+  "proxyId",
+  "requestedProxyIdSuffix",
+  "retrievedProxyIdSuffix",
+  "provider",
+  "providerName",
+  "service",
+  "serviceName",
+  "vendor",
+  "vendorName",
+  "organization",
+  "organizationId",
+  "accountId",
+  // Provider attempt telemetry is projected field-by-field by sanitizeAttempts.
+  // Keeping these keys private here prevents an unreviewed nested object from
+  // bypassing that projection through the generic diagnostics sanitizer.
+  "browser_provider",
+  "provider_session_id",
+  "provider_lookup_status",
+  "provider_status",
+  "provider_error_code",
+  "provider_error_message",
+  "provider_reason_source",
+  "provider_duration_ms",
+  "provider_bandwidth_bytes",
+  "provider_captcha",
+  "provider_checked_at",
+  "provider_detail",
+  "provider_detail_json",
+  "provider_disconnect_observed",
+  "provider_disconnect_message",
+  "browserProvider",
+  "providerSessionId",
+  "providerLookupStatus",
+  "providerStatus",
+  "providerErrorCode",
+  "providerErrorMessage",
+  "providerReasonSource",
+  "providerDurationMs",
+  "providerBandwidthBytes",
+  "providerCaptcha",
+  "providerCheckedAt",
+  "providerDetail",
+  "providerDisconnectObserved",
+  "providerDisconnectMessage"
+]);
+var VENDOR_URL_KEYS = /* @__PURE__ */ new Set([
+  "hosted_url",
+  "hostedUrl",
+  "live_view_url",
+  "liveViewUrl",
+  "cdp_ws_url",
+  "cdpWsUrl",
+  "browser_live_view_url"
+]);
+var VENDOR_URL_RE = /\b(?:wss?|https?):\/\/[^\s"'<>]*\bonkernel\.com[^\s"'<>]*/gi;
+function redactVendorUrls(value) {
+  return value.replace(VENDOR_URL_RE, "[browser-service]");
+}
+var PUBLIC_BROWSER_PROVIDERS = /* @__PURE__ */ new Set(["bright_data_browser_api", "kernel", "local"]);
+var PUBLIC_PROVIDER_LOOKUP_STATUSES = /* @__PURE__ */ new Set([
+  "pending",
+  "complete",
+  "not_found",
+  "unavailable_credentials",
+  "temporary_error",
+  "invalid_payload",
+  "not_applicable"
+]);
+var PUBLIC_PROVIDER_STATUSES = /* @__PURE__ */ new Set([
+  "active",
+  "running",
+  "complete",
+  "completed",
+  "failed",
+  "closed",
+  "stopped",
+  "terminated",
+  "timed_out"
+]);
+var PUBLIC_PROVIDER_REASON_SOURCES = /* @__PURE__ */ new Set([
+  "provider_session_api",
+  "client_runtime",
+  "extractor",
+  "unknown"
+]);
+var PUBLIC_PROVIDER_ERROR_CODES = /* @__PURE__ */ new Set([
+  "browser_disconnected",
+  "worker_disconnect",
+  "job_killed",
+  "cdp_cmd_timeout",
+  "network_inactivity_timeout",
+  "session_timeout",
+  "client_timeout",
+  "navigate_domains_limit",
+  "cdp_error",
+  "browser_session_interrupted",
+  "provider_detail_pending",
+  "provider_lookup_unavailable",
+  "provider_reported_no_error",
+  "provider_session_not_found",
+  "provider_invalid_json",
+  "provider_invalid_payload",
+  "provider_lookup_timeout",
+  "provider_lookup_network_error",
+  "unknown"
+]);
+var PROVIDER_TELEMETRY_KEYS = /* @__PURE__ */ new Set([
+  "browser_provider",
+  "provider_session_id",
+  "provider_lookup_status",
+  "provider_status",
+  "provider_error_code",
+  "provider_error_message",
+  "provider_reason_source",
+  "provider_duration_ms",
+  "provider_bandwidth_bytes",
+  "provider_captcha",
+  "provider_checked_at",
+  "provider_detail",
+  "provider_detail_json",
+  "provider_disconnect_observed",
+  "provider_disconnect_message"
+]);
+var MAX_PROVIDER_SESSION_ID_LENGTH = 512;
+var MAX_PROVIDER_MESSAGE_LENGTH = 320;
+function allowlistedString(value, values) {
+  return typeof value === "string" && values.has(value) ? value : null;
+}
+function publicProviderErrorCode(value) {
+  if (typeof value !== "string") return null;
+  if (PUBLIC_PROVIDER_ERROR_CODES.has(value)) return value;
+  return /^provider_http_(?:429|5\d\d)$/.test(value) ? value : null;
+}
+function publicProviderSessionId(value) {
+  if (typeof value !== "string") return null;
+  if (value.length < 1 || value.length > MAX_PROVIDER_SESSION_ID_LENGTH) return null;
+  if (/^(?:authorization|bearer|password|secret|token|api[_-]?key|account)/i.test(value)) return null;
+  return /^[A-Za-z0-9._:-]+$/.test(value) && !value.includes("://") ? value : null;
+}
+function publicNonNegativeInteger(value) {
+  return typeof value === "number" && Number.isSafeInteger(value) && value >= 0 ? value : null;
+}
+function publicIsoTimestamp(value) {
+  if (typeof value !== "string" || value.length > 40) return null;
+  const parsed = Date.parse(value);
+  return Number.isFinite(parsed) && new Date(parsed).toISOString() === value ? value : null;
+}
+function publicProviderMessage(value) {
+  if (typeof value !== "string") return null;
+  const normalized = value.replace(/[\u0000-\u001f\u007f]+/g, " ").replace(/\s+/g, " ").trim();
+  if (!normalized || normalized.startsWith("{") || normalized.startsWith("[")) return null;
+  const redacted = normalized.replace(/\b(?:wss?|https?):\/\/\S+/gi, "[redacted endpoint]").replace(/\b(?:authorization|proxy-authorization)\s*:\s*(?:Bearer\s+)?\S+/gi, "[redacted credential]").replace(/\bBearer\s+\S+/gi, "[redacted credential]").replace(/\b(?:api[_-]?key|password|secret|token|account[_-]?id)\s*[=:]\s*[^\s,;]+/gi, "[redacted credential]").replace(/\b(?:account|organization)\s+(?:id\s+)?[A-Za-z0-9_-]{4,}/gi, "[redacted account]");
+  return redactVendorUrls(redacted).slice(0, MAX_PROVIDER_MESSAGE_LENGTH);
+}
+function publicNullableBoolean(value) {
+  return typeof value === "boolean" ? value : null;
+}
+function projectAttemptProviderTelemetry(attempt) {
+  return {
+    browser_provider: allowlistedString(attempt.browser_provider, PUBLIC_BROWSER_PROVIDERS),
+    provider_session_id: publicProviderSessionId(attempt.provider_session_id),
+    provider_lookup_status: allowlistedString(attempt.provider_lookup_status, PUBLIC_PROVIDER_LOOKUP_STATUSES),
+    provider_status: allowlistedString(attempt.provider_status, PUBLIC_PROVIDER_STATUSES),
+    provider_error_code: publicProviderErrorCode(attempt.provider_error_code),
+    provider_error_message: publicProviderMessage(attempt.provider_error_message),
+    provider_reason_source: allowlistedString(attempt.provider_reason_source, PUBLIC_PROVIDER_REASON_SOURCES),
+    provider_duration_ms: publicNonNegativeInteger(attempt.provider_duration_ms),
+    provider_bandwidth_bytes: publicNonNegativeInteger(attempt.provider_bandwidth_bytes),
+    provider_captcha: publicNullableBoolean(attempt.provider_captcha),
+    provider_checked_at: publicIsoTimestamp(attempt.provider_checked_at),
+    provider_disconnect_observed: publicNullableBoolean(attempt.provider_disconnect_observed),
+    provider_disconnect_message: publicProviderMessage(attempt.provider_disconnect_message)
+  };
+}
+function sanitizeOutboundDiagnostics(value, parentKey = "") {
+  if (typeof value === "string") {
+    let out = redactVendorUrls(value);
+    if (ERROR_VALUE_KEYS.test(parentKey) && (PRIVATE_UPSTREAM_FAILURE.test(value) || PRIVATE_RUNTIME_FAILURE.test(value))) {
+      out = publicErrorMessage("service_unavailable");
+    } else if (ERROR_VALUE_KEYS.test(parentKey)) {
+      out = sanitizeVendorName(out);
+    }
+    return out;
+  }
+  if (Array.isArray(value)) return value.map((v) => sanitizeOutboundDiagnostics(v, parentKey));
+  if (value !== null && typeof value === "object") {
+    const out = {};
+    for (const [key, val] of Object.entries(value)) {
+      if (INTERNAL_DIAGNOSTIC_KEYS.has(key)) continue;
+      const renamed = KEY_RENAMES[key] ?? key;
+      if (VENDOR_URL_KEYS.has(key)) {
+        out[renamed] = null;
+        continue;
+      }
+      out[renamed] = sanitizeOutboundDiagnostics(val, key);
+    }
+    return out;
+  }
+  return value;
+}
+function sanitizeAttempts(attempts) {
+  return attempts.map((attempt) => {
+    if (attempt === null || typeof attempt !== "object" || Array.isArray(attempt)) {
+      return sanitizeOutboundDiagnostics(attempt);
+    }
+    const row = attempt;
+    const sanitized = sanitizeOutboundDiagnostics(row);
+    const hasProviderTelemetry = Object.keys(row).some((key) => PROVIDER_TELEMETRY_KEYS.has(key));
+    if (!hasProviderTelemetry) return sanitized;
+    return {
+      ...sanitized,
+      ...projectAttemptProviderTelemetry(row)
+    };
+  });
+}
+function sanitizeHarvestResult(result) {
+  const diagnostics = result?.diagnostics;
+  if (!diagnostics?.debug) return result;
+  return {
+    ...result,
+    diagnostics: {
+      ...diagnostics,
+      debug: sanitizeOutboundDiagnostics(diagnostics.debug)
+    }
+  };
+}
+
+export {
+  sanitizeOutboundDiagnostics,
+  sanitizeAttempts,
+  sanitizeHarvestResult
+};

package/dist/{chunk-UL4ZKAWZ.js → chunk-A66DGFOU.js}

@@ -1,6 +1,6 @@
 import {
   getDb
-} from "./chunk-65FTMB7G.js";
+} from "./chunk-WEFPBAAG.js";
 
 // src/api/lead-list-enrichment-repository.ts
 import { createHash, randomUUID } from "crypto";

package/dist/{chunk-ES25GP6C.js → chunk-CXY5WV45.js}

@@ -1,26 +1,6 @@
 import {
   getDb
-} from "./chunk-65FTMB7G.js";
-
-// src/lib/browser-service-env.ts
-function browserServiceApiKey() {
-  const value = (process.env.BROWSER_SERVICE_API_KEY ?? process.env.KERNEL_API_KEY)?.trim();
-  return value || void 0;
-}
-function browserServiceProxyId() {
-  return void 0;
-}
-function browserServiceProfileName() {
-  const value = (process.env.BROWSER_AGENT_PROFILE_NAME ?? process.env.BROWSER_SERVICE_PROFILE_NAME ?? process.env.KERNEL_BROWSER_PROFILE_NAME ?? process.env.KERNEL_PROFILE_NAME)?.trim();
-  return value || void 0;
-}
-function browserServiceProfileSaveChanges() {
-  const value = (process.env.BROWSER_AGENT_PROFILE_SAVE_CHANGES ?? process.env.BROWSER_SERVICE_PROFILE_SAVE_CHANGES ?? process.env.KERNEL_BROWSER_PROFILE_SAVE_CHANGES ?? process.env.KERNEL_PROFILE_SAVE_CHANGES)?.trim().toLowerCase();
-  if (!value) return void 0;
-  if (["1", "true", "yes", "on"].includes(value)) return true;
-  if (["0", "false", "no", "off"].includes(value)) return false;
-  return void 0;
-}
+} from "./chunk-WEFPBAAG.js";
 
 // src/api/cost-context.ts
 import { AsyncLocalStorage } from "async_hooks";
@@ -53,6 +33,7 @@ var NANGO_USD_PER_CONNECTION_MONTH = envRate("NANGO_USD_PER_CONNECTION_MONTH", 1
 var NANGO_USD_PER_FUNCTION_RUN = envRate("NANGO_USD_PER_FUNCTION_RUN", 1e-4);
 var NANGO_USD_PER_PROXY_REQUEST = envRate("NANGO_USD_PER_PROXY_REQUEST", 1e-4);
 var NANGO_USD_PER_COMPUTE_SEC = envRate("NANGO_USD_PER_COMPUTE_SEC", 2e-4);
+var BRIGHTDATA_BROWSER_USD_PER_GB = envRate("BRIGHTDATA_BROWSER_USD_PER_GB", 0);
 function kernelCostUsd(ms, headful) {
   const sec = Math.max(0, ms) / 1e3;
   return sec * (headful ? KERNEL_HEADFUL_USD_PER_SEC : KERNEL_HEADLESS_USD_PER_SEC);
@@ -70,6 +51,9 @@ function vendorCostUsd(vendor, units) {
   if (vendor === "nango_function_run") return Math.max(0, units) * NANGO_USD_PER_FUNCTION_RUN;
   if (vendor === "nango_proxy_request") return Math.max(0, units) * NANGO_USD_PER_PROXY_REQUEST;
   if (vendor === "nango_compute") return Math.max(0, units) * NANGO_USD_PER_COMPUTE_SEC;
+  if (vendor === "brightdata_browser_api") {
+    return Math.max(0, units) / 1e9 * BRIGHTDATA_BROWSER_USD_PER_GB;
+  }
   return 0;
 }
 
@@ -90,7 +74,8 @@ async function runCostTelemetryMigration() {
     SELECT
       (SELECT COUNT(*) FROM sqlite_master WHERE type = 'table' AND name IN ('kernel_session_log', 'vendor_usage_log', 'cost_probe_runs')) = 3
       AND (SELECT COUNT(*) FROM pragma_table_info('kernel_session_log') WHERE name IN ('proxy_source', 'proxy_type', 'method')) = 3
-      AND (SELECT COUNT(*) FROM pragma_table_info('vendor_usage_log') WHERE name = 'method') = 1
+      AND (SELECT COUNT(*) FROM pragma_table_info('vendor_usage_log') WHERE name IN ('method', 'source_key')) = 2
+      AND (SELECT COUNT(*) FROM sqlite_master WHERE type = 'index' AND name = 'vendor_usage_log_vendor_source_key') = 1
       AND (SELECT COUNT(*) FROM pragma_table_info('cost_probe_runs') WHERE name IN ('units', 'unit_type', 'mode')) = 3
       AS ready
   `);
@@ -146,6 +131,7 @@ async function runCostTelemetryMigration() {
       unit_type    TEXT,
       est_cost_usd REAL NOT NULL DEFAULT 0,
       error        TEXT,
+      source_key   TEXT,
       created_at   TEXT NOT NULL DEFAULT (datetime('now'))
     )
   `);
@@ -156,6 +142,11 @@ async function runCostTelemetryMigration() {
     await db.execute(`ALTER TABLE vendor_usage_log ADD COLUMN method TEXT`);
   } catch {
   }
+  try {
+    await db.execute(`ALTER TABLE vendor_usage_log ADD COLUMN source_key TEXT`);
+  } catch {
+  }
+  await db.execute(`CREATE UNIQUE INDEX IF NOT EXISTS vendor_usage_log_vendor_source_key ON vendor_usage_log(vendor, source_key) WHERE source_key IS NOT NULL`);
   await db.execute(`
     CREATE TABLE IF NOT EXISTS cost_probe_runs (
       id                            TEXT PRIMARY KEY,
@@ -235,26 +226,29 @@ async function recordVendorUsage(r) {
     await migrateCostTelemetry();
     const ctx = currentCostContext();
     const db = getDb();
-    await db.execute({
-      sql: `INSERT INTO vendor_usage_log
-        (id, op, probe_run_id, user_id, vendor, model, units, unit_type, est_cost_usd, error, method)
-        VALUES (?,?,?,?,?,?,?,?,?,?,?)`,
+    const result = await db.execute({
+      sql: `INSERT OR IGNORE INTO vendor_usage_log
+        (id, op, probe_run_id, user_id, vendor, model, units, unit_type, est_cost_usd, error, method, source_key)
+        VALUES (?,?,?,?,?,?,?,?,?,?,?,?)`,
       args: [
         randomUUID(),
-        ctx?.op ?? null,
-        ctx?.probeRunId ?? null,
-        ctx?.userId ?? null,
+        r.op ?? ctx?.op ?? null,
+        r.probeRunId ?? ctx?.probeRunId ?? null,
+        r.userId ?? ctx?.userId ?? null,
         r.vendor,
         r.model ?? null,
         r.units,
         r.unitType,
         vendorCostUsd(r.vendor, r.units),
         r.error ?? null,
-        ctx?.subOp ?? null
+        r.method ?? ctx?.subOp ?? null,
+        r.sourceKey ?? null
       ]
     });
+    return result.rowsAffected === 1;
   } catch (err) {
     console.warn("[cost-telemetry] recordVendorUsage failed:", err instanceof Error ? err.message : String(err));
+    return false;
   }
 }
 function boolToInt(v) {
@@ -263,10 +257,6 @@ function boolToInt(v) {
 }
 
 export {
-  browserServiceApiKey,
-  browserServiceProxyId,
-  browserServiceProfileName,
-  browserServiceProfileSaveChanges,
   HEADLESS_HARVEST_TEST_ENV,
   runWithCostContext,
   currentCostContext,

package/dist/{chunk-Y46YNQMM.js → chunk-F5TK4KMT.js}

@@ -3,7 +3,7 @@ import {
 } from "./chunk-RJ7JVYKU.js";
 import {
   getDb
-} from "./chunk-65FTMB7G.js";
+} from "./chunk-WEFPBAAG.js";
 
 // src/api/location-data-repository.ts
 import { createHash, randomUUID } from "crypto";