mcp-scraper 0.15.0 → 0.16.0

package/dist/{server-DDKXWYTO.js → server-QEXOVJPR.js}

@@ -37,7 +37,7 @@ import {
   renewConnectedDataArtifactDownload,
   sanitizeAttempts,
   sanitizeHarvestResult
-} from "./chunk-ETJBTYZX.js";
+} from "./chunk-YIV4IKFG.js";
 import {
   auditImages,
   buildLinkReport,
@@ -73,7 +73,7 @@ import {
   RawMapsOverviewSchema,
   RawMapsReviewStatsSchema
 } from "./chunk-XGIPATLV.js";
-import "./chunk-TK2S2M7G.js";
+import "./chunk-HQYIP5X3.js";
 import {
   completeExtractJob,
   countSuccessfulPages,
@@ -20356,6 +20356,10 @@ async function collectConnectedDataExport(args) {
     recordBytes += pageBytes;
     cursor = page.nextCursor ?? void 0;
     continuationCursor = page.nextCursor ?? null;
+    if (page.complete === false && typeof beforePageCursor === "string" && page.nextCursor === beforePageCursor) {
+      warnings.add("The provider could not advance this page after bounded retries. Its records and warnings were included once; retry this exact continuation later.");
+      break;
+    }
     if (page.complete || page.nextCursor === null || page.nextCursor === void 0) {
       complete = true;
       continuationCursor = null;
@@ -20629,6 +20633,7 @@ async function dbUsage(identity, plan) {
 }
 
 // src/api/nango-control.ts
+import { createHash as createHash5 } from "crypto";
 var DEFAULT_NANGO_CONTROL_URL = "https://mcp-scraper-scheduler.vercel.app";
 var DISABLED_NANGO_TOOLS = {
   "google-mail": /* @__PURE__ */ new Set(["list-filters"]),
@@ -20707,10 +20712,14 @@ function connectionSyncSelectionError(connections) {
 }
 var NangoControlError = class extends Error {
   status;
-  constructor(message, status = 502) {
+  code;
+  retryable;
+  constructor(message, status = 502, code = status === 503 ? "connection_transport_unavailable" : "connection_control_failed", retryable = status === 429 || status >= 500) {
     super(message);
     this.name = "NangoControlError";
     this.status = status;
+    this.code = code;
+    this.retryable = retryable;
   }
 };
 var ScheduleConnectionValidationError = class extends Error {
@@ -20775,6 +20784,63 @@ function arrayFromPayload(value, keys) {
   }
   return [];
 }
+var SAFE_CONTROL_ERROR_CODES = /* @__PURE__ */ new Set([
+  "invalid_request",
+  "tool_not_allowed",
+  "invalid_schema",
+  "connection_not_found",
+  "connection_inactive",
+  "actions_disabled",
+  "action_not_allowed",
+  "upstream_rate_limited",
+  "tool_discovery_failed",
+  "live_tool_missing",
+  "connection_transport_unavailable",
+  "connection_control_failed"
+]);
+var CONTROL_ERROR_CODE_ALIASES = /* @__PURE__ */ new Map([
+  ["connection_not_active", "connection_inactive"],
+  ["actions_not_enabled", "actions_disabled"],
+  ["action_tool_not_allowed", "action_not_allowed"],
+  ["no_action_for_provider", "action_not_allowed"],
+  ["tool_not_available", "live_tool_missing"]
+]);
+function controlErrorStatus(status) {
+  if (status === 400 || status === 403 || status === 404 || status === 409 || status === 429 || status === 502 || status === 503) {
+    return status;
+  }
+  return status >= 500 ? 503 : 502;
+}
+function defaultControlErrorCode(status) {
+  if (status === 400) return "invalid_request";
+  if (status === 403) return "actions_disabled";
+  if (status === 404) return "connection_not_found";
+  if (status === 409) return "connection_inactive";
+  if (status === 429) return "upstream_rate_limited";
+  if (status === 503) return "connection_transport_unavailable";
+  return "connection_control_failed";
+}
+function defaultControlErrorMessage(status) {
+  if (status === 400) return "The connection request was invalid.";
+  if (status === 403) return "Actions are disabled for this service connection.";
+  if (status === 404) return "The service connection was not found.";
+  if (status === 409) return "The service connection is not currently usable.";
+  if (status === 429) return "The connected service is temporarily rate limited.";
+  if (status === 503) return "The service connection transport is temporarily unavailable.";
+  return "The service connection control request failed.";
+}
+function safeControlErrorPayload(body, responseStatus) {
+  const status = controlErrorStatus(responseStatus);
+  const unwrapped = unwrapData(body);
+  const record = isRecord(unwrapped) ? unwrapped : isRecord(body) ? body : null;
+  const candidateCode = record ? firstString(record, ["code", "errorCode", "error_code"], 100) : null;
+  const normalizedCandidateCode = candidateCode ? CONTROL_ERROR_CODE_ALIASES.get(candidateCode) ?? candidateCode : null;
+  const code = normalizedCandidateCode && SAFE_CONTROL_ERROR_CODES.has(normalizedCandidateCode) ? normalizedCandidateCode : defaultControlErrorCode(status);
+  const candidateMessage = record ? firstString(record, ["error", "message"], 500) : null;
+  const message = normalizedCandidateCode && SAFE_CONTROL_ERROR_CODES.has(normalizedCandidateCode) && candidateMessage ? candidateMessage.replace(/[\u0000-\u001f\u007f]/g, " ") : defaultControlErrorMessage(status);
+  const retryable = record && typeof record.retryable === "boolean" ? record.retryable : status === 429 || status >= 500;
+  return { status, code, message, retryable };
+}
 async function controlRequest(path5, init, timeoutMs = 2e4) {
   const response = await fetch(`${controlBaseUrl()}${path5}`, {
     ...init,
@@ -20786,7 +20852,13 @@ async function controlRequest(path5, init, timeoutMs = 2e4) {
     },
     signal: AbortSignal.timeout(timeoutMs)
   }).catch((err) => {
-    throw new NangoControlError(`Scheduled service connection control is unavailable: ${err instanceof Error ? err.message : "network error"}`);
+    const cause = err instanceof Error && err.name === "TimeoutError" ? "request timed out" : "network error";
+    throw new NangoControlError(
+      `Scheduled service connection control is unavailable: ${cause}.`,
+      503,
+      "connection_transport_unavailable",
+      true
+    );
   });
   const text = await response.text();
   let body = {};
@@ -20796,8 +20868,8 @@ async function controlRequest(path5, init, timeoutMs = 2e4) {
     body = {};
   }
   if (!response.ok) {
-    const upstream = isRecord(body) ? cleanString(body.error, 300) : null;
-    throw new NangoControlError(upstream || `Scheduled service connection control failed (${response.status}).`);
+    const error = safeControlErrorPayload(body, response.status);
+    throw new NangoControlError(error.message, error.status, error.code, error.retryable);
   }
   return body;
 }
@@ -20892,6 +20964,9 @@ async function getNangoConnections(identity) {
       actionsEnabled: row.actionsEnabled === true || row.actions_enabled === true,
       readTools: cleanTools(row.readTools ?? row.read_tools),
       actionTools: cleanTools(row.actionTools ?? row.action_tools ?? row.writeTools ?? row.write_tools),
+      mcpEndpoint: null,
+      schemaDiscovery: "compatibility_describe",
+      toolRevision: firstString(row, ["toolRevision", "tool_revision"], 200),
       vaultName: firstString(row, ["vaultName", "vault_name"], 100),
       tableName: firstString(row, ["tableName", "table_name"], 100),
       createdAt: firstString(row, ["createdAt", "created_at"], 100),
@@ -21024,6 +21099,145 @@ async function callScheduleConnectionRead(identity, connectionId, tool, args) {
   const data = unwrapData(body);
   return isRecord(data) ? data.result ?? data : data;
 }
+function sanitizeToolSchema(value) {
+  if (!isRecord(value) || value.type !== "object") return null;
+  try {
+    const serialized = JSON.stringify(value);
+    if (Buffer.byteLength(serialized, "utf8") > 256 * 1024) return null;
+    const cloned = JSON.parse(serialized);
+    return isRecord(cloned) ? cloned : null;
+  } catch {
+    return null;
+  }
+}
+function sanitizeToolAnnotations(value) {
+  if (!isRecord(value)) return void 0;
+  const annotations = {};
+  const title = cleanString(value.title, 200);
+  if (title) annotations.title = title;
+  for (const key of ["readOnlyHint", "destructiveHint", "idempotentHint", "openWorldHint"]) {
+    if (typeof value[key] === "boolean") annotations[key] = value[key];
+  }
+  return Object.keys(annotations).length > 0 ? annotations : void 0;
+}
+function sanitizeToolIconSource(value) {
+  const candidate = cleanString(value, 1e4);
+  if (!candidate) return null;
+  if (/^data:image\/(?:png|jpeg|webp|gif|svg\+xml);base64,[a-z0-9+/=]+$/i.test(candidate)) return candidate;
+  return cleanHttpsUrl(candidate);
+}
+function sanitizeToolIcons(value) {
+  if (!Array.isArray(value)) return void 0;
+  const icons = [];
+  for (const item of value.slice(0, 16)) {
+    if (!isRecord(item)) continue;
+    const src = sanitizeToolIconSource(item.src);
+    if (!src) continue;
+    const mimeType = cleanString(item.mimeType ?? item.mime_type, 100);
+    const sizes = cleanStringArray(item.sizes, 16, 32);
+    const theme = item.theme === "light" || item.theme === "dark" ? item.theme : void 0;
+    icons.push({
+      src,
+      ...mimeType ? { mimeType } : {},
+      ...sizes.length > 0 ? { sizes } : {},
+      ...theme ? { theme } : {}
+    });
+  }
+  return icons.length > 0 ? icons : void 0;
+}
+function canonicalJson(value) {
+  if (Array.isArray(value)) return `[${value.map(canonicalJson).join(",")}]`;
+  if (isRecord(value)) {
+    return `{${Object.keys(value).sort().map((key) => `${JSON.stringify(key)}:${canonicalJson(value[key])}`).join(",")}}`;
+  }
+  return JSON.stringify(value);
+}
+function projectedToolSchemaHash(tool) {
+  return createHash5("sha256").update(canonicalJson(tool)).digest("hex");
+}
+async function describeNangoTool(identity, connectionId, tool, fresh) {
+  const body = await controlRequest("/api/internal/nango/connections/actions/describe", {
+    method: "POST",
+    body: JSON.stringify({
+      identity,
+      connectionId,
+      tool,
+      ...fresh === void 0 ? {} : { fresh }
+    })
+  });
+  const data = unwrapData(body);
+  const rawTool = isRecord(data) && isRecord(data.tool) ? data.tool : data;
+  if (!isRecord(rawTool)) {
+    throw new NangoControlError(
+      "The connection service returned an invalid live tool description.",
+      502,
+      "tool_discovery_failed",
+      true
+    );
+  }
+  let serializedBytes = Number.POSITIVE_INFINITY;
+  try {
+    serializedBytes = Buffer.byteLength(JSON.stringify(rawTool), "utf8");
+  } catch {
+  }
+  const name = cleanString(rawTool.name, 200);
+  const classification = cleanString(rawTool.classification, 20);
+  const transport = cleanString(rawTool.transport, 20);
+  const providerConfigKey = cleanString(rawTool.providerConfigKey ?? rawTool.provider_config_key, 200);
+  const schemaSource = cleanString(rawTool.schemaSource ?? rawTool.schema_source, 50);
+  const upstreamSchemaHash = cleanString(rawTool.schemaHash ?? rawTool.schema_hash, 200);
+  const fetchedAt = cleanString(rawTool.fetchedAt ?? rawTool.fetched_at, 100);
+  const inputSchema = sanitizeToolSchema(rawTool.inputSchema ?? rawTool.input_schema);
+  const outputSchemaValue = rawTool.outputSchema ?? rawTool.output_schema;
+  const outputSchema = outputSchemaValue === void 0 ? void 0 : sanitizeToolSchema(outputSchemaValue);
+  const blockedReasonValue = rawTool.blockedReason ?? rawTool.blocked_reason;
+  const blockedReason = blockedReasonValue === null || blockedReasonValue === void 0 ? null : blockedReasonValue === "actions_disabled" || blockedReasonValue === "inactive_connection" ? blockedReasonValue : void 0;
+  if (serializedBytes > 256 * 1024 || !name || name !== tool || classification !== "read" && classification !== "action" || typeof rawTool.callable !== "boolean" || blockedReason === void 0 || transport !== "nango" && transport !== "remote_mcp" || !providerConfigKey || schemaSource !== "live_tools_list" || !upstreamSchemaHash || !/^[a-f0-9]{64}$/i.test(upstreamSchemaHash) || !fetchedAt || Number.isNaN(Date.parse(fetchedAt)) || !inputSchema || outputSchemaValue !== void 0 && !outputSchema) {
+    throw new NangoControlError(
+      "The connection service returned an invalid live tool description.",
+      502,
+      "tool_discovery_failed",
+      true
+    );
+  }
+  const protocolVersion = rawTool.protocolVersion === null || rawTool.protocol_version === null ? null : cleanString(rawTool.protocolVersion ?? rawTool.protocol_version, 100);
+  const executionValue = rawTool.execution;
+  const taskSupport = isRecord(executionValue) && (executionValue.taskSupport === "forbidden" || executionValue.taskSupport === "optional" || executionValue.taskSupport === "required") ? executionValue.taskSupport : null;
+  const title = cleanString(rawTool.title, 200);
+  const description = cleanString(rawTool.description, 12e3);
+  const annotations = sanitizeToolAnnotations(rawTool.annotations);
+  const icons = sanitizeToolIcons(rawTool.icons);
+  const execution = taskSupport ? { taskSupport } : void 0;
+  const schemaHash = projectedToolSchemaHash({
+    name,
+    inputSchema,
+    ...title ? { title } : {},
+    ...description ? { description } : {},
+    ...outputSchema ? { outputSchema } : {},
+    ...annotations ? { annotations } : {},
+    ...icons ? { icons } : {},
+    ...execution ? { execution } : {}
+  });
+  return {
+    name,
+    title,
+    description,
+    classification,
+    callable: rawTool.callable,
+    blockedReason,
+    transport,
+    providerConfigKey,
+    protocolVersion,
+    schemaSource: "live_tools_list",
+    inputSchema,
+    ...outputSchema ? { outputSchema } : {},
+    ...annotations ? { annotations } : {},
+    ...icons ? { icons } : {},
+    ...execution ? { execution } : {},
+    schemaHash,
+    fetchedAt
+  };
+}
 async function callScheduleConnectionExportPage(identity, input) {
   const body = await controlRequest("/api/internal/nango/connections/export-page", {
     method: "POST",
@@ -21243,6 +21457,9 @@ async function getResendConnections(identity) {
       readTools: cleanTools2(row.readTools ?? row.read_tools),
       actionTools: cleanTools2(row.actionTools ?? row.action_tools ?? row.writeTools ?? row.write_tools),
       adminBlockedTools: cleanTools2(row.adminBlockedTools ?? row.admin_blocked_tools).length > 0 ? cleanTools2(row.adminBlockedTools ?? row.admin_blocked_tools) : [...RESEND_ADMIN_BLOCKED_TOOLS],
+      mcpEndpoint: null,
+      schemaDiscovery: "compatibility_describe",
+      toolRevision: firstString2(row, ["toolRevision", "tool_revision"], 200),
       vaultName: firstString2(row, ["vaultName", "vault_name"], 100),
       tableName: firstString2(row, ["tableName", "table_name"], 100),
       createdAt: firstString2(row, ["createdAt", "created_at"], 100),
@@ -21985,7 +22202,15 @@ app.post("/schedule-checkout", auth2, async (c) => {
 });
 function scheduleConnectionError(c, err, fallback) {
   if (err instanceof ScheduleConnectionValidationError) return c.json({ ok: false, error: err.message }, 400);
-  if (err instanceof NangoControlError) return c.json({ ok: false, error: err.message }, err.status);
+  if (err instanceof NangoControlError) {
+    return c.json({
+      ok: false,
+      error: err.message,
+      code: err.code,
+      errorCode: err.code,
+      retryable: err.retryable
+    }, err.status);
+  }
   if (err instanceof ResendControlError) return c.json({ ok: false, error: err.message }, err.status);
   console.error("[schedule-connections]", err instanceof Error ? err.message : String(err));
   return c.json({ ok: false, error: fallback }, 502);
@@ -22239,11 +22464,12 @@ app.post("/schedule-connections/actions/describe", auth2, async (c) => {
   const providerConfigKey = providerConfigKeyFrom(body.providerConfigKey);
   const tool = providerConfigKeyFrom(body.tool);
   if (!connectionId || !tool) return c.json({ ok: false, error: "connectionId and tool are required." }, 400);
+  if (body.fresh !== void 0 && typeof body.fresh !== "boolean") {
+    return c.json({ ok: false, error: "fresh must be a boolean when provided." }, 400);
+  }
   try {
-    if (!await isResendConnection(user.email, connectionId, providerConfigKey)) {
-      return c.json({ ok: false, error: "Typed tool descriptions are not available for this connection transport yet." }, 501);
-    }
-    return c.json({ ok: true, tool: await describeResendTool(user.email, connectionId, tool) });
+    const description = await isResendConnection(user.email, connectionId, providerConfigKey) ? await describeResendTool(user.email, connectionId, tool) : await describeNangoTool(user.email, connectionId, tool, body.fresh);
+    return c.json({ ok: true, tool: description });
   } catch (err) {
     return scheduleConnectionError(c, err, "Unable to describe this connection tool.");
   }
@@ -23562,4 +23788,4 @@ app.get("/blog/:slug/", (c) => {
 export {
   app
 };
-//# sourceMappingURL=server-DDKXWYTO.js.map
+//# sourceMappingURL=server-QEXOVJPR.js.map