mcp-scraper 0.15.1 → 0.16.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +7 -5
- package/dist/bin/api-server.cjs +274 -20
- package/dist/bin/api-server.cjs.map +1 -1
- package/dist/bin/api-server.js +1 -1
- package/dist/bin/mcp-scraper-cli.cjs +1 -1
- package/dist/bin/mcp-scraper-cli.cjs.map +1 -1
- package/dist/bin/mcp-scraper-cli.js +1 -1
- package/dist/bin/mcp-scraper-install.cjs +1 -1
- package/dist/bin/mcp-scraper-install.cjs.map +1 -1
- package/dist/bin/mcp-scraper-install.js +1 -1
- package/dist/bin/mcp-stdio-server.cjs +42 -10
- package/dist/bin/mcp-stdio-server.cjs.map +1 -1
- package/dist/bin/mcp-stdio-server.js +2 -2
- package/dist/chunk-HQYIP5X3.js +7 -0
- package/dist/chunk-HQYIP5X3.js.map +1 -0
- package/dist/{chunk-QPYYOTT2.js → chunk-YIV4IKFG.js} +43 -11
- package/dist/chunk-YIV4IKFG.js.map +1 -0
- package/dist/{server-FFNOTEQI.js → server-QEXOVJPR.js} +234 -12
- package/dist/server-QEXOVJPR.js.map +1 -0
- package/docs/mcp-tool-manifest.generated.json +162 -6
- package/docs/specs/google-drive-bulk-access-and-mcp-schema-passthrough-spec.md +1680 -0
- package/package.json +1 -1
- package/dist/chunk-EZAQWOKI.js +0 -7
- package/dist/chunk-EZAQWOKI.js.map +0 -1
- package/dist/chunk-QPYYOTT2.js.map +0 -1
- package/dist/server-FFNOTEQI.js.map +0 -1
package/README.md
CHANGED
|
@@ -88,7 +88,7 @@ Build the branded one-click bundle:
|
|
|
88
88
|
npm run build:mcpb
|
|
89
89
|
```
|
|
90
90
|
|
|
91
|
-
The generated bundle is written to `build/mcpb/mcp-scraper-<version>.mcpb` and copied to `public/downloads/` for the hosted download. The current public bundle is `https://mcpscraper.dev/downloads/mcp-scraper.mcpb` (`0.
|
|
91
|
+
The generated bundle is written to `build/mcpb/mcp-scraper-<version>.mcpb` and copied to `public/downloads/` for the hosted download. The current public bundle is `https://mcpscraper.dev/downloads/mcp-scraper.mcpb` (`0.16.0`, SHA-256 `01623d11e3096cc867afee6c8abc345aaebf0eb15ffafdd875302a80cb90dd31`). Install it by opening or dragging it into Claude Desktop. Claude displays the `MCP Scraper` install card, icon, and API-key configuration field from the bundle manifest.
|
|
92
92
|
|
|
93
93
|
The MCPB install exposes every tool — web-intelligence plus all `browser_*` tools — through the one `mcp-scraper` server.
|
|
94
94
|
|
|
@@ -176,16 +176,18 @@ env = { MCP_SCRAPER_API_KEY = "sk_live_your_key" }
|
|
|
176
176
|
|
|
177
177
|
### Connected-account tools
|
|
178
178
|
|
|
179
|
-
- `list_service_connections` — list this caller's tenant-owned Nango OAuth and official remote MCP connections, including exact live reads, gated actions, permanently blocked administrative tools, and
|
|
180
|
-
- `describe_service_connection_tool` — fetch the
|
|
179
|
+
- `list_service_connections` — list this caller's tenant-owned Nango OAuth and official remote MCP connections, including exact live reads, gated actions, permanently blocked administrative tools, credential transport, and schema-discovery metadata. Connections are never shared between customers.
|
|
180
|
+
- `describe_service_connection_tool` — fetch the sanitized live MCP Tool definition for one tool listed on one tenant-owned connection, including its current callability, input schema, optional output schema, safe annotations, and schema hash. Use this before constructing provider-native arguments; provider functions stay behind the generic bridges instead of becoming dozens of permanent top-level tools.
|
|
181
181
|
- `export_connected_service_data` — fetch a bounded Gmail, Google Calendar, Zoom, or Resend time range in one MCP call. Resend's aggregate `resend_data` export walks 12 practical safe collections: sent mail, received mail, logs, contacts, broadcasts, templates, domains, segments, topics, webhooks, contact imports, and contact properties. The six core collections are also individually selectable. The server handles provider pagination, bounded detail retrieval, normalization, per-category warnings, and signed continuation. Small exports return inline; larger exports become private JSONL retained for seven days with a 15-minute signed URL. Attachments remain metadata-only and oversized records are truncated with warnings.
|
|
182
182
|
- `renew_connected_data_download` — issue a fresh 15-minute signed URL for an unexpired private export artifact without pulling the provider again.
|
|
183
|
-
- `read_service_connection` — run one small live read by exact allowlisted name across Nango OAuth or official remote MCP connections. Do not loop it over a time range
|
|
184
|
-
- `call_service_connection_action` — run one exact mutation only after the account action switch is enabled. Resend, GitHub, Gmail, Calendar, Slack, Zoom, Facebook, Google Analytics, X, and YouTube actions remain provider-policy and
|
|
183
|
+
- `read_service_connection` — run one small live read by exact allowlisted name across Nango OAuth or official remote MCP connections, including bounded Google Drive inventory, change, Doc, Sheet, and text-file tools. Do not loop it over a time range when `export_connected_service_data` supports that provider's collection.
|
|
184
|
+
- `call_service_connection_action` — run one exact mutation only after the account action switch is enabled. Google Drive, Resend, GitHub, Gmail, Calendar, Slack, Zoom, Facebook, Google Analytics, X, and YouTube actions remain live-inventory, provider-policy, tenant, and audit gated; credential, OAuth-grant, and raw editor-session administration is never exposed.
|
|
185
185
|
- `set_scheduled_action_connections` — bind exact tenant connections and exact tool grants to one scheduled action. Deterministic connection syncs are model-free and one schedule occurrence counts as one metered event even when provider pagination spans many pages.
|
|
186
186
|
|
|
187
187
|
GitHub uses the same provider-neutral read/action bridges: 27 live reads cover repository discovery, issues, pull requests, commits, releases, and workflows; 20 live mutations remain account- and schedule-gated. The 47 Nango functions and expanded reconnect scopes are deployed. The current test token remains `public_repo` until owners approve its two pending organization requests, so private-repository and workflow parity is not yet claimed.
|
|
188
188
|
|
|
189
|
+
Google Drive Phase 1 exposes ten bounded reads plus the separately gated `create-drive-folder` and `copy-drive-file` actions through those same bridges and through per-run Mastra toolsets. It is interactive tool support, not the later deterministic bulk-export/`connection_sync` phase; large binary delivery and broader Drive mutations remain unreleased.
|
|
190
|
+
|
|
189
191
|
### Browser-agent tools
|
|
190
192
|
|
|
191
193
|
- `browser_open` — open a direct/no-proxy hosted browser session. Pass a saved profile name to load an authenticated session. Returns a human `watch_url`.
|
package/dist/bin/api-server.cjs
CHANGED
|
@@ -27642,7 +27642,7 @@ var PACKAGE_VERSION;
|
|
|
27642
27642
|
var init_version = __esm({
|
|
27643
27643
|
"src/version.ts"() {
|
|
27644
27644
|
"use strict";
|
|
27645
|
-
PACKAGE_VERSION = "0.
|
|
27645
|
+
PACKAGE_VERSION = "0.16.0";
|
|
27646
27646
|
}
|
|
27647
27647
|
});
|
|
27648
27648
|
|
|
@@ -28854,6 +28854,9 @@ var init_mcp_tool_schemas = __esm({
|
|
|
28854
28854
|
readTools: import_zod33.z.array(import_zod33.z.string()).describe("Tool names this connection can be read with via read_service_connection."),
|
|
28855
28855
|
actionTools: import_zod33.z.array(import_zod33.z.string()).describe("Explicitly allowlisted write or mutation tool names callable through call_service_connection_action after actions are enabled for this connection."),
|
|
28856
28856
|
adminBlockedTools: import_zod33.z.array(import_zod33.z.string()).describe("Credential, OAuth-grant, or other administrative tools permanently blocked from the MCP and scheduler."),
|
|
28857
|
+
mcpEndpoint: import_zod33.z.string().url().nullable().describe("Authenticated connection-scoped MCP endpoint when native provider tools/list projection is available. Null means use describe_service_connection_tool on this root MCP."),
|
|
28858
|
+
schemaDiscovery: import_zod33.z.enum(["connection_tools_list", "compatibility_describe"]).describe("How clients discover this connection's exact live provider schemas."),
|
|
28859
|
+
toolRevision: import_zod33.z.string().nullable().describe("Opaque revision of the resolved live tool catalog, when available. It changes when provider tools or policy change."),
|
|
28857
28860
|
vaultName: import_zod33.z.string().nullable().describe("Memory vault this connection's digest writes into, if it has run at least once. Search it with memory-search."),
|
|
28858
28861
|
tableName: import_zod33.z.string().nullable().describe("Table this connection's digest writes structured rows into, if it has run at least once. Query it with table-query.")
|
|
28859
28862
|
}))
|
|
@@ -28870,7 +28873,8 @@ var init_mcp_tool_schemas = __esm({
|
|
|
28870
28873
|
};
|
|
28871
28874
|
DescribeServiceConnectionToolInputSchema = {
|
|
28872
28875
|
connectionId: import_zod33.z.string().min(1).describe("A tenant-owned connectionId from list_service_connections."),
|
|
28873
|
-
tool: import_zod33.z.string().min(1).describe("One exact name from that connection's readTools or actionTools. Admin-blocked and arbitrary names are rejected.")
|
|
28876
|
+
tool: import_zod33.z.string().min(1).describe("One exact name from that connection's readTools or actionTools. Admin-blocked and arbitrary names are rejected."),
|
|
28877
|
+
fresh: import_zod33.z.boolean().optional().describe("Bypass the short-lived sanitized schema cache. Ownership, connection state, and tool policy are still rechecked; use only when a provider tool catalog just changed.")
|
|
28874
28878
|
};
|
|
28875
28879
|
DescribeServiceConnectionToolOutputSchema = {
|
|
28876
28880
|
ok: import_zod33.z.boolean(),
|
|
@@ -28879,8 +28883,33 @@ var init_mcp_tool_schemas = __esm({
|
|
|
28879
28883
|
title: NullableString,
|
|
28880
28884
|
description: NullableString,
|
|
28881
28885
|
classification: import_zod33.z.enum(["read", "action"]),
|
|
28882
|
-
|
|
28886
|
+
callable: import_zod33.z.boolean().optional().describe("Whether the tool is callable now. A gated action can be described while actions are off and return false."),
|
|
28887
|
+
blockedReason: import_zod33.z.enum(["actions_disabled", "inactive_connection"]).nullable().optional(),
|
|
28888
|
+
transport: import_zod33.z.enum(["nango", "remote_mcp"]).optional(),
|
|
28889
|
+
providerConfigKey: import_zod33.z.string().optional(),
|
|
28890
|
+
protocolVersion: NullableString.optional(),
|
|
28891
|
+
schemaSource: import_zod33.z.literal("live_tools_list").optional(),
|
|
28892
|
+
inputSchema: import_zod33.z.record(import_zod33.z.string(), import_zod33.z.unknown()).describe("JSON Schema for the exact connected-provider tool arguments."),
|
|
28893
|
+
outputSchema: import_zod33.z.record(import_zod33.z.string(), import_zod33.z.unknown()).optional().describe("Provider-native JSON output schema when the live MCP tool publishes one."),
|
|
28894
|
+
annotations: import_zod33.z.object({
|
|
28895
|
+
title: import_zod33.z.string().optional(),
|
|
28896
|
+
readOnlyHint: import_zod33.z.boolean().optional(),
|
|
28897
|
+
destructiveHint: import_zod33.z.boolean().optional(),
|
|
28898
|
+
idempotentHint: import_zod33.z.boolean().optional(),
|
|
28899
|
+
openWorldHint: import_zod33.z.boolean().optional()
|
|
28900
|
+
}).optional(),
|
|
28901
|
+
icons: import_zod33.z.array(import_zod33.z.object({
|
|
28902
|
+
src: import_zod33.z.string(),
|
|
28903
|
+
mimeType: import_zod33.z.string().optional(),
|
|
28904
|
+
sizes: import_zod33.z.array(import_zod33.z.string()).optional(),
|
|
28905
|
+
theme: import_zod33.z.enum(["light", "dark"]).optional()
|
|
28906
|
+
})).optional(),
|
|
28907
|
+
execution: import_zod33.z.object({ taskSupport: import_zod33.z.enum(["forbidden", "optional", "required"]) }).optional(),
|
|
28908
|
+
schemaHash: import_zod33.z.string().optional().describe("SHA-256 of the sanitized live Tool definition."),
|
|
28909
|
+
fetchedAt: import_zod33.z.string().optional()
|
|
28883
28910
|
}).optional(),
|
|
28911
|
+
retryable: import_zod33.z.boolean().optional(),
|
|
28912
|
+
errorCode: import_zod33.z.string().optional(),
|
|
28884
28913
|
error: NullableString
|
|
28885
28914
|
};
|
|
28886
28915
|
ConnectedDataContinuationSchema = import_zod33.z.object({
|
|
@@ -29696,7 +29725,7 @@ function registerPaaExtractorMcpTools(server, executor, options = {}) {
|
|
|
29696
29725
|
}, async (input) => formatCreditsInfo(await executor.creditsInfo(input), input));
|
|
29697
29726
|
server.registerTool("list_service_connections", {
|
|
29698
29727
|
title: "List Connected Services",
|
|
29699
|
-
description: "List every third-party service connection this MCP Scraper account has authorized, including Resend, GitHub, Google Analytics, YouTube, Facebook Pages, LinkedIn, X, Meta Marketing, Slack, Gmail, Calendar, Drive, Zoom, Xero, and others. Returns the tenant-scoped connectionId, credential transport, exact readTools and actionTools,
|
|
29728
|
+
description: "List every third-party service connection this MCP Scraper account has authorized, including Resend, GitHub, Google Analytics, YouTube, Facebook Pages, LinkedIn, X, Meta Marketing, Slack, Gmail, Calendar, Google Drive, Zoom, Xero, and others. Returns the tenant-scoped connectionId, credential transport, exact live readTools and gated actionTools, permanently blocked administrative tools, and schema-discovery metadata. Get a connectionId and exact tool name here before calling describe_service_connection_tool, read_service_connection, or call_service_connection_action. Nango OAuth and official remote MCP connections use the same provider-neutral bridges; mutations still require the account action switch and an exact allowed action. For already-digested history, prefer the returned vaultName or tableName.",
|
|
29700
29729
|
inputSchema: ListServiceConnectionsInputSchema,
|
|
29701
29730
|
outputSchema: recordOutputSchema("list_service_connections", ListServiceConnectionsOutputSchema),
|
|
29702
29731
|
annotations: { title: "List Connected Services", readOnlyHint: true, destructiveHint: false, idempotentHint: true, openWorldHint: false }
|
|
@@ -29731,14 +29760,14 @@ function registerPaaExtractorMcpTools(server, executor, options = {}) {
|
|
|
29731
29760
|
}, async (input) => executor.zoomCreateMeeting(input));
|
|
29732
29761
|
server.registerTool("read_service_connection", {
|
|
29733
29762
|
title: "Read Connected Service",
|
|
29734
|
-
description: "Call one small live, read-only operation on any connected service, including Resend, GitHub
|
|
29763
|
+
description: "Call one small live, read-only operation on any connected service, including Google Drive metadata/search tools, Resend, GitHub, Gmail, Calendar, Zoom, and other approved providers. Call describe_service_connection_tool first when arguments are not already known. Do not loop this tool once per file or record to fetch a corpus: use export_connected_service_data when that provider/dataset supports bulk delivery. Requires a connectionId and an exact name from that connection's live readTools in list_service_connections; an unlisted tool is rejected server-side.",
|
|
29735
29764
|
inputSchema: ReadServiceConnectionInputSchema,
|
|
29736
29765
|
outputSchema: recordOutputSchema("read_service_connection", ReadServiceConnectionOutputSchema),
|
|
29737
29766
|
annotations: { title: "Read Connected Service", readOnlyHint: true, destructiveHint: false, idempotentHint: true, openWorldHint: true }
|
|
29738
29767
|
}, async (input) => executor.readServiceConnection(input));
|
|
29739
29768
|
server.registerTool("describe_service_connection_tool", {
|
|
29740
29769
|
title: "Describe Connected Service Tool",
|
|
29741
|
-
description: "
|
|
29770
|
+
description: "Fetch the sanitized live MCP Tool definition for one exact tool exposed by a tenant-owned Nango OAuth or official remote MCP connection. Returns provider-native title, description, read/action classification, current callability, input schema, optional output schema, safe annotations, and a schema hash. Call list_service_connections first, then describe a listed readTools or actionTools name before constructing arguments. This is a compatibility tool on MCP Scraper's fixed root MCP; protocol-native connection endpoints discover the same definitions through MCP tools/list, not a custom tools/describe method. Arbitrary names and permanently blocked administrative tools are rejected.",
|
|
29742
29771
|
inputSchema: DescribeServiceConnectionToolInputSchema,
|
|
29743
29772
|
outputSchema: recordOutputSchema("describe_service_connection_tool", DescribeServiceConnectionToolOutputSchema),
|
|
29744
29773
|
annotations: { title: "Describe Connected Service Tool", readOnlyHint: true, destructiveHint: false, idempotentHint: true, openWorldHint: false }
|
|
@@ -29759,7 +29788,7 @@ function registerPaaExtractorMcpTools(server, executor, options = {}) {
|
|
|
29759
29788
|
}, async (input) => executor.renewConnectedDataDownload(input));
|
|
29760
29789
|
server.registerTool("call_service_connection_action", {
|
|
29761
29790
|
title: "Run Connected Service Action",
|
|
29762
|
-
description: "Run one explicitly allowlisted write or mutation on a tenant-owned OAuth or remote MCP connection. First call list_service_connections, use a connection with actionsEnabled true,
|
|
29791
|
+
description: "Run one explicitly allowlisted write or mutation on a tenant-owned OAuth or remote MCP connection. First call list_service_connections, use a connection with actionsEnabled true, describe the exact actionTools entry to obtain its live schema, and supply only that action's arguments. The server rejects arbitrary action names, inactive or foreign connections, disabled actions, and every adminBlockedTools entry. This can include Google Drive folder creation or file copies, Resend delivery, and GitHub mutations only when those exact actions are live and approved. Sends, deletes, merges, workflow execution, and content changes are high impact.",
|
|
29763
29792
|
inputSchema: CallServiceConnectionActionInputSchema,
|
|
29764
29793
|
outputSchema: recordOutputSchema("call_service_connection_action", CallServiceConnectionActionOutputSchema),
|
|
29765
29794
|
annotations: { title: "Run Connected Service Action", readOnlyHint: false, destructiveHint: true, idempotentHint: false, openWorldHint: true }
|
|
@@ -29824,12 +29853,15 @@ async function readResponseData(res) {
|
|
|
29824
29853
|
}
|
|
29825
29854
|
function httpErrorPayload(path6, res, data) {
|
|
29826
29855
|
const objectData = data && typeof data === "object" && !Array.isArray(data) ? data : null;
|
|
29827
|
-
const
|
|
29856
|
+
const rawCode = objectData?.code ?? objectData?.errorCode ?? objectData?.error_code;
|
|
29857
|
+
const safeCode = typeof rawCode === "string" && /^[a-z0-9][a-z0-9_-]{0,99}$/.test(rawCode) ? rawCode : "mcp_http_error";
|
|
29858
|
+
const bodyMessage = objectData ? objectData.message ?? objectData.error ?? rawCode : data;
|
|
29828
29859
|
return {
|
|
29829
29860
|
...objectData ?? { body: data },
|
|
29830
|
-
error:
|
|
29861
|
+
error: safeCode,
|
|
29862
|
+
error_code: safeCode,
|
|
29831
29863
|
error_type: "http",
|
|
29832
|
-
retryable: res.status === 429 || res.status >= 500,
|
|
29864
|
+
retryable: typeof objectData?.retryable === "boolean" ? objectData.retryable : res.status === 429 || res.status >= 500,
|
|
29833
29865
|
status: res.status,
|
|
29834
29866
|
statusText: res.statusText,
|
|
29835
29867
|
path: path6,
|
|
@@ -39522,6 +39554,42 @@ function arrayFromPayload(value, keys) {
|
|
|
39522
39554
|
}
|
|
39523
39555
|
return [];
|
|
39524
39556
|
}
|
|
39557
|
+
function controlErrorStatus(status) {
|
|
39558
|
+
if (status === 400 || status === 403 || status === 404 || status === 409 || status === 429 || status === 502 || status === 503) {
|
|
39559
|
+
return status;
|
|
39560
|
+
}
|
|
39561
|
+
return status >= 500 ? 503 : 502;
|
|
39562
|
+
}
|
|
39563
|
+
function defaultControlErrorCode(status) {
|
|
39564
|
+
if (status === 400) return "invalid_request";
|
|
39565
|
+
if (status === 403) return "actions_disabled";
|
|
39566
|
+
if (status === 404) return "connection_not_found";
|
|
39567
|
+
if (status === 409) return "connection_inactive";
|
|
39568
|
+
if (status === 429) return "upstream_rate_limited";
|
|
39569
|
+
if (status === 503) return "connection_transport_unavailable";
|
|
39570
|
+
return "connection_control_failed";
|
|
39571
|
+
}
|
|
39572
|
+
function defaultControlErrorMessage(status) {
|
|
39573
|
+
if (status === 400) return "The connection request was invalid.";
|
|
39574
|
+
if (status === 403) return "Actions are disabled for this service connection.";
|
|
39575
|
+
if (status === 404) return "The service connection was not found.";
|
|
39576
|
+
if (status === 409) return "The service connection is not currently usable.";
|
|
39577
|
+
if (status === 429) return "The connected service is temporarily rate limited.";
|
|
39578
|
+
if (status === 503) return "The service connection transport is temporarily unavailable.";
|
|
39579
|
+
return "The service connection control request failed.";
|
|
39580
|
+
}
|
|
39581
|
+
function safeControlErrorPayload(body, responseStatus) {
|
|
39582
|
+
const status = controlErrorStatus(responseStatus);
|
|
39583
|
+
const unwrapped = unwrapData(body);
|
|
39584
|
+
const record = isRecord(unwrapped) ? unwrapped : isRecord(body) ? body : null;
|
|
39585
|
+
const candidateCode = record ? firstString(record, ["code", "errorCode", "error_code"], 100) : null;
|
|
39586
|
+
const normalizedCandidateCode = candidateCode ? CONTROL_ERROR_CODE_ALIASES.get(candidateCode) ?? candidateCode : null;
|
|
39587
|
+
const code = normalizedCandidateCode && SAFE_CONTROL_ERROR_CODES.has(normalizedCandidateCode) ? normalizedCandidateCode : defaultControlErrorCode(status);
|
|
39588
|
+
const candidateMessage = record ? firstString(record, ["error", "message"], 500) : null;
|
|
39589
|
+
const message = normalizedCandidateCode && SAFE_CONTROL_ERROR_CODES.has(normalizedCandidateCode) && candidateMessage ? candidateMessage.replace(/[\u0000-\u001f\u007f]/g, " ") : defaultControlErrorMessage(status);
|
|
39590
|
+
const retryable = record && typeof record.retryable === "boolean" ? record.retryable : status === 429 || status >= 500;
|
|
39591
|
+
return { status, code, message, retryable };
|
|
39592
|
+
}
|
|
39525
39593
|
async function controlRequest(path6, init, timeoutMs = 2e4) {
|
|
39526
39594
|
const response = await fetch(`${controlBaseUrl()}${path6}`, {
|
|
39527
39595
|
...init,
|
|
@@ -39533,7 +39601,13 @@ async function controlRequest(path6, init, timeoutMs = 2e4) {
|
|
|
39533
39601
|
},
|
|
39534
39602
|
signal: AbortSignal.timeout(timeoutMs)
|
|
39535
39603
|
}).catch((err) => {
|
|
39536
|
-
|
|
39604
|
+
const cause = err instanceof Error && err.name === "TimeoutError" ? "request timed out" : "network error";
|
|
39605
|
+
throw new NangoControlError(
|
|
39606
|
+
`Scheduled service connection control is unavailable: ${cause}.`,
|
|
39607
|
+
503,
|
|
39608
|
+
"connection_transport_unavailable",
|
|
39609
|
+
true
|
|
39610
|
+
);
|
|
39537
39611
|
});
|
|
39538
39612
|
const text = await response.text();
|
|
39539
39613
|
let body = {};
|
|
@@ -39543,8 +39617,8 @@ async function controlRequest(path6, init, timeoutMs = 2e4) {
|
|
|
39543
39617
|
body = {};
|
|
39544
39618
|
}
|
|
39545
39619
|
if (!response.ok) {
|
|
39546
|
-
const
|
|
39547
|
-
throw new NangoControlError(
|
|
39620
|
+
const error = safeControlErrorPayload(body, response.status);
|
|
39621
|
+
throw new NangoControlError(error.message, error.status, error.code, error.retryable);
|
|
39548
39622
|
}
|
|
39549
39623
|
return body;
|
|
39550
39624
|
}
|
|
@@ -39639,6 +39713,9 @@ async function getNangoConnections(identity) {
|
|
|
39639
39713
|
actionsEnabled: row.actionsEnabled === true || row.actions_enabled === true,
|
|
39640
39714
|
readTools: cleanTools(row.readTools ?? row.read_tools),
|
|
39641
39715
|
actionTools: cleanTools(row.actionTools ?? row.action_tools ?? row.writeTools ?? row.write_tools),
|
|
39716
|
+
mcpEndpoint: null,
|
|
39717
|
+
schemaDiscovery: "compatibility_describe",
|
|
39718
|
+
toolRevision: firstString(row, ["toolRevision", "tool_revision"], 200),
|
|
39642
39719
|
vaultName: firstString(row, ["vaultName", "vault_name"], 100),
|
|
39643
39720
|
tableName: firstString(row, ["tableName", "table_name"], 100),
|
|
39644
39721
|
createdAt: firstString(row, ["createdAt", "created_at"], 100),
|
|
@@ -39771,6 +39848,145 @@ async function callScheduleConnectionRead(identity, connectionId, tool, args) {
|
|
|
39771
39848
|
const data = unwrapData(body);
|
|
39772
39849
|
return isRecord(data) ? data.result ?? data : data;
|
|
39773
39850
|
}
|
|
39851
|
+
function sanitizeToolSchema(value) {
|
|
39852
|
+
if (!isRecord(value) || value.type !== "object") return null;
|
|
39853
|
+
try {
|
|
39854
|
+
const serialized = JSON.stringify(value);
|
|
39855
|
+
if (Buffer.byteLength(serialized, "utf8") > 256 * 1024) return null;
|
|
39856
|
+
const cloned = JSON.parse(serialized);
|
|
39857
|
+
return isRecord(cloned) ? cloned : null;
|
|
39858
|
+
} catch {
|
|
39859
|
+
return null;
|
|
39860
|
+
}
|
|
39861
|
+
}
|
|
39862
|
+
function sanitizeToolAnnotations(value) {
|
|
39863
|
+
if (!isRecord(value)) return void 0;
|
|
39864
|
+
const annotations = {};
|
|
39865
|
+
const title = cleanString(value.title, 200);
|
|
39866
|
+
if (title) annotations.title = title;
|
|
39867
|
+
for (const key of ["readOnlyHint", "destructiveHint", "idempotentHint", "openWorldHint"]) {
|
|
39868
|
+
if (typeof value[key] === "boolean") annotations[key] = value[key];
|
|
39869
|
+
}
|
|
39870
|
+
return Object.keys(annotations).length > 0 ? annotations : void 0;
|
|
39871
|
+
}
|
|
39872
|
+
function sanitizeToolIconSource(value) {
|
|
39873
|
+
const candidate = cleanString(value, 1e4);
|
|
39874
|
+
if (!candidate) return null;
|
|
39875
|
+
if (/^data:image\/(?:png|jpeg|webp|gif|svg\+xml);base64,[a-z0-9+/=]+$/i.test(candidate)) return candidate;
|
|
39876
|
+
return cleanHttpsUrl(candidate);
|
|
39877
|
+
}
|
|
39878
|
+
function sanitizeToolIcons(value) {
|
|
39879
|
+
if (!Array.isArray(value)) return void 0;
|
|
39880
|
+
const icons = [];
|
|
39881
|
+
for (const item of value.slice(0, 16)) {
|
|
39882
|
+
if (!isRecord(item)) continue;
|
|
39883
|
+
const src = sanitizeToolIconSource(item.src);
|
|
39884
|
+
if (!src) continue;
|
|
39885
|
+
const mimeType = cleanString(item.mimeType ?? item.mime_type, 100);
|
|
39886
|
+
const sizes = cleanStringArray(item.sizes, 16, 32);
|
|
39887
|
+
const theme = item.theme === "light" || item.theme === "dark" ? item.theme : void 0;
|
|
39888
|
+
icons.push({
|
|
39889
|
+
src,
|
|
39890
|
+
...mimeType ? { mimeType } : {},
|
|
39891
|
+
...sizes.length > 0 ? { sizes } : {},
|
|
39892
|
+
...theme ? { theme } : {}
|
|
39893
|
+
});
|
|
39894
|
+
}
|
|
39895
|
+
return icons.length > 0 ? icons : void 0;
|
|
39896
|
+
}
|
|
39897
|
+
function canonicalJson(value) {
|
|
39898
|
+
if (Array.isArray(value)) return `[${value.map(canonicalJson).join(",")}]`;
|
|
39899
|
+
if (isRecord(value)) {
|
|
39900
|
+
return `{${Object.keys(value).sort().map((key) => `${JSON.stringify(key)}:${canonicalJson(value[key])}`).join(",")}}`;
|
|
39901
|
+
}
|
|
39902
|
+
return JSON.stringify(value);
|
|
39903
|
+
}
|
|
39904
|
+
function projectedToolSchemaHash(tool) {
|
|
39905
|
+
return (0, import_node_crypto17.createHash)("sha256").update(canonicalJson(tool)).digest("hex");
|
|
39906
|
+
}
|
|
39907
|
+
async function describeNangoTool(identity, connectionId, tool, fresh) {
|
|
39908
|
+
const body = await controlRequest("/api/internal/nango/connections/actions/describe", {
|
|
39909
|
+
method: "POST",
|
|
39910
|
+
body: JSON.stringify({
|
|
39911
|
+
identity,
|
|
39912
|
+
connectionId,
|
|
39913
|
+
tool,
|
|
39914
|
+
...fresh === void 0 ? {} : { fresh }
|
|
39915
|
+
})
|
|
39916
|
+
});
|
|
39917
|
+
const data = unwrapData(body);
|
|
39918
|
+
const rawTool = isRecord(data) && isRecord(data.tool) ? data.tool : data;
|
|
39919
|
+
if (!isRecord(rawTool)) {
|
|
39920
|
+
throw new NangoControlError(
|
|
39921
|
+
"The connection service returned an invalid live tool description.",
|
|
39922
|
+
502,
|
|
39923
|
+
"tool_discovery_failed",
|
|
39924
|
+
true
|
|
39925
|
+
);
|
|
39926
|
+
}
|
|
39927
|
+
let serializedBytes = Number.POSITIVE_INFINITY;
|
|
39928
|
+
try {
|
|
39929
|
+
serializedBytes = Buffer.byteLength(JSON.stringify(rawTool), "utf8");
|
|
39930
|
+
} catch {
|
|
39931
|
+
}
|
|
39932
|
+
const name = cleanString(rawTool.name, 200);
|
|
39933
|
+
const classification = cleanString(rawTool.classification, 20);
|
|
39934
|
+
const transport = cleanString(rawTool.transport, 20);
|
|
39935
|
+
const providerConfigKey = cleanString(rawTool.providerConfigKey ?? rawTool.provider_config_key, 200);
|
|
39936
|
+
const schemaSource = cleanString(rawTool.schemaSource ?? rawTool.schema_source, 50);
|
|
39937
|
+
const upstreamSchemaHash = cleanString(rawTool.schemaHash ?? rawTool.schema_hash, 200);
|
|
39938
|
+
const fetchedAt = cleanString(rawTool.fetchedAt ?? rawTool.fetched_at, 100);
|
|
39939
|
+
const inputSchema = sanitizeToolSchema(rawTool.inputSchema ?? rawTool.input_schema);
|
|
39940
|
+
const outputSchemaValue = rawTool.outputSchema ?? rawTool.output_schema;
|
|
39941
|
+
const outputSchema = outputSchemaValue === void 0 ? void 0 : sanitizeToolSchema(outputSchemaValue);
|
|
39942
|
+
const blockedReasonValue = rawTool.blockedReason ?? rawTool.blocked_reason;
|
|
39943
|
+
const blockedReason = blockedReasonValue === null || blockedReasonValue === void 0 ? null : blockedReasonValue === "actions_disabled" || blockedReasonValue === "inactive_connection" ? blockedReasonValue : void 0;
|
|
39944
|
+
if (serializedBytes > 256 * 1024 || !name || name !== tool || classification !== "read" && classification !== "action" || typeof rawTool.callable !== "boolean" || blockedReason === void 0 || transport !== "nango" && transport !== "remote_mcp" || !providerConfigKey || schemaSource !== "live_tools_list" || !upstreamSchemaHash || !/^[a-f0-9]{64}$/i.test(upstreamSchemaHash) || !fetchedAt || Number.isNaN(Date.parse(fetchedAt)) || !inputSchema || outputSchemaValue !== void 0 && !outputSchema) {
|
|
39945
|
+
throw new NangoControlError(
|
|
39946
|
+
"The connection service returned an invalid live tool description.",
|
|
39947
|
+
502,
|
|
39948
|
+
"tool_discovery_failed",
|
|
39949
|
+
true
|
|
39950
|
+
);
|
|
39951
|
+
}
|
|
39952
|
+
const protocolVersion = rawTool.protocolVersion === null || rawTool.protocol_version === null ? null : cleanString(rawTool.protocolVersion ?? rawTool.protocol_version, 100);
|
|
39953
|
+
const executionValue = rawTool.execution;
|
|
39954
|
+
const taskSupport = isRecord(executionValue) && (executionValue.taskSupport === "forbidden" || executionValue.taskSupport === "optional" || executionValue.taskSupport === "required") ? executionValue.taskSupport : null;
|
|
39955
|
+
const title = cleanString(rawTool.title, 200);
|
|
39956
|
+
const description = cleanString(rawTool.description, 12e3);
|
|
39957
|
+
const annotations = sanitizeToolAnnotations(rawTool.annotations);
|
|
39958
|
+
const icons = sanitizeToolIcons(rawTool.icons);
|
|
39959
|
+
const execution = taskSupport ? { taskSupport } : void 0;
|
|
39960
|
+
const schemaHash = projectedToolSchemaHash({
|
|
39961
|
+
name,
|
|
39962
|
+
inputSchema,
|
|
39963
|
+
...title ? { title } : {},
|
|
39964
|
+
...description ? { description } : {},
|
|
39965
|
+
...outputSchema ? { outputSchema } : {},
|
|
39966
|
+
...annotations ? { annotations } : {},
|
|
39967
|
+
...icons ? { icons } : {},
|
|
39968
|
+
...execution ? { execution } : {}
|
|
39969
|
+
});
|
|
39970
|
+
return {
|
|
39971
|
+
name,
|
|
39972
|
+
title,
|
|
39973
|
+
description,
|
|
39974
|
+
classification,
|
|
39975
|
+
callable: rawTool.callable,
|
|
39976
|
+
blockedReason,
|
|
39977
|
+
transport,
|
|
39978
|
+
providerConfigKey,
|
|
39979
|
+
protocolVersion,
|
|
39980
|
+
schemaSource: "live_tools_list",
|
|
39981
|
+
inputSchema,
|
|
39982
|
+
...outputSchema ? { outputSchema } : {},
|
|
39983
|
+
...annotations ? { annotations } : {},
|
|
39984
|
+
...icons ? { icons } : {},
|
|
39985
|
+
...execution ? { execution } : {},
|
|
39986
|
+
schemaHash,
|
|
39987
|
+
fetchedAt
|
|
39988
|
+
};
|
|
39989
|
+
}
|
|
39774
39990
|
async function callScheduleConnectionExportPage(identity, input) {
|
|
39775
39991
|
const body = await controlRequest("/api/internal/nango/connections/export-page", {
|
|
39776
39992
|
method: "POST",
|
|
@@ -39808,10 +40024,11 @@ async function callScheduleConnectionExportPage(identity, input) {
|
|
|
39808
40024
|
untrustedContent: true
|
|
39809
40025
|
};
|
|
39810
40026
|
}
|
|
39811
|
-
var DEFAULT_NANGO_CONTROL_URL, DISABLED_NANGO_TOOLS, CONNECTION_SYNC_REQUIRED_TOOLS, NangoControlError, ScheduleConnectionValidationError;
|
|
40027
|
+
var import_node_crypto17, DEFAULT_NANGO_CONTROL_URL, DISABLED_NANGO_TOOLS, CONNECTION_SYNC_REQUIRED_TOOLS, NangoControlError, ScheduleConnectionValidationError, SAFE_CONTROL_ERROR_CODES, CONTROL_ERROR_CODE_ALIASES;
|
|
39812
40028
|
var init_nango_control = __esm({
|
|
39813
40029
|
"src/api/nango-control.ts"() {
|
|
39814
40030
|
"use strict";
|
|
40031
|
+
import_node_crypto17 = require("crypto");
|
|
39815
40032
|
init_connected_data_export();
|
|
39816
40033
|
DEFAULT_NANGO_CONTROL_URL = "https://mcp-scraper-scheduler.vercel.app";
|
|
39817
40034
|
DISABLED_NANGO_TOOLS = {
|
|
@@ -39854,10 +40071,14 @@ var init_nango_control = __esm({
|
|
|
39854
40071
|
};
|
|
39855
40072
|
NangoControlError = class extends Error {
|
|
39856
40073
|
status;
|
|
39857
|
-
|
|
40074
|
+
code;
|
|
40075
|
+
retryable;
|
|
40076
|
+
constructor(message, status = 502, code = status === 503 ? "connection_transport_unavailable" : "connection_control_failed", retryable = status === 429 || status >= 500) {
|
|
39858
40077
|
super(message);
|
|
39859
40078
|
this.name = "NangoControlError";
|
|
39860
40079
|
this.status = status;
|
|
40080
|
+
this.code = code;
|
|
40081
|
+
this.retryable = retryable;
|
|
39861
40082
|
}
|
|
39862
40083
|
};
|
|
39863
40084
|
ScheduleConnectionValidationError = class extends Error {
|
|
@@ -39866,6 +40087,27 @@ var init_nango_control = __esm({
|
|
|
39866
40087
|
this.name = "ScheduleConnectionValidationError";
|
|
39867
40088
|
}
|
|
39868
40089
|
};
|
|
40090
|
+
SAFE_CONTROL_ERROR_CODES = /* @__PURE__ */ new Set([
|
|
40091
|
+
"invalid_request",
|
|
40092
|
+
"tool_not_allowed",
|
|
40093
|
+
"invalid_schema",
|
|
40094
|
+
"connection_not_found",
|
|
40095
|
+
"connection_inactive",
|
|
40096
|
+
"actions_disabled",
|
|
40097
|
+
"action_not_allowed",
|
|
40098
|
+
"upstream_rate_limited",
|
|
40099
|
+
"tool_discovery_failed",
|
|
40100
|
+
"live_tool_missing",
|
|
40101
|
+
"connection_transport_unavailable",
|
|
40102
|
+
"connection_control_failed"
|
|
40103
|
+
]);
|
|
40104
|
+
CONTROL_ERROR_CODE_ALIASES = /* @__PURE__ */ new Map([
|
|
40105
|
+
["connection_not_active", "connection_inactive"],
|
|
40106
|
+
["actions_not_enabled", "actions_disabled"],
|
|
40107
|
+
["action_tool_not_allowed", "action_not_allowed"],
|
|
40108
|
+
["no_action_for_provider", "action_not_allowed"],
|
|
40109
|
+
["tool_not_available", "live_tool_missing"]
|
|
40110
|
+
]);
|
|
39869
40111
|
}
|
|
39870
40112
|
});
|
|
39871
40113
|
|
|
@@ -40015,6 +40257,9 @@ async function getResendConnections(identity) {
|
|
|
40015
40257
|
readTools: cleanTools2(row.readTools ?? row.read_tools),
|
|
40016
40258
|
actionTools: cleanTools2(row.actionTools ?? row.action_tools ?? row.writeTools ?? row.write_tools),
|
|
40017
40259
|
adminBlockedTools: cleanTools2(row.adminBlockedTools ?? row.admin_blocked_tools).length > 0 ? cleanTools2(row.adminBlockedTools ?? row.admin_blocked_tools) : [...RESEND_ADMIN_BLOCKED_TOOLS],
|
|
40260
|
+
mcpEndpoint: null,
|
|
40261
|
+
schemaDiscovery: "compatibility_describe",
|
|
40262
|
+
toolRevision: firstString2(row, ["toolRevision", "tool_revision"], 200),
|
|
40018
40263
|
vaultName: firstString2(row, ["vaultName", "vault_name"], 100),
|
|
40019
40264
|
tableName: firstString2(row, ["tableName", "table_name"], 100),
|
|
40020
40265
|
createdAt: firstString2(row, ["createdAt", "created_at"], 100),
|
|
@@ -40412,7 +40657,15 @@ function maybeProvisionInBackground(user, vaultCount) {
|
|
|
40412
40657
|
}
|
|
40413
40658
|
function scheduleConnectionError(c, err, fallback) {
|
|
40414
40659
|
if (err instanceof ScheduleConnectionValidationError) return c.json({ ok: false, error: err.message }, 400);
|
|
40415
|
-
if (err instanceof NangoControlError)
|
|
40660
|
+
if (err instanceof NangoControlError) {
|
|
40661
|
+
return c.json({
|
|
40662
|
+
ok: false,
|
|
40663
|
+
error: err.message,
|
|
40664
|
+
code: err.code,
|
|
40665
|
+
errorCode: err.code,
|
|
40666
|
+
retryable: err.retryable
|
|
40667
|
+
}, err.status);
|
|
40668
|
+
}
|
|
40416
40669
|
if (err instanceof ResendControlError) return c.json({ ok: false, error: err.message }, err.status);
|
|
40417
40670
|
console.error("[schedule-connections]", err instanceof Error ? err.message : String(err));
|
|
40418
40671
|
return c.json({ ok: false, error: fallback }, 502);
|
|
@@ -41296,11 +41549,12 @@ var init_server = __esm({
|
|
|
41296
41549
|
const providerConfigKey = providerConfigKeyFrom(body.providerConfigKey);
|
|
41297
41550
|
const tool = providerConfigKeyFrom(body.tool);
|
|
41298
41551
|
if (!connectionId || !tool) return c.json({ ok: false, error: "connectionId and tool are required." }, 400);
|
|
41552
|
+
if (body.fresh !== void 0 && typeof body.fresh !== "boolean") {
|
|
41553
|
+
return c.json({ ok: false, error: "fresh must be a boolean when provided." }, 400);
|
|
41554
|
+
}
|
|
41299
41555
|
try {
|
|
41300
|
-
|
|
41301
|
-
|
|
41302
|
-
}
|
|
41303
|
-
return c.json({ ok: true, tool: await describeResendTool(user.email, connectionId, tool) });
|
|
41556
|
+
const description = await isResendConnection(user.email, connectionId, providerConfigKey) ? await describeResendTool(user.email, connectionId, tool) : await describeNangoTool(user.email, connectionId, tool, body.fresh);
|
|
41557
|
+
return c.json({ ok: true, tool: description });
|
|
41304
41558
|
} catch (err) {
|
|
41305
41559
|
return scheduleConnectionError(c, err, "Unable to describe this connection tool.");
|
|
41306
41560
|
}
|