mcp-scraper 0.15.0 → 0.16.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +7 -5
- package/dist/bin/api-server.cjs +278 -20
- package/dist/bin/api-server.cjs.map +1 -1
- package/dist/bin/api-server.js +1 -1
- package/dist/bin/mcp-scraper-cli.cjs +1 -1
- package/dist/bin/mcp-scraper-cli.cjs.map +1 -1
- package/dist/bin/mcp-scraper-cli.js +1 -1
- package/dist/bin/mcp-scraper-install.cjs +1 -1
- package/dist/bin/mcp-scraper-install.cjs.map +1 -1
- package/dist/bin/mcp-scraper-install.js +1 -1
- package/dist/bin/mcp-stdio-server.cjs +42 -10
- package/dist/bin/mcp-stdio-server.cjs.map +1 -1
- package/dist/bin/mcp-stdio-server.js +2 -2
- package/dist/chunk-HQYIP5X3.js +7 -0
- package/dist/chunk-HQYIP5X3.js.map +1 -0
- package/dist/{chunk-ETJBTYZX.js → chunk-YIV4IKFG.js} +43 -11
- package/dist/chunk-YIV4IKFG.js.map +1 -0
- package/dist/{server-DDKXWYTO.js → server-QEXOVJPR.js} +238 -12
- package/dist/server-QEXOVJPR.js.map +1 -0
- package/docs/mcp-tool-manifest.generated.json +162 -6
- package/docs/specs/google-drive-bulk-access-and-mcp-schema-passthrough-spec.md +1680 -0
- package/package.json +1 -1
- package/dist/chunk-ETJBTYZX.js.map +0 -1
- package/dist/chunk-TK2S2M7G.js +0 -7
- package/dist/chunk-TK2S2M7G.js.map +0 -1
- package/dist/server-DDKXWYTO.js.map +0 -1
package/README.md
CHANGED
|
@@ -88,7 +88,7 @@ Build the branded one-click bundle:
|
|
|
88
88
|
npm run build:mcpb
|
|
89
89
|
```
|
|
90
90
|
|
|
91
|
-
The generated bundle is written to `build/mcpb/mcp-scraper-<version>.mcpb` and copied to `public/downloads/` for the hosted download. The current public bundle is `https://mcpscraper.dev/downloads/mcp-scraper.mcpb` (`0.
|
|
91
|
+
The generated bundle is written to `build/mcpb/mcp-scraper-<version>.mcpb` and copied to `public/downloads/` for the hosted download. The current public bundle is `https://mcpscraper.dev/downloads/mcp-scraper.mcpb` (`0.16.0`, SHA-256 `01623d11e3096cc867afee6c8abc345aaebf0eb15ffafdd875302a80cb90dd31`). Install it by opening or dragging it into Claude Desktop. Claude displays the `MCP Scraper` install card, icon, and API-key configuration field from the bundle manifest.
|
|
92
92
|
|
|
93
93
|
The MCPB install exposes every tool — web-intelligence plus all `browser_*` tools — through the one `mcp-scraper` server.
|
|
94
94
|
|
|
@@ -176,16 +176,18 @@ env = { MCP_SCRAPER_API_KEY = "sk_live_your_key" }
|
|
|
176
176
|
|
|
177
177
|
### Connected-account tools
|
|
178
178
|
|
|
179
|
-
- `list_service_connections` — list this caller's tenant-owned Nango OAuth and official remote MCP connections, including exact live reads, gated actions, permanently blocked administrative tools, and
|
|
180
|
-
- `describe_service_connection_tool` — fetch the
|
|
179
|
+
- `list_service_connections` — list this caller's tenant-owned Nango OAuth and official remote MCP connections, including exact live reads, gated actions, permanently blocked administrative tools, credential transport, and schema-discovery metadata. Connections are never shared between customers.
|
|
180
|
+
- `describe_service_connection_tool` — fetch the sanitized live MCP Tool definition for one tool listed on one tenant-owned connection, including its current callability, input schema, optional output schema, safe annotations, and schema hash. Use this before constructing provider-native arguments; provider functions stay behind the generic bridges instead of becoming dozens of permanent top-level tools.
|
|
181
181
|
- `export_connected_service_data` — fetch a bounded Gmail, Google Calendar, Zoom, or Resend time range in one MCP call. Resend's aggregate `resend_data` export walks 12 practical safe collections: sent mail, received mail, logs, contacts, broadcasts, templates, domains, segments, topics, webhooks, contact imports, and contact properties. The six core collections are also individually selectable. The server handles provider pagination, bounded detail retrieval, normalization, per-category warnings, and signed continuation. Small exports return inline; larger exports become private JSONL retained for seven days with a 15-minute signed URL. Attachments remain metadata-only and oversized records are truncated with warnings.
|
|
182
182
|
- `renew_connected_data_download` — issue a fresh 15-minute signed URL for an unexpired private export artifact without pulling the provider again.
|
|
183
|
-
- `read_service_connection` — run one small live read by exact allowlisted name across Nango OAuth or official remote MCP connections. Do not loop it over a time range
|
|
184
|
-
- `call_service_connection_action` — run one exact mutation only after the account action switch is enabled. Resend, GitHub, Gmail, Calendar, Slack, Zoom, Facebook, Google Analytics, X, and YouTube actions remain provider-policy and
|
|
183
|
+
- `read_service_connection` — run one small live read by exact allowlisted name across Nango OAuth or official remote MCP connections, including bounded Google Drive inventory, change, Doc, Sheet, and text-file tools. Do not loop it over a time range when `export_connected_service_data` supports that provider's collection.
|
|
184
|
+
- `call_service_connection_action` — run one exact mutation only after the account action switch is enabled. Google Drive, Resend, GitHub, Gmail, Calendar, Slack, Zoom, Facebook, Google Analytics, X, and YouTube actions remain live-inventory, provider-policy, tenant, and audit gated; credential, OAuth-grant, and raw editor-session administration is never exposed.
|
|
185
185
|
- `set_scheduled_action_connections` — bind exact tenant connections and exact tool grants to one scheduled action. Deterministic connection syncs are model-free and one schedule occurrence counts as one metered event even when provider pagination spans many pages.
|
|
186
186
|
|
|
187
187
|
GitHub uses the same provider-neutral read/action bridges: 27 live reads cover repository discovery, issues, pull requests, commits, releases, and workflows; 20 live mutations remain account- and schedule-gated. The 47 Nango functions and expanded reconnect scopes are deployed. The current test token remains `public_repo` until owners approve its two pending organization requests, so private-repository and workflow parity is not yet claimed.
|
|
188
188
|
|
|
189
|
+
Google Drive Phase 1 exposes ten bounded reads plus the separately gated `create-drive-folder` and `copy-drive-file` actions through those same bridges and through per-run Mastra toolsets. It is interactive tool support, not the later deterministic bulk-export/`connection_sync` phase; large binary delivery and broader Drive mutations remain unreleased.
|
|
190
|
+
|
|
189
191
|
### Browser-agent tools
|
|
190
192
|
|
|
191
193
|
- `browser_open` — open a direct/no-proxy hosted browser session. Pass a saved profile name to load an authenticated session. Returns a human `watch_url`.
|
package/dist/bin/api-server.cjs
CHANGED
|
@@ -27642,7 +27642,7 @@ var PACKAGE_VERSION;
|
|
|
27642
27642
|
var init_version = __esm({
|
|
27643
27643
|
"src/version.ts"() {
|
|
27644
27644
|
"use strict";
|
|
27645
|
-
PACKAGE_VERSION = "0.
|
|
27645
|
+
PACKAGE_VERSION = "0.16.0";
|
|
27646
27646
|
}
|
|
27647
27647
|
});
|
|
27648
27648
|
|
|
@@ -28854,6 +28854,9 @@ var init_mcp_tool_schemas = __esm({
|
|
|
28854
28854
|
readTools: import_zod33.z.array(import_zod33.z.string()).describe("Tool names this connection can be read with via read_service_connection."),
|
|
28855
28855
|
actionTools: import_zod33.z.array(import_zod33.z.string()).describe("Explicitly allowlisted write or mutation tool names callable through call_service_connection_action after actions are enabled for this connection."),
|
|
28856
28856
|
adminBlockedTools: import_zod33.z.array(import_zod33.z.string()).describe("Credential, OAuth-grant, or other administrative tools permanently blocked from the MCP and scheduler."),
|
|
28857
|
+
mcpEndpoint: import_zod33.z.string().url().nullable().describe("Authenticated connection-scoped MCP endpoint when native provider tools/list projection is available. Null means use describe_service_connection_tool on this root MCP."),
|
|
28858
|
+
schemaDiscovery: import_zod33.z.enum(["connection_tools_list", "compatibility_describe"]).describe("How clients discover this connection's exact live provider schemas."),
|
|
28859
|
+
toolRevision: import_zod33.z.string().nullable().describe("Opaque revision of the resolved live tool catalog, when available. It changes when provider tools or policy change."),
|
|
28857
28860
|
vaultName: import_zod33.z.string().nullable().describe("Memory vault this connection's digest writes into, if it has run at least once. Search it with memory-search."),
|
|
28858
28861
|
tableName: import_zod33.z.string().nullable().describe("Table this connection's digest writes structured rows into, if it has run at least once. Query it with table-query.")
|
|
28859
28862
|
}))
|
|
@@ -28870,7 +28873,8 @@ var init_mcp_tool_schemas = __esm({
|
|
|
28870
28873
|
};
|
|
28871
28874
|
DescribeServiceConnectionToolInputSchema = {
|
|
28872
28875
|
connectionId: import_zod33.z.string().min(1).describe("A tenant-owned connectionId from list_service_connections."),
|
|
28873
|
-
tool: import_zod33.z.string().min(1).describe("One exact name from that connection's readTools or actionTools. Admin-blocked and arbitrary names are rejected.")
|
|
28876
|
+
tool: import_zod33.z.string().min(1).describe("One exact name from that connection's readTools or actionTools. Admin-blocked and arbitrary names are rejected."),
|
|
28877
|
+
fresh: import_zod33.z.boolean().optional().describe("Bypass the short-lived sanitized schema cache. Ownership, connection state, and tool policy are still rechecked; use only when a provider tool catalog just changed.")
|
|
28874
28878
|
};
|
|
28875
28879
|
DescribeServiceConnectionToolOutputSchema = {
|
|
28876
28880
|
ok: import_zod33.z.boolean(),
|
|
@@ -28879,8 +28883,33 @@ var init_mcp_tool_schemas = __esm({
|
|
|
28879
28883
|
title: NullableString,
|
|
28880
28884
|
description: NullableString,
|
|
28881
28885
|
classification: import_zod33.z.enum(["read", "action"]),
|
|
28882
|
-
|
|
28886
|
+
callable: import_zod33.z.boolean().optional().describe("Whether the tool is callable now. A gated action can be described while actions are off and return false."),
|
|
28887
|
+
blockedReason: import_zod33.z.enum(["actions_disabled", "inactive_connection"]).nullable().optional(),
|
|
28888
|
+
transport: import_zod33.z.enum(["nango", "remote_mcp"]).optional(),
|
|
28889
|
+
providerConfigKey: import_zod33.z.string().optional(),
|
|
28890
|
+
protocolVersion: NullableString.optional(),
|
|
28891
|
+
schemaSource: import_zod33.z.literal("live_tools_list").optional(),
|
|
28892
|
+
inputSchema: import_zod33.z.record(import_zod33.z.string(), import_zod33.z.unknown()).describe("JSON Schema for the exact connected-provider tool arguments."),
|
|
28893
|
+
outputSchema: import_zod33.z.record(import_zod33.z.string(), import_zod33.z.unknown()).optional().describe("Provider-native JSON output schema when the live MCP tool publishes one."),
|
|
28894
|
+
annotations: import_zod33.z.object({
|
|
28895
|
+
title: import_zod33.z.string().optional(),
|
|
28896
|
+
readOnlyHint: import_zod33.z.boolean().optional(),
|
|
28897
|
+
destructiveHint: import_zod33.z.boolean().optional(),
|
|
28898
|
+
idempotentHint: import_zod33.z.boolean().optional(),
|
|
28899
|
+
openWorldHint: import_zod33.z.boolean().optional()
|
|
28900
|
+
}).optional(),
|
|
28901
|
+
icons: import_zod33.z.array(import_zod33.z.object({
|
|
28902
|
+
src: import_zod33.z.string(),
|
|
28903
|
+
mimeType: import_zod33.z.string().optional(),
|
|
28904
|
+
sizes: import_zod33.z.array(import_zod33.z.string()).optional(),
|
|
28905
|
+
theme: import_zod33.z.enum(["light", "dark"]).optional()
|
|
28906
|
+
})).optional(),
|
|
28907
|
+
execution: import_zod33.z.object({ taskSupport: import_zod33.z.enum(["forbidden", "optional", "required"]) }).optional(),
|
|
28908
|
+
schemaHash: import_zod33.z.string().optional().describe("SHA-256 of the sanitized live Tool definition."),
|
|
28909
|
+
fetchedAt: import_zod33.z.string().optional()
|
|
28883
28910
|
}).optional(),
|
|
28911
|
+
retryable: import_zod33.z.boolean().optional(),
|
|
28912
|
+
errorCode: import_zod33.z.string().optional(),
|
|
28884
28913
|
error: NullableString
|
|
28885
28914
|
};
|
|
28886
28915
|
ConnectedDataContinuationSchema = import_zod33.z.object({
|
|
@@ -29696,7 +29725,7 @@ function registerPaaExtractorMcpTools(server, executor, options = {}) {
|
|
|
29696
29725
|
}, async (input) => formatCreditsInfo(await executor.creditsInfo(input), input));
|
|
29697
29726
|
server.registerTool("list_service_connections", {
|
|
29698
29727
|
title: "List Connected Services",
|
|
29699
|
-
description: "List every third-party service connection this MCP Scraper account has authorized, including Resend, GitHub, Google Analytics, YouTube, Facebook Pages, LinkedIn, X, Meta Marketing, Slack, Gmail, Calendar, Drive, Zoom, Xero, and others. Returns the tenant-scoped connectionId, credential transport, exact readTools and actionTools,
|
|
29728
|
+
description: "List every third-party service connection this MCP Scraper account has authorized, including Resend, GitHub, Google Analytics, YouTube, Facebook Pages, LinkedIn, X, Meta Marketing, Slack, Gmail, Calendar, Google Drive, Zoom, Xero, and others. Returns the tenant-scoped connectionId, credential transport, exact live readTools and gated actionTools, permanently blocked administrative tools, and schema-discovery metadata. Get a connectionId and exact tool name here before calling describe_service_connection_tool, read_service_connection, or call_service_connection_action. Nango OAuth and official remote MCP connections use the same provider-neutral bridges; mutations still require the account action switch and an exact allowed action. For already-digested history, prefer the returned vaultName or tableName.",
|
|
29700
29729
|
inputSchema: ListServiceConnectionsInputSchema,
|
|
29701
29730
|
outputSchema: recordOutputSchema("list_service_connections", ListServiceConnectionsOutputSchema),
|
|
29702
29731
|
annotations: { title: "List Connected Services", readOnlyHint: true, destructiveHint: false, idempotentHint: true, openWorldHint: false }
|
|
@@ -29731,14 +29760,14 @@ function registerPaaExtractorMcpTools(server, executor, options = {}) {
|
|
|
29731
29760
|
}, async (input) => executor.zoomCreateMeeting(input));
|
|
29732
29761
|
server.registerTool("read_service_connection", {
|
|
29733
29762
|
title: "Read Connected Service",
|
|
29734
|
-
description: "Call one small live, read-only operation on any connected service, including Resend, GitHub
|
|
29763
|
+
description: "Call one small live, read-only operation on any connected service, including Google Drive metadata/search tools, Resend, GitHub, Gmail, Calendar, Zoom, and other approved providers. Call describe_service_connection_tool first when arguments are not already known. Do not loop this tool once per file or record to fetch a corpus: use export_connected_service_data when that provider/dataset supports bulk delivery. Requires a connectionId and an exact name from that connection's live readTools in list_service_connections; an unlisted tool is rejected server-side.",
|
|
29735
29764
|
inputSchema: ReadServiceConnectionInputSchema,
|
|
29736
29765
|
outputSchema: recordOutputSchema("read_service_connection", ReadServiceConnectionOutputSchema),
|
|
29737
29766
|
annotations: { title: "Read Connected Service", readOnlyHint: true, destructiveHint: false, idempotentHint: true, openWorldHint: true }
|
|
29738
29767
|
}, async (input) => executor.readServiceConnection(input));
|
|
29739
29768
|
server.registerTool("describe_service_connection_tool", {
|
|
29740
29769
|
title: "Describe Connected Service Tool",
|
|
29741
|
-
description: "
|
|
29770
|
+
description: "Fetch the sanitized live MCP Tool definition for one exact tool exposed by a tenant-owned Nango OAuth or official remote MCP connection. Returns provider-native title, description, read/action classification, current callability, input schema, optional output schema, safe annotations, and a schema hash. Call list_service_connections first, then describe a listed readTools or actionTools name before constructing arguments. This is a compatibility tool on MCP Scraper's fixed root MCP; protocol-native connection endpoints discover the same definitions through MCP tools/list, not a custom tools/describe method. Arbitrary names and permanently blocked administrative tools are rejected.",
|
|
29742
29771
|
inputSchema: DescribeServiceConnectionToolInputSchema,
|
|
29743
29772
|
outputSchema: recordOutputSchema("describe_service_connection_tool", DescribeServiceConnectionToolOutputSchema),
|
|
29744
29773
|
annotations: { title: "Describe Connected Service Tool", readOnlyHint: true, destructiveHint: false, idempotentHint: true, openWorldHint: false }
|
|
@@ -29759,7 +29788,7 @@ function registerPaaExtractorMcpTools(server, executor, options = {}) {
|
|
|
29759
29788
|
}, async (input) => executor.renewConnectedDataDownload(input));
|
|
29760
29789
|
server.registerTool("call_service_connection_action", {
|
|
29761
29790
|
title: "Run Connected Service Action",
|
|
29762
|
-
description: "Run one explicitly allowlisted write or mutation on a tenant-owned OAuth or remote MCP connection. First call list_service_connections, use a connection with actionsEnabled true,
|
|
29791
|
+
description: "Run one explicitly allowlisted write or mutation on a tenant-owned OAuth or remote MCP connection. First call list_service_connections, use a connection with actionsEnabled true, describe the exact actionTools entry to obtain its live schema, and supply only that action's arguments. The server rejects arbitrary action names, inactive or foreign connections, disabled actions, and every adminBlockedTools entry. This can include Google Drive folder creation or file copies, Resend delivery, and GitHub mutations only when those exact actions are live and approved. Sends, deletes, merges, workflow execution, and content changes are high impact.",
|
|
29763
29792
|
inputSchema: CallServiceConnectionActionInputSchema,
|
|
29764
29793
|
outputSchema: recordOutputSchema("call_service_connection_action", CallServiceConnectionActionOutputSchema),
|
|
29765
29794
|
annotations: { title: "Run Connected Service Action", readOnlyHint: false, destructiveHint: true, idempotentHint: false, openWorldHint: true }
|
|
@@ -29824,12 +29853,15 @@ async function readResponseData(res) {
|
|
|
29824
29853
|
}
|
|
29825
29854
|
function httpErrorPayload(path6, res, data) {
|
|
29826
29855
|
const objectData = data && typeof data === "object" && !Array.isArray(data) ? data : null;
|
|
29827
|
-
const
|
|
29856
|
+
const rawCode = objectData?.code ?? objectData?.errorCode ?? objectData?.error_code;
|
|
29857
|
+
const safeCode = typeof rawCode === "string" && /^[a-z0-9][a-z0-9_-]{0,99}$/.test(rawCode) ? rawCode : "mcp_http_error";
|
|
29858
|
+
const bodyMessage = objectData ? objectData.message ?? objectData.error ?? rawCode : data;
|
|
29828
29859
|
return {
|
|
29829
29860
|
...objectData ?? { body: data },
|
|
29830
|
-
error:
|
|
29861
|
+
error: safeCode,
|
|
29862
|
+
error_code: safeCode,
|
|
29831
29863
|
error_type: "http",
|
|
29832
|
-
retryable: res.status === 429 || res.status >= 500,
|
|
29864
|
+
retryable: typeof objectData?.retryable === "boolean" ? objectData.retryable : res.status === 429 || res.status >= 500,
|
|
29833
29865
|
status: res.status,
|
|
29834
29866
|
statusText: res.statusText,
|
|
29835
29867
|
path: path6,
|
|
@@ -39057,6 +39089,10 @@ async function collectConnectedDataExport(args) {
|
|
|
39057
39089
|
recordBytes += pageBytes;
|
|
39058
39090
|
cursor = page.nextCursor ?? void 0;
|
|
39059
39091
|
continuationCursor = page.nextCursor ?? null;
|
|
39092
|
+
if (page.complete === false && typeof beforePageCursor === "string" && page.nextCursor === beforePageCursor) {
|
|
39093
|
+
warnings.add("The provider could not advance this page after bounded retries. Its records and warnings were included once; retry this exact continuation later.");
|
|
39094
|
+
break;
|
|
39095
|
+
}
|
|
39060
39096
|
if (page.complete || page.nextCursor === null || page.nextCursor === void 0) {
|
|
39061
39097
|
complete = true;
|
|
39062
39098
|
continuationCursor = null;
|
|
@@ -39518,6 +39554,42 @@ function arrayFromPayload(value, keys) {
|
|
|
39518
39554
|
}
|
|
39519
39555
|
return [];
|
|
39520
39556
|
}
|
|
39557
|
+
function controlErrorStatus(status) {
|
|
39558
|
+
if (status === 400 || status === 403 || status === 404 || status === 409 || status === 429 || status === 502 || status === 503) {
|
|
39559
|
+
return status;
|
|
39560
|
+
}
|
|
39561
|
+
return status >= 500 ? 503 : 502;
|
|
39562
|
+
}
|
|
39563
|
+
function defaultControlErrorCode(status) {
|
|
39564
|
+
if (status === 400) return "invalid_request";
|
|
39565
|
+
if (status === 403) return "actions_disabled";
|
|
39566
|
+
if (status === 404) return "connection_not_found";
|
|
39567
|
+
if (status === 409) return "connection_inactive";
|
|
39568
|
+
if (status === 429) return "upstream_rate_limited";
|
|
39569
|
+
if (status === 503) return "connection_transport_unavailable";
|
|
39570
|
+
return "connection_control_failed";
|
|
39571
|
+
}
|
|
39572
|
+
function defaultControlErrorMessage(status) {
|
|
39573
|
+
if (status === 400) return "The connection request was invalid.";
|
|
39574
|
+
if (status === 403) return "Actions are disabled for this service connection.";
|
|
39575
|
+
if (status === 404) return "The service connection was not found.";
|
|
39576
|
+
if (status === 409) return "The service connection is not currently usable.";
|
|
39577
|
+
if (status === 429) return "The connected service is temporarily rate limited.";
|
|
39578
|
+
if (status === 503) return "The service connection transport is temporarily unavailable.";
|
|
39579
|
+
return "The service connection control request failed.";
|
|
39580
|
+
}
|
|
39581
|
+
function safeControlErrorPayload(body, responseStatus) {
|
|
39582
|
+
const status = controlErrorStatus(responseStatus);
|
|
39583
|
+
const unwrapped = unwrapData(body);
|
|
39584
|
+
const record = isRecord(unwrapped) ? unwrapped : isRecord(body) ? body : null;
|
|
39585
|
+
const candidateCode = record ? firstString(record, ["code", "errorCode", "error_code"], 100) : null;
|
|
39586
|
+
const normalizedCandidateCode = candidateCode ? CONTROL_ERROR_CODE_ALIASES.get(candidateCode) ?? candidateCode : null;
|
|
39587
|
+
const code = normalizedCandidateCode && SAFE_CONTROL_ERROR_CODES.has(normalizedCandidateCode) ? normalizedCandidateCode : defaultControlErrorCode(status);
|
|
39588
|
+
const candidateMessage = record ? firstString(record, ["error", "message"], 500) : null;
|
|
39589
|
+
const message = normalizedCandidateCode && SAFE_CONTROL_ERROR_CODES.has(normalizedCandidateCode) && candidateMessage ? candidateMessage.replace(/[\u0000-\u001f\u007f]/g, " ") : defaultControlErrorMessage(status);
|
|
39590
|
+
const retryable = record && typeof record.retryable === "boolean" ? record.retryable : status === 429 || status >= 500;
|
|
39591
|
+
return { status, code, message, retryable };
|
|
39592
|
+
}
|
|
39521
39593
|
async function controlRequest(path6, init, timeoutMs = 2e4) {
|
|
39522
39594
|
const response = await fetch(`${controlBaseUrl()}${path6}`, {
|
|
39523
39595
|
...init,
|
|
@@ -39529,7 +39601,13 @@ async function controlRequest(path6, init, timeoutMs = 2e4) {
|
|
|
39529
39601
|
},
|
|
39530
39602
|
signal: AbortSignal.timeout(timeoutMs)
|
|
39531
39603
|
}).catch((err) => {
|
|
39532
|
-
|
|
39604
|
+
const cause = err instanceof Error && err.name === "TimeoutError" ? "request timed out" : "network error";
|
|
39605
|
+
throw new NangoControlError(
|
|
39606
|
+
`Scheduled service connection control is unavailable: ${cause}.`,
|
|
39607
|
+
503,
|
|
39608
|
+
"connection_transport_unavailable",
|
|
39609
|
+
true
|
|
39610
|
+
);
|
|
39533
39611
|
});
|
|
39534
39612
|
const text = await response.text();
|
|
39535
39613
|
let body = {};
|
|
@@ -39539,8 +39617,8 @@ async function controlRequest(path6, init, timeoutMs = 2e4) {
|
|
|
39539
39617
|
body = {};
|
|
39540
39618
|
}
|
|
39541
39619
|
if (!response.ok) {
|
|
39542
|
-
const
|
|
39543
|
-
throw new NangoControlError(
|
|
39620
|
+
const error = safeControlErrorPayload(body, response.status);
|
|
39621
|
+
throw new NangoControlError(error.message, error.status, error.code, error.retryable);
|
|
39544
39622
|
}
|
|
39545
39623
|
return body;
|
|
39546
39624
|
}
|
|
@@ -39635,6 +39713,9 @@ async function getNangoConnections(identity) {
|
|
|
39635
39713
|
actionsEnabled: row.actionsEnabled === true || row.actions_enabled === true,
|
|
39636
39714
|
readTools: cleanTools(row.readTools ?? row.read_tools),
|
|
39637
39715
|
actionTools: cleanTools(row.actionTools ?? row.action_tools ?? row.writeTools ?? row.write_tools),
|
|
39716
|
+
mcpEndpoint: null,
|
|
39717
|
+
schemaDiscovery: "compatibility_describe",
|
|
39718
|
+
toolRevision: firstString(row, ["toolRevision", "tool_revision"], 200),
|
|
39638
39719
|
vaultName: firstString(row, ["vaultName", "vault_name"], 100),
|
|
39639
39720
|
tableName: firstString(row, ["tableName", "table_name"], 100),
|
|
39640
39721
|
createdAt: firstString(row, ["createdAt", "created_at"], 100),
|
|
@@ -39767,6 +39848,145 @@ async function callScheduleConnectionRead(identity, connectionId, tool, args) {
|
|
|
39767
39848
|
const data = unwrapData(body);
|
|
39768
39849
|
return isRecord(data) ? data.result ?? data : data;
|
|
39769
39850
|
}
|
|
39851
|
+
function sanitizeToolSchema(value) {
|
|
39852
|
+
if (!isRecord(value) || value.type !== "object") return null;
|
|
39853
|
+
try {
|
|
39854
|
+
const serialized = JSON.stringify(value);
|
|
39855
|
+
if (Buffer.byteLength(serialized, "utf8") > 256 * 1024) return null;
|
|
39856
|
+
const cloned = JSON.parse(serialized);
|
|
39857
|
+
return isRecord(cloned) ? cloned : null;
|
|
39858
|
+
} catch {
|
|
39859
|
+
return null;
|
|
39860
|
+
}
|
|
39861
|
+
}
|
|
39862
|
+
function sanitizeToolAnnotations(value) {
|
|
39863
|
+
if (!isRecord(value)) return void 0;
|
|
39864
|
+
const annotations = {};
|
|
39865
|
+
const title = cleanString(value.title, 200);
|
|
39866
|
+
if (title) annotations.title = title;
|
|
39867
|
+
for (const key of ["readOnlyHint", "destructiveHint", "idempotentHint", "openWorldHint"]) {
|
|
39868
|
+
if (typeof value[key] === "boolean") annotations[key] = value[key];
|
|
39869
|
+
}
|
|
39870
|
+
return Object.keys(annotations).length > 0 ? annotations : void 0;
|
|
39871
|
+
}
|
|
39872
|
+
function sanitizeToolIconSource(value) {
|
|
39873
|
+
const candidate = cleanString(value, 1e4);
|
|
39874
|
+
if (!candidate) return null;
|
|
39875
|
+
if (/^data:image\/(?:png|jpeg|webp|gif|svg\+xml);base64,[a-z0-9+/=]+$/i.test(candidate)) return candidate;
|
|
39876
|
+
return cleanHttpsUrl(candidate);
|
|
39877
|
+
}
|
|
39878
|
+
function sanitizeToolIcons(value) {
|
|
39879
|
+
if (!Array.isArray(value)) return void 0;
|
|
39880
|
+
const icons = [];
|
|
39881
|
+
for (const item of value.slice(0, 16)) {
|
|
39882
|
+
if (!isRecord(item)) continue;
|
|
39883
|
+
const src = sanitizeToolIconSource(item.src);
|
|
39884
|
+
if (!src) continue;
|
|
39885
|
+
const mimeType = cleanString(item.mimeType ?? item.mime_type, 100);
|
|
39886
|
+
const sizes = cleanStringArray(item.sizes, 16, 32);
|
|
39887
|
+
const theme = item.theme === "light" || item.theme === "dark" ? item.theme : void 0;
|
|
39888
|
+
icons.push({
|
|
39889
|
+
src,
|
|
39890
|
+
...mimeType ? { mimeType } : {},
|
|
39891
|
+
...sizes.length > 0 ? { sizes } : {},
|
|
39892
|
+
...theme ? { theme } : {}
|
|
39893
|
+
});
|
|
39894
|
+
}
|
|
39895
|
+
return icons.length > 0 ? icons : void 0;
|
|
39896
|
+
}
|
|
39897
|
+
function canonicalJson(value) {
|
|
39898
|
+
if (Array.isArray(value)) return `[${value.map(canonicalJson).join(",")}]`;
|
|
39899
|
+
if (isRecord(value)) {
|
|
39900
|
+
return `{${Object.keys(value).sort().map((key) => `${JSON.stringify(key)}:${canonicalJson(value[key])}`).join(",")}}`;
|
|
39901
|
+
}
|
|
39902
|
+
return JSON.stringify(value);
|
|
39903
|
+
}
|
|
39904
|
+
function projectedToolSchemaHash(tool) {
|
|
39905
|
+
return (0, import_node_crypto17.createHash)("sha256").update(canonicalJson(tool)).digest("hex");
|
|
39906
|
+
}
|
|
39907
|
+
async function describeNangoTool(identity, connectionId, tool, fresh) {
|
|
39908
|
+
const body = await controlRequest("/api/internal/nango/connections/actions/describe", {
|
|
39909
|
+
method: "POST",
|
|
39910
|
+
body: JSON.stringify({
|
|
39911
|
+
identity,
|
|
39912
|
+
connectionId,
|
|
39913
|
+
tool,
|
|
39914
|
+
...fresh === void 0 ? {} : { fresh }
|
|
39915
|
+
})
|
|
39916
|
+
});
|
|
39917
|
+
const data = unwrapData(body);
|
|
39918
|
+
const rawTool = isRecord(data) && isRecord(data.tool) ? data.tool : data;
|
|
39919
|
+
if (!isRecord(rawTool)) {
|
|
39920
|
+
throw new NangoControlError(
|
|
39921
|
+
"The connection service returned an invalid live tool description.",
|
|
39922
|
+
502,
|
|
39923
|
+
"tool_discovery_failed",
|
|
39924
|
+
true
|
|
39925
|
+
);
|
|
39926
|
+
}
|
|
39927
|
+
let serializedBytes = Number.POSITIVE_INFINITY;
|
|
39928
|
+
try {
|
|
39929
|
+
serializedBytes = Buffer.byteLength(JSON.stringify(rawTool), "utf8");
|
|
39930
|
+
} catch {
|
|
39931
|
+
}
|
|
39932
|
+
const name = cleanString(rawTool.name, 200);
|
|
39933
|
+
const classification = cleanString(rawTool.classification, 20);
|
|
39934
|
+
const transport = cleanString(rawTool.transport, 20);
|
|
39935
|
+
const providerConfigKey = cleanString(rawTool.providerConfigKey ?? rawTool.provider_config_key, 200);
|
|
39936
|
+
const schemaSource = cleanString(rawTool.schemaSource ?? rawTool.schema_source, 50);
|
|
39937
|
+
const upstreamSchemaHash = cleanString(rawTool.schemaHash ?? rawTool.schema_hash, 200);
|
|
39938
|
+
const fetchedAt = cleanString(rawTool.fetchedAt ?? rawTool.fetched_at, 100);
|
|
39939
|
+
const inputSchema = sanitizeToolSchema(rawTool.inputSchema ?? rawTool.input_schema);
|
|
39940
|
+
const outputSchemaValue = rawTool.outputSchema ?? rawTool.output_schema;
|
|
39941
|
+
const outputSchema = outputSchemaValue === void 0 ? void 0 : sanitizeToolSchema(outputSchemaValue);
|
|
39942
|
+
const blockedReasonValue = rawTool.blockedReason ?? rawTool.blocked_reason;
|
|
39943
|
+
const blockedReason = blockedReasonValue === null || blockedReasonValue === void 0 ? null : blockedReasonValue === "actions_disabled" || blockedReasonValue === "inactive_connection" ? blockedReasonValue : void 0;
|
|
39944
|
+
if (serializedBytes > 256 * 1024 || !name || name !== tool || classification !== "read" && classification !== "action" || typeof rawTool.callable !== "boolean" || blockedReason === void 0 || transport !== "nango" && transport !== "remote_mcp" || !providerConfigKey || schemaSource !== "live_tools_list" || !upstreamSchemaHash || !/^[a-f0-9]{64}$/i.test(upstreamSchemaHash) || !fetchedAt || Number.isNaN(Date.parse(fetchedAt)) || !inputSchema || outputSchemaValue !== void 0 && !outputSchema) {
|
|
39945
|
+
throw new NangoControlError(
|
|
39946
|
+
"The connection service returned an invalid live tool description.",
|
|
39947
|
+
502,
|
|
39948
|
+
"tool_discovery_failed",
|
|
39949
|
+
true
|
|
39950
|
+
);
|
|
39951
|
+
}
|
|
39952
|
+
const protocolVersion = rawTool.protocolVersion === null || rawTool.protocol_version === null ? null : cleanString(rawTool.protocolVersion ?? rawTool.protocol_version, 100);
|
|
39953
|
+
const executionValue = rawTool.execution;
|
|
39954
|
+
const taskSupport = isRecord(executionValue) && (executionValue.taskSupport === "forbidden" || executionValue.taskSupport === "optional" || executionValue.taskSupport === "required") ? executionValue.taskSupport : null;
|
|
39955
|
+
const title = cleanString(rawTool.title, 200);
|
|
39956
|
+
const description = cleanString(rawTool.description, 12e3);
|
|
39957
|
+
const annotations = sanitizeToolAnnotations(rawTool.annotations);
|
|
39958
|
+
const icons = sanitizeToolIcons(rawTool.icons);
|
|
39959
|
+
const execution = taskSupport ? { taskSupport } : void 0;
|
|
39960
|
+
const schemaHash = projectedToolSchemaHash({
|
|
39961
|
+
name,
|
|
39962
|
+
inputSchema,
|
|
39963
|
+
...title ? { title } : {},
|
|
39964
|
+
...description ? { description } : {},
|
|
39965
|
+
...outputSchema ? { outputSchema } : {},
|
|
39966
|
+
...annotations ? { annotations } : {},
|
|
39967
|
+
...icons ? { icons } : {},
|
|
39968
|
+
...execution ? { execution } : {}
|
|
39969
|
+
});
|
|
39970
|
+
return {
|
|
39971
|
+
name,
|
|
39972
|
+
title,
|
|
39973
|
+
description,
|
|
39974
|
+
classification,
|
|
39975
|
+
callable: rawTool.callable,
|
|
39976
|
+
blockedReason,
|
|
39977
|
+
transport,
|
|
39978
|
+
providerConfigKey,
|
|
39979
|
+
protocolVersion,
|
|
39980
|
+
schemaSource: "live_tools_list",
|
|
39981
|
+
inputSchema,
|
|
39982
|
+
...outputSchema ? { outputSchema } : {},
|
|
39983
|
+
...annotations ? { annotations } : {},
|
|
39984
|
+
...icons ? { icons } : {},
|
|
39985
|
+
...execution ? { execution } : {},
|
|
39986
|
+
schemaHash,
|
|
39987
|
+
fetchedAt
|
|
39988
|
+
};
|
|
39989
|
+
}
|
|
39770
39990
|
async function callScheduleConnectionExportPage(identity, input) {
|
|
39771
39991
|
const body = await controlRequest("/api/internal/nango/connections/export-page", {
|
|
39772
39992
|
method: "POST",
|
|
@@ -39804,10 +40024,11 @@ async function callScheduleConnectionExportPage(identity, input) {
|
|
|
39804
40024
|
untrustedContent: true
|
|
39805
40025
|
};
|
|
39806
40026
|
}
|
|
39807
|
-
var DEFAULT_NANGO_CONTROL_URL, DISABLED_NANGO_TOOLS, CONNECTION_SYNC_REQUIRED_TOOLS, NangoControlError, ScheduleConnectionValidationError;
|
|
40027
|
+
var import_node_crypto17, DEFAULT_NANGO_CONTROL_URL, DISABLED_NANGO_TOOLS, CONNECTION_SYNC_REQUIRED_TOOLS, NangoControlError, ScheduleConnectionValidationError, SAFE_CONTROL_ERROR_CODES, CONTROL_ERROR_CODE_ALIASES;
|
|
39808
40028
|
var init_nango_control = __esm({
|
|
39809
40029
|
"src/api/nango-control.ts"() {
|
|
39810
40030
|
"use strict";
|
|
40031
|
+
import_node_crypto17 = require("crypto");
|
|
39811
40032
|
init_connected_data_export();
|
|
39812
40033
|
DEFAULT_NANGO_CONTROL_URL = "https://mcp-scraper-scheduler.vercel.app";
|
|
39813
40034
|
DISABLED_NANGO_TOOLS = {
|
|
@@ -39850,10 +40071,14 @@ var init_nango_control = __esm({
|
|
|
39850
40071
|
};
|
|
39851
40072
|
NangoControlError = class extends Error {
|
|
39852
40073
|
status;
|
|
39853
|
-
|
|
40074
|
+
code;
|
|
40075
|
+
retryable;
|
|
40076
|
+
constructor(message, status = 502, code = status === 503 ? "connection_transport_unavailable" : "connection_control_failed", retryable = status === 429 || status >= 500) {
|
|
39854
40077
|
super(message);
|
|
39855
40078
|
this.name = "NangoControlError";
|
|
39856
40079
|
this.status = status;
|
|
40080
|
+
this.code = code;
|
|
40081
|
+
this.retryable = retryable;
|
|
39857
40082
|
}
|
|
39858
40083
|
};
|
|
39859
40084
|
ScheduleConnectionValidationError = class extends Error {
|
|
@@ -39862,6 +40087,27 @@ var init_nango_control = __esm({
|
|
|
39862
40087
|
this.name = "ScheduleConnectionValidationError";
|
|
39863
40088
|
}
|
|
39864
40089
|
};
|
|
40090
|
+
SAFE_CONTROL_ERROR_CODES = /* @__PURE__ */ new Set([
|
|
40091
|
+
"invalid_request",
|
|
40092
|
+
"tool_not_allowed",
|
|
40093
|
+
"invalid_schema",
|
|
40094
|
+
"connection_not_found",
|
|
40095
|
+
"connection_inactive",
|
|
40096
|
+
"actions_disabled",
|
|
40097
|
+
"action_not_allowed",
|
|
40098
|
+
"upstream_rate_limited",
|
|
40099
|
+
"tool_discovery_failed",
|
|
40100
|
+
"live_tool_missing",
|
|
40101
|
+
"connection_transport_unavailable",
|
|
40102
|
+
"connection_control_failed"
|
|
40103
|
+
]);
|
|
40104
|
+
CONTROL_ERROR_CODE_ALIASES = /* @__PURE__ */ new Map([
|
|
40105
|
+
["connection_not_active", "connection_inactive"],
|
|
40106
|
+
["actions_not_enabled", "actions_disabled"],
|
|
40107
|
+
["action_tool_not_allowed", "action_not_allowed"],
|
|
40108
|
+
["no_action_for_provider", "action_not_allowed"],
|
|
40109
|
+
["tool_not_available", "live_tool_missing"]
|
|
40110
|
+
]);
|
|
39865
40111
|
}
|
|
39866
40112
|
});
|
|
39867
40113
|
|
|
@@ -40011,6 +40257,9 @@ async function getResendConnections(identity) {
|
|
|
40011
40257
|
readTools: cleanTools2(row.readTools ?? row.read_tools),
|
|
40012
40258
|
actionTools: cleanTools2(row.actionTools ?? row.action_tools ?? row.writeTools ?? row.write_tools),
|
|
40013
40259
|
adminBlockedTools: cleanTools2(row.adminBlockedTools ?? row.admin_blocked_tools).length > 0 ? cleanTools2(row.adminBlockedTools ?? row.admin_blocked_tools) : [...RESEND_ADMIN_BLOCKED_TOOLS],
|
|
40260
|
+
mcpEndpoint: null,
|
|
40261
|
+
schemaDiscovery: "compatibility_describe",
|
|
40262
|
+
toolRevision: firstString2(row, ["toolRevision", "tool_revision"], 200),
|
|
40014
40263
|
vaultName: firstString2(row, ["vaultName", "vault_name"], 100),
|
|
40015
40264
|
tableName: firstString2(row, ["tableName", "table_name"], 100),
|
|
40016
40265
|
createdAt: firstString2(row, ["createdAt", "created_at"], 100),
|
|
@@ -40408,7 +40657,15 @@ function maybeProvisionInBackground(user, vaultCount) {
|
|
|
40408
40657
|
}
|
|
40409
40658
|
function scheduleConnectionError(c, err, fallback) {
|
|
40410
40659
|
if (err instanceof ScheduleConnectionValidationError) return c.json({ ok: false, error: err.message }, 400);
|
|
40411
|
-
if (err instanceof NangoControlError)
|
|
40660
|
+
if (err instanceof NangoControlError) {
|
|
40661
|
+
return c.json({
|
|
40662
|
+
ok: false,
|
|
40663
|
+
error: err.message,
|
|
40664
|
+
code: err.code,
|
|
40665
|
+
errorCode: err.code,
|
|
40666
|
+
retryable: err.retryable
|
|
40667
|
+
}, err.status);
|
|
40668
|
+
}
|
|
40412
40669
|
if (err instanceof ResendControlError) return c.json({ ok: false, error: err.message }, err.status);
|
|
40413
40670
|
console.error("[schedule-connections]", err instanceof Error ? err.message : String(err));
|
|
40414
40671
|
return c.json({ ok: false, error: fallback }, 502);
|
|
@@ -41292,11 +41549,12 @@ var init_server = __esm({
|
|
|
41292
41549
|
const providerConfigKey = providerConfigKeyFrom(body.providerConfigKey);
|
|
41293
41550
|
const tool = providerConfigKeyFrom(body.tool);
|
|
41294
41551
|
if (!connectionId || !tool) return c.json({ ok: false, error: "connectionId and tool are required." }, 400);
|
|
41552
|
+
if (body.fresh !== void 0 && typeof body.fresh !== "boolean") {
|
|
41553
|
+
return c.json({ ok: false, error: "fresh must be a boolean when provided." }, 400);
|
|
41554
|
+
}
|
|
41295
41555
|
try {
|
|
41296
|
-
|
|
41297
|
-
|
|
41298
|
-
}
|
|
41299
|
-
return c.json({ ok: true, tool: await describeResendTool(user.email, connectionId, tool) });
|
|
41556
|
+
const description = await isResendConnection(user.email, connectionId, providerConfigKey) ? await describeResendTool(user.email, connectionId, tool) : await describeNangoTool(user.email, connectionId, tool, body.fresh);
|
|
41557
|
+
return c.json({ ok: true, tool: description });
|
|
41300
41558
|
} catch (err) {
|
|
41301
41559
|
return scheduleConnectionError(c, err, "Unable to describe this connection tool.");
|
|
41302
41560
|
}
|