mcp-scraper 0.83.5 → 0.85.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +41 -1
- package/README.md +2 -2
- package/dist/{analytics-repository-GA6P5S5J.js → analytics-repository-4VZQ4TKW.js} +9 -1
- package/dist/bin/api-server.js +3 -3
- package/dist/bin/mcp-scraper-cli.js +3 -3
- package/dist/bin/mcp-scraper-core.js +6 -6
- package/dist/bin/mcp-scraper-install.js +2 -2
- package/dist/bin/mcp-stdio-server.js +6 -6
- package/dist/{chunk-VJHMPJR7.js → chunk-2ZDYUTOZ.js} +3 -3
- package/dist/{chunk-ACDNW7TG.js → chunk-6IOGTFNS.js} +1 -1
- package/dist/{chunk-HGJJQY24.js → chunk-C7OKENCR.js} +1 -1
- package/dist/{chunk-BGD7U3AE.js → chunk-P7FWOMU7.js} +10 -4
- package/dist/{chunk-2B636U7P.js → chunk-RHAOKXQK.js} +1 -1
- package/dist/{chunk-CNAJWRJT.js → chunk-TLRJV4SL.js} +583 -29
- package/dist/{chunk-ZHPGPVJJ.js → chunk-W3SGF7GX.js} +33 -7
- package/dist/{chunk-2BOBOKFQ.js → chunk-W4IY6CB4.js} +2858 -1328
- package/dist/{chunk-UXVT7FWW.js → chunk-WN7N4ESQ.js} +2 -2
- package/dist/{chunk-4NZ6JFK5.js → chunk-X5HRAAQM.js} +2 -2
- package/dist/{extract-bundle-HTWWJD6N.js → extract-bundle-NSMGHEG6.js} +2 -2
- package/dist/{gmail-service-VXE7OYNA.js → gmail-service-BTRAHGJS.js} +2 -2
- package/dist/{server-2E4YF6TY.js → server-EOPETL7Q.js} +2291 -2267
- package/dist/{site-extract-repository-Y5WQFSQW.js → site-extract-repository-DWU52BRQ.js} +2 -2
- package/dist/{worker-FM3ZRDY4.js → worker-5A375BGN.js} +1 -1
- package/package.json +1 -1
- package/public/.well-known/xray-install.json +9 -4
- package/public/agent-install.md +8 -2
|
@@ -19,7 +19,7 @@ import {
|
|
|
19
19
|
LedgerOperation,
|
|
20
20
|
MC_PER_CREDIT,
|
|
21
21
|
calculateConnectedUsageCharge
|
|
22
|
-
} from "./chunk-
|
|
22
|
+
} from "./chunk-X5HRAAQM.js";
|
|
23
23
|
import {
|
|
24
24
|
recordVendorUsage,
|
|
25
25
|
runWithCostContext
|
|
@@ -528,6 +528,11 @@ var NANGO_TOOL_CALL_TIMEOUT_MS = 24e4;
|
|
|
528
528
|
var NANGO_CONNECTION_PAGE_SIZE = 100;
|
|
529
529
|
var NANGO_CONNECTION_MAX_PAGES = 20;
|
|
530
530
|
var PROVIDER_IDENTITY_RETRY_MS = 6 * 60 * 60 * 1e3;
|
|
531
|
+
var WITHDRAWN_NANGO_PROVIDER_CONFIG_KEYS = /* @__PURE__ */ new Set([
|
|
532
|
+
"stripe_revenue",
|
|
533
|
+
"salesforce",
|
|
534
|
+
"pipedrive"
|
|
535
|
+
]);
|
|
531
536
|
var MAIN_INTEGRATION_CONTRACT_VERSION = "2026-07-16.1";
|
|
532
537
|
var MAIN_INTEGRATION_CONTRACT_HASH = "f037859ec80c2ea2fac6e4a67c1a59f3200dd22d3642c97789b9bb3236723d3e";
|
|
533
538
|
var MainNangoTransportError = class extends Error {
|
|
@@ -544,6 +549,18 @@ var MainNangoTransportError = class extends Error {
|
|
|
544
549
|
this.retryAfterMs = retryAfterMs;
|
|
545
550
|
}
|
|
546
551
|
};
|
|
552
|
+
function isWithdrawnNangoProvider(providerConfigKey) {
|
|
553
|
+
return typeof providerConfigKey === "string" && WITHDRAWN_NANGO_PROVIDER_CONFIG_KEYS.has(providerConfigKey.normalize("NFKC").trim().toLowerCase());
|
|
554
|
+
}
|
|
555
|
+
function assertNangoProviderAvailable(providerConfigKey) {
|
|
556
|
+
if (!isWithdrawnNangoProvider(providerConfigKey)) return;
|
|
557
|
+
throw new MainNangoTransportError(
|
|
558
|
+
"This provider integration is not currently available.",
|
|
559
|
+
409,
|
|
560
|
+
"provider_unavailable",
|
|
561
|
+
false
|
|
562
|
+
);
|
|
563
|
+
}
|
|
547
564
|
function nangoClientErrorText(error) {
|
|
548
565
|
if (error instanceof Error) return error.message;
|
|
549
566
|
if (typeof error === "string") return error;
|
|
@@ -907,12 +924,13 @@ async function discoverOwnedNangoConnections(identity, options = {}) {
|
|
|
907
924
|
}
|
|
908
925
|
const discovered = rows.filter((row) => connectionIdentityMatches(row, identity)).map(discoveredConnection).filter((connection) => connection !== null);
|
|
909
926
|
const reconciled = await reconcileDiscoveredNangoConnections(identity, discovered);
|
|
910
|
-
|
|
927
|
+
const visibleConnections = reconciled.filter((connection) => !isWithdrawnNangoProvider(connection.providerConfigKey));
|
|
928
|
+
if (options.resolveProviderIdentity === false) return visibleConnections;
|
|
911
929
|
const remoteByKey = new Map(discovered.map((connection) => [
|
|
912
930
|
`${connection.providerConfigKey}\0${connection.upstreamConnectionId}`,
|
|
913
931
|
connection
|
|
914
932
|
]));
|
|
915
|
-
await Promise.all(
|
|
933
|
+
await Promise.all(visibleConnections.map(async (connection) => {
|
|
916
934
|
if (connection.lifecycleStatus !== "connected" || !connection.upstreamConnectionId) return;
|
|
917
935
|
const remote = remoteByKey.get(`${connection.providerConfigKey}\0${connection.upstreamConnectionId}`);
|
|
918
936
|
if (!remote) return;
|
|
@@ -935,7 +953,7 @@ async function discoverOwnedNangoConnections(identity, options = {}) {
|
|
|
935
953
|
sourceUpdatedAt
|
|
936
954
|
});
|
|
937
955
|
}));
|
|
938
|
-
return reconcileDiscoveredNangoConnections(identity, discovered);
|
|
956
|
+
return (await reconcileDiscoveredNangoConnections(identity, discovered)).filter((connection) => !isWithdrawnNangoProvider(connection.providerConfigKey));
|
|
939
957
|
}
|
|
940
958
|
async function listNangoCatalogDirect() {
|
|
941
959
|
const [integrationsPayload, providersPayload] = await Promise.all([
|
|
@@ -952,7 +970,7 @@ async function listNangoCatalogDirect() {
|
|
|
952
970
|
const row = asRecord(value);
|
|
953
971
|
const id = text(row?.unique_key) ?? text(row?.id);
|
|
954
972
|
const provider = text(row?.provider);
|
|
955
|
-
if (!id || !provider) return [];
|
|
973
|
+
if (!id || !provider || isWithdrawnNangoProvider(id)) return [];
|
|
956
974
|
const definition = providers.get(provider);
|
|
957
975
|
return [{
|
|
958
976
|
id,
|
|
@@ -986,6 +1004,7 @@ function sessionPayload(value) {
|
|
|
986
1004
|
return { connectLink, expiresAt: text(data?.expires_at) };
|
|
987
1005
|
}
|
|
988
1006
|
async function createNangoSessionDirect(identity, providerConfigKey) {
|
|
1007
|
+
assertNangoProviderAvailable(providerConfigKey);
|
|
989
1008
|
const defaults = connectDefaults(providerConfigKey);
|
|
990
1009
|
return sessionPayload(await nangoRequest("/connect/sessions", {
|
|
991
1010
|
method: "POST",
|
|
@@ -1010,7 +1029,7 @@ async function createNangoReconnectSessionDirect(identity, connectionId) {
|
|
|
1010
1029
|
}));
|
|
1011
1030
|
}
|
|
1012
1031
|
async function deleteNangoConnectionDirect(identity, connectionId) {
|
|
1013
|
-
const connection = await
|
|
1032
|
+
const connection = await requireOwnedNangoConnectionForCleanup(identity, connectionId);
|
|
1014
1033
|
try {
|
|
1015
1034
|
await nangoRequest(`/connections/${encodeURIComponent(connection.upstreamConnectionId)}`, {
|
|
1016
1035
|
method: "DELETE",
|
|
@@ -1020,11 +1039,16 @@ async function deleteNangoConnectionDirect(identity, connectionId) {
|
|
|
1020
1039
|
if (!(error instanceof MainNangoTransportError) || error.status !== 404) throw error;
|
|
1021
1040
|
}
|
|
1022
1041
|
}
|
|
1023
|
-
async function
|
|
1042
|
+
async function requireOwnedNangoConnectionForCleanup(identity, connectionId) {
|
|
1024
1043
|
const connection = await getOwnedServiceConnection(identity, connectionId);
|
|
1025
1044
|
if (!connection || connection.transport !== "nango" || !connection.upstreamConnectionId) {
|
|
1026
1045
|
throw new MainNangoTransportError("Service connection not found.", 404, "connection_not_found", false);
|
|
1027
1046
|
}
|
|
1047
|
+
return connection;
|
|
1048
|
+
}
|
|
1049
|
+
async function requireOwnedActiveConnection(identity, connectionId, allowNeedsReauth = false) {
|
|
1050
|
+
const connection = await requireOwnedNangoConnectionForCleanup(identity, connectionId);
|
|
1051
|
+
assertNangoProviderAvailable(connection.providerConfigKey);
|
|
1028
1052
|
if (connection.lifecycleStatus !== "connected" && !(allowNeedsReauth && connection.lifecycleStatus === "needs_reauth")) {
|
|
1029
1053
|
throw new MainNangoTransportError("Service connection requires reconnection.", 409, "connection_inactive", false);
|
|
1030
1054
|
}
|
|
@@ -2028,6 +2052,8 @@ export {
|
|
|
2028
2052
|
settleConnectedUsage,
|
|
2029
2053
|
listConnectedUsageHistory,
|
|
2030
2054
|
MainNangoTransportError,
|
|
2055
|
+
isWithdrawnNangoProvider,
|
|
2056
|
+
assertNangoProviderAvailable,
|
|
2031
2057
|
discoverOwnedNangoConnections,
|
|
2032
2058
|
listNangoCatalogDirect,
|
|
2033
2059
|
createNangoSessionDirect,
|