mcp-scraper 0.17.2 → 0.19.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -2
- package/dist/bin/api-server.cjs +580 -89
- package/dist/bin/api-server.cjs.map +1 -1
- package/dist/bin/api-server.js +2 -2
- package/dist/bin/mcp-scraper-cli.cjs +1 -1
- package/dist/bin/mcp-scraper-cli.cjs.map +1 -1
- package/dist/bin/mcp-scraper-cli.js +1 -1
- package/dist/bin/mcp-scraper-install.cjs +2 -2
- package/dist/bin/mcp-scraper-install.cjs.map +1 -1
- package/dist/bin/mcp-scraper-install.js +2 -2
- package/dist/bin/mcp-stdio-server.cjs +2141 -1765
- package/dist/bin/mcp-stdio-server.cjs.map +1 -1
- package/dist/bin/mcp-stdio-server.js +5 -4
- package/dist/bin/mcp-stdio-server.js.map +1 -1
- package/dist/bin/paa-harvest.js +2 -2
- package/dist/{chunk-RMPPYKUV.js → chunk-2HDMYW4B.js} +7 -218
- package/dist/chunk-2HDMYW4B.js.map +1 -0
- package/dist/chunk-D7ZT27HY.js +244 -0
- package/dist/chunk-D7ZT27HY.js.map +1 -0
- package/dist/{chunk-HJ3XIHWC.js → chunk-O2S5TOCG.js} +407 -29
- package/dist/chunk-O2S5TOCG.js.map +1 -0
- package/dist/chunk-OQHYDW4Q.js +7 -0
- package/dist/chunk-OQHYDW4Q.js.map +1 -0
- package/dist/{chunk-BX5RCOG5.js → chunk-Q44DN6T2.js} +2 -2
- package/dist/{chunk-BX5RCOG5.js.map → chunk-Q44DN6T2.js.map} +1 -1
- package/dist/index.js +4 -3
- package/dist/index.js.map +1 -1
- package/dist/{server-U5VODSSW.js → server-VWXDE64Y.js} +176 -66
- package/dist/server-VWXDE64Y.js.map +1 -0
- package/dist/{worker-JQTS437L.js → worker-AM2DHUWG.js} +6 -6
- package/docs/mcp-tool-manifest.generated.json +548 -15
- package/docs/specs/meta-ad-creative-media-resolution-spec.md +31 -0
- package/package.json +1 -1
- package/dist/chunk-HJ3XIHWC.js.map +0 -1
- package/dist/chunk-HPV4VOQX.js +0 -27
- package/dist/chunk-HPV4VOQX.js.map +0 -1
- package/dist/chunk-IDRSO4HX.js +0 -7
- package/dist/chunk-IDRSO4HX.js.map +0 -1
- package/dist/chunk-RMPPYKUV.js.map +0 -1
- package/dist/server-U5VODSSW.js.map +0 -1
- /package/dist/{worker-JQTS437L.js.map → worker-AM2DHUWG.js.map} +0 -0
|
@@ -8,20 +8,14 @@ import {
|
|
|
8
8
|
BrowserDriver,
|
|
9
9
|
MapsSelectors,
|
|
10
10
|
buildYouTubeChannelVideosUrl,
|
|
11
|
-
currentCostContext,
|
|
12
11
|
deleteKernelProxyId,
|
|
13
12
|
harvest,
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
recordKernelSession,
|
|
17
|
-
recordVendorUsage,
|
|
18
|
-
resolveKernelProxyId,
|
|
19
|
-
runWithCostContext,
|
|
20
|
-
vendorCostUsd
|
|
21
|
-
} from "./chunk-RMPPYKUV.js";
|
|
13
|
+
resolveKernelProxyId
|
|
14
|
+
} from "./chunk-2HDMYW4B.js";
|
|
22
15
|
import {
|
|
23
16
|
HttpMcpToolExecutor,
|
|
24
17
|
MemoryMcpToolExecutor,
|
|
18
|
+
assessTranscriptSignal,
|
|
25
19
|
buildLinkGraph,
|
|
26
20
|
buildPaaExtractorMcpServer,
|
|
27
21
|
cleanupExpiredConnectedDataArtifacts,
|
|
@@ -36,8 +30,9 @@ import {
|
|
|
36
30
|
registerSerpIntelligenceCaptureTools,
|
|
37
31
|
renewConnectedDataArtifactDownload,
|
|
38
32
|
sanitizeAttempts,
|
|
39
|
-
sanitizeHarvestResult
|
|
40
|
-
|
|
33
|
+
sanitizeHarvestResult,
|
|
34
|
+
transcribeMediaUrl
|
|
35
|
+
} from "./chunk-O2S5TOCG.js";
|
|
41
36
|
import {
|
|
42
37
|
auditImages,
|
|
43
38
|
buildLinkReport,
|
|
@@ -48,8 +43,14 @@ import {
|
|
|
48
43
|
browserServiceApiKey,
|
|
49
44
|
browserServiceProfileName,
|
|
50
45
|
browserServiceProfileSaveChanges,
|
|
51
|
-
browserServiceProxyId
|
|
52
|
-
|
|
46
|
+
browserServiceProxyId,
|
|
47
|
+
currentCostContext,
|
|
48
|
+
kernelCostUsd,
|
|
49
|
+
migrateCostTelemetry,
|
|
50
|
+
recordKernelSession,
|
|
51
|
+
recordVendorUsage,
|
|
52
|
+
runWithCostContext
|
|
53
|
+
} from "./chunk-D7ZT27HY.js";
|
|
53
54
|
import {
|
|
54
55
|
csvRecords,
|
|
55
56
|
listWorkflowDefinitions,
|
|
@@ -73,7 +74,7 @@ import {
|
|
|
73
74
|
RawMapsOverviewSchema,
|
|
74
75
|
RawMapsReviewStatsSchema
|
|
75
76
|
} from "./chunk-XGIPATLV.js";
|
|
76
|
-
import "./chunk-
|
|
77
|
+
import "./chunk-OQHYDW4Q.js";
|
|
77
78
|
import {
|
|
78
79
|
completeExtractJob,
|
|
79
80
|
countSuccessfulPages,
|
|
@@ -8797,41 +8798,6 @@ async function resolveYouTubeAudio(videoId) {
|
|
|
8797
8798
|
}
|
|
8798
8799
|
}
|
|
8799
8800
|
|
|
8800
|
-
// src/services/media-transcription.ts
|
|
8801
|
-
import { fal } from "@fal-ai/client";
|
|
8802
|
-
function transcriptMarkdown(title, text, chunks, durationMs) {
|
|
8803
|
-
const fmtTs2 = (s) => `${Math.floor(s / 60)}:${String(Math.floor(s % 60)).padStart(2, "0")}`;
|
|
8804
|
-
const lines = [title, "", `*Transcribed in ${(durationMs / 1e3).toFixed(1)}s*`, "", "## Full Text", "", text, ""];
|
|
8805
|
-
if (chunks.length) {
|
|
8806
|
-
lines.push("## Timestamped Segments", "");
|
|
8807
|
-
for (const ch of chunks) {
|
|
8808
|
-
lines.push(`**[${fmtTs2(ch.timestamp[0])} -> ${fmtTs2(ch.timestamp[1])}]** ${ch.text.trim()}`, "");
|
|
8809
|
-
}
|
|
8810
|
-
}
|
|
8811
|
-
return lines.join("\n");
|
|
8812
|
-
}
|
|
8813
|
-
async function transcribeMediaUrl(mediaUrl, markdownTitle = "# Media Transcript") {
|
|
8814
|
-
const startMs = Date.now();
|
|
8815
|
-
const result = await fal.subscribe("fal-ai/wizper", {
|
|
8816
|
-
input: { audio_url: mediaUrl, task: "transcribe", language: "en" },
|
|
8817
|
-
logs: false,
|
|
8818
|
-
pollInterval: 3e3
|
|
8819
|
-
});
|
|
8820
|
-
const data = result.data;
|
|
8821
|
-
const text = data.text ?? "";
|
|
8822
|
-
const chunks = data.chunks ?? [];
|
|
8823
|
-
const durationMs = Date.now() - startMs;
|
|
8824
|
-
void recordVendorUsage({ vendor: "fal_wizper", model: "fal-ai/wizper", units: durationMs / 1e3, unitType: "compute_sec" });
|
|
8825
|
-
const costUsd = vendorCostUsd("fal_wizper", durationMs / 1e3);
|
|
8826
|
-
return {
|
|
8827
|
-
text,
|
|
8828
|
-
chunks,
|
|
8829
|
-
durationMs,
|
|
8830
|
-
costUsd,
|
|
8831
|
-
markdown: transcriptMarkdown(markdownTitle, text, chunks, durationMs)
|
|
8832
|
-
};
|
|
8833
|
-
}
|
|
8834
|
-
|
|
8835
8801
|
// src/youtube/CaptionFetcher.ts
|
|
8836
8802
|
async function fetchViaYoutubeTranscript(videoId) {
|
|
8837
8803
|
try {
|
|
@@ -10047,11 +10013,13 @@ var FacebookAdExtractor = class {
|
|
|
10047
10013
|
if (n) nameFreq.set(n, (nameFreq.get(n) ?? 0) + 1);
|
|
10048
10014
|
}
|
|
10049
10015
|
const advertiserName = nameFreq.size ? [...nameFreq.entries()].sort((a, b) => b[1] - a[1])[0][0] : null;
|
|
10016
|
+
const matchedAdvertisers = [...nameFreq.entries()].map(([name, adCount]) => ({ name, adCount })).sort((a, b) => b.adCount - a.adCount).slice(0, 20);
|
|
10050
10017
|
return {
|
|
10051
10018
|
listingUrl,
|
|
10052
10019
|
scannedAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
10053
10020
|
advertiserPageId,
|
|
10054
10021
|
advertiserName,
|
|
10022
|
+
matchedAdvertisers,
|
|
10055
10023
|
summary: {
|
|
10056
10024
|
totalAds,
|
|
10057
10025
|
activeCount,
|
|
@@ -10435,7 +10403,7 @@ async function extractFacebookOrganicVideoFromPage(page, sourceUrl, quality = "b
|
|
|
10435
10403
|
}
|
|
10436
10404
|
|
|
10437
10405
|
// src/api/facebook-ad-routes.ts
|
|
10438
|
-
import { fal
|
|
10406
|
+
import { fal } from "@fal-ai/client";
|
|
10439
10407
|
var FacebookAdBodySchema = z13.object({
|
|
10440
10408
|
url: z13.string().trim().optional(),
|
|
10441
10409
|
libraryId: z13.string().trim().optional(),
|
|
@@ -10486,6 +10454,45 @@ function buildPageIntelUrl(body, country) {
|
|
|
10486
10454
|
if (body.pageId?.trim()) return `https://www.facebook.com/ads/library/?active_status=all&ad_type=all&country=${country}&is_targeted_country=false&media_type=all&search_type=page&view_all_page_id=${body.pageId.trim()}`;
|
|
10487
10455
|
return `https://www.facebook.com/ads/library/?active_status=all&ad_type=all&country=${country}&q=${encodeURIComponent(body.query.trim())}&search_type=keyword_unordered`;
|
|
10488
10456
|
}
|
|
10457
|
+
function normalizeAdvertiserName(value) {
|
|
10458
|
+
return value.toLowerCase().normalize("NFKD").replace(/[^a-z0-9]+/g, " ").trim().replace(/\s+/g, " ");
|
|
10459
|
+
}
|
|
10460
|
+
function advertiserTokenOverlap(query, candidate) {
|
|
10461
|
+
const queryTokens = new Set(normalizeAdvertiserName(query).split(" ").filter(Boolean));
|
|
10462
|
+
const candidateTokens = new Set(normalizeAdvertiserName(candidate).split(" ").filter(Boolean));
|
|
10463
|
+
if (queryTokens.size === 0 || candidateTokens.size === 0) return 0;
|
|
10464
|
+
let overlap = 0;
|
|
10465
|
+
for (const token of queryTokens) if (candidateTokens.has(token)) overlap++;
|
|
10466
|
+
return overlap / queryTokens.size;
|
|
10467
|
+
}
|
|
10468
|
+
function assessPageIntelMatch(body, result) {
|
|
10469
|
+
if (body.pageId) return { inputMode: "pageId", matchConfidence: "high", matchReason: "Exact Facebook page ID filter.", warnings: [] };
|
|
10470
|
+
if (body.libraryId) return { inputMode: "libraryId", matchConfidence: "high", matchReason: "Exact Ad Library archive ID filter.", warnings: [] };
|
|
10471
|
+
const query = body.query?.trim() ?? "";
|
|
10472
|
+
const normalizedQuery = normalizeAdvertiserName(query);
|
|
10473
|
+
const matches = result.matchedAdvertisers ?? [];
|
|
10474
|
+
const exact = matches.find((match) => normalizeAdvertiserName(match.name) === normalizedQuery);
|
|
10475
|
+
const top = matches[0];
|
|
10476
|
+
const topShare = top && result.summary.totalAds > 0 ? top.adCount / result.summary.totalAds : 0;
|
|
10477
|
+
const overlap = top ? advertiserTokenOverlap(query, top.name) : 0;
|
|
10478
|
+
const warnings = [];
|
|
10479
|
+
let matchConfidence;
|
|
10480
|
+
let matchReason;
|
|
10481
|
+
if (exact && exact.adCount / Math.max(1, result.summary.totalAds) >= 0.7) {
|
|
10482
|
+
matchConfidence = "high";
|
|
10483
|
+
matchReason = "The exact advertiser name accounts for most returned ads.";
|
|
10484
|
+
} else if (top && overlap >= 0.75 && topShare >= 0.5) {
|
|
10485
|
+
matchConfidence = "medium";
|
|
10486
|
+
matchReason = "The leading advertiser overlaps strongly with the query, but keyword search is not page-scoped.";
|
|
10487
|
+
} else {
|
|
10488
|
+
matchConfidence = "low";
|
|
10489
|
+
matchReason = "Keyword search did not identify one dominant exact advertiser match.";
|
|
10490
|
+
}
|
|
10491
|
+
if (matches.length > 1) warnings.push(`Keyword search mixed ${matches.length} advertiser names. Confirm the intended pageId before using these ads for performance or creative analysis.`);
|
|
10492
|
+
if (matchConfidence === "low") warnings.push("Low-specificity Ad Library match: treat these results as discovery candidates, not as verified ads from the requested brand.");
|
|
10493
|
+
warnings.push("Paused, cancelled, or otherwise inactive non-political ads may be absent from Ad Library. Use meta_ad_creative_media for connected ad-account creatives.");
|
|
10494
|
+
return { inputMode: "query", matchConfidence, matchReason, warnings };
|
|
10495
|
+
}
|
|
10489
10496
|
function configuredKernelProxy() {
|
|
10490
10497
|
return (process.env.BROWSER_SERVICE_PROXY_ID ?? process.env.KERNEL_PROXY_ID)?.trim() || void 0;
|
|
10491
10498
|
}
|
|
@@ -10582,8 +10589,10 @@ facebookAdApp.post("/page-intel", createApiKeyAuth(), async (c) => {
|
|
|
10582
10589
|
await logRequestEvent({ userId: fbUser.id, source: "facebook_page_intel", status: "failed", query: body.pageId ?? body.query ?? body.libraryId ?? "", error: "soft-block: empty result refunded" });
|
|
10583
10590
|
return c.json({ error: "soft-block: no ads returned (refunded)" }, 503);
|
|
10584
10591
|
}
|
|
10585
|
-
|
|
10586
|
-
|
|
10592
|
+
const match = assessPageIntelMatch(body, result);
|
|
10593
|
+
const output = { ...result, ...match };
|
|
10594
|
+
await logRequestEvent({ userId: fbUser.id, source: "facebook_page_intel", status: "done", query: body.pageId ?? body.query ?? body.libraryId ?? "", resultCount: result.ads.length, result: output });
|
|
10595
|
+
return c.json(output);
|
|
10587
10596
|
} catch (err) {
|
|
10588
10597
|
const msg = err instanceof Error ? err.message : String(err);
|
|
10589
10598
|
if (debited && !refunded) await creditMc(fbUser.id, MC_COSTS.fb_ad, LedgerOperation.FB_AD_REFUND, "failed call");
|
|
@@ -10615,7 +10624,7 @@ facebookAdApp.post("/transcribe", createApiKeyAuth(), async (c) => {
|
|
|
10615
10624
|
metadata: { videoUrl }
|
|
10616
10625
|
});
|
|
10617
10626
|
if (!gate.ok) return c.json(concurrencyLimitExceededResponse(gate), 429, { "Retry-After": String(gate.retryAfterSeconds) });
|
|
10618
|
-
|
|
10627
|
+
fal.config({ credentials: process.env.FAL_KEY });
|
|
10619
10628
|
const holdMc = MEDIA_TRANSCRIBE_HOLD_MC;
|
|
10620
10629
|
let debited = false;
|
|
10621
10630
|
try {
|
|
@@ -10659,7 +10668,7 @@ facebookAdApp.post("/video-transcribe", createApiKeyAuth(), async (c) => {
|
|
|
10659
10668
|
metadata: { url: sourceUrl.href }
|
|
10660
10669
|
});
|
|
10661
10670
|
if (!gate.ok) return c.json(concurrencyLimitExceededResponse(gate), 429, { "Retry-After": String(gate.retryAfterSeconds) });
|
|
10662
|
-
|
|
10671
|
+
fal.config({ credentials: process.env.FAL_KEY });
|
|
10663
10672
|
const driver = new BrowserDriver();
|
|
10664
10673
|
const holdMc = MEDIA_TRANSCRIBE_HOLD_MC;
|
|
10665
10674
|
let debited = false;
|
|
@@ -10684,7 +10693,8 @@ facebookAdApp.post("/video-transcribe", createApiKeyAuth(), async (c) => {
|
|
|
10684
10693
|
text: transcript.text,
|
|
10685
10694
|
chunks: transcript.chunks,
|
|
10686
10695
|
durationMs: transcript.durationMs,
|
|
10687
|
-
markdown: transcript.markdown
|
|
10696
|
+
markdown: transcript.markdown,
|
|
10697
|
+
transcriptSignal: assessTranscriptSignal(transcript.text, transcript.chunks, video.durationSec)
|
|
10688
10698
|
};
|
|
10689
10699
|
await logRequestEvent({ userId: fbUser.id, source: "facebook_video_transcribe", status: "done", query: sourceUrl.href, resultCount: transcript.chunks.length, result });
|
|
10690
10700
|
return c.json(result);
|
|
@@ -11018,7 +11028,7 @@ var GoogleAdsExtractor = class {
|
|
|
11018
11028
|
};
|
|
11019
11029
|
|
|
11020
11030
|
// src/api/google-ads-routes.ts
|
|
11021
|
-
import { fal as
|
|
11031
|
+
import { fal as fal2 } from "@fal-ai/client";
|
|
11022
11032
|
var GoogleAdsSearchBodySchema = z14.object({
|
|
11023
11033
|
query: z14.string().trim().min(1, "query is required"),
|
|
11024
11034
|
region: z14.string().trim().toUpperCase().length(2).optional(),
|
|
@@ -11148,7 +11158,7 @@ googleAdsApp.post("/transcribe", createApiKeyAuth(), async (c) => {
|
|
|
11148
11158
|
const user = c.get("user");
|
|
11149
11159
|
const gate = await acquireConcurrencyGate(user, "google_ads_transcribe", { reuseLockId: c.req.header("x-mcp-scraper-concurrency-lock"), metadata: { videoUrl } });
|
|
11150
11160
|
if (!gate.ok) return c.json(concurrencyLimitExceededResponse(gate), 429, { "Retry-After": String(gate.retryAfterSeconds) });
|
|
11151
|
-
|
|
11161
|
+
fal2.config({ credentials: process.env.FAL_KEY });
|
|
11152
11162
|
const holdMc = MEDIA_TRANSCRIBE_HOLD_MC;
|
|
11153
11163
|
let debited = false;
|
|
11154
11164
|
try {
|
|
@@ -20678,6 +20688,7 @@ var CONNECTED_DATA_DATASETS = [
|
|
|
20678
20688
|
"calendar_events",
|
|
20679
20689
|
"zoom_recordings",
|
|
20680
20690
|
"zoom_transcripts",
|
|
20691
|
+
"meta_ads_insights",
|
|
20681
20692
|
"resend_data",
|
|
20682
20693
|
"resend_emails",
|
|
20683
20694
|
"resend_received_emails",
|
|
@@ -21157,10 +21168,17 @@ var CONNECTION_SYNC_REQUIRED_TOOLS = {
|
|
|
21157
21168
|
"list-ad-sets",
|
|
21158
21169
|
"list-ads",
|
|
21159
21170
|
"list-ad-creatives",
|
|
21160
|
-
"get-insights"
|
|
21171
|
+
"get-insights"
|
|
21172
|
+
]
|
|
21173
|
+
};
|
|
21174
|
+
var CONNECTION_SYNC_OPTIONAL_TOOLS = {
|
|
21175
|
+
"meta-marketing-api": [
|
|
21161
21176
|
"list-catalogs",
|
|
21162
21177
|
"list-datasets",
|
|
21163
|
-
"list-custom-audiences"
|
|
21178
|
+
"list-custom-audiences",
|
|
21179
|
+
"list-ad-images",
|
|
21180
|
+
"list-ad-videos",
|
|
21181
|
+
"list-ad-rules"
|
|
21164
21182
|
]
|
|
21165
21183
|
};
|
|
21166
21184
|
function connectionSyncPolicyIssues(selections) {
|
|
@@ -21176,8 +21194,11 @@ function connectionSyncPolicyIssues(selections) {
|
|
|
21176
21194
|
if (missingTools.length > 0) {
|
|
21177
21195
|
issues.push({ providerConfigKey: selection.providerConfigKey, code: "missing_required_tools", missingTools });
|
|
21178
21196
|
}
|
|
21179
|
-
const
|
|
21180
|
-
|
|
21197
|
+
const permittedSet = /* @__PURE__ */ new Set([
|
|
21198
|
+
...required,
|
|
21199
|
+
...CONNECTION_SYNC_OPTIONAL_TOOLS[selection.providerConfigKey] ?? []
|
|
21200
|
+
]);
|
|
21201
|
+
const unexpectedTools = [...allowed].filter((tool) => !permittedSet.has(tool));
|
|
21181
21202
|
if (unexpectedTools.length > 0) {
|
|
21182
21203
|
issues.push({ providerConfigKey: selection.providerConfigKey, code: "unexpected_tools", unexpectedTools });
|
|
21183
21204
|
}
|
|
@@ -21286,7 +21307,10 @@ var SAFE_CONTROL_ERROR_CODES = /* @__PURE__ */ new Set([
|
|
|
21286
21307
|
"tool_discovery_failed",
|
|
21287
21308
|
"live_tool_missing",
|
|
21288
21309
|
"connection_transport_unavailable",
|
|
21289
|
-
"connection_control_failed"
|
|
21310
|
+
"connection_control_failed",
|
|
21311
|
+
"missing_provider_permission",
|
|
21312
|
+
"permission_verification_unavailable",
|
|
21313
|
+
"meta_app_feature_not_enabled"
|
|
21290
21314
|
]);
|
|
21291
21315
|
var CONTROL_ERROR_CODE_ALIASES = /* @__PURE__ */ new Map([
|
|
21292
21316
|
["connection_not_active", "connection_inactive"],
|
|
@@ -21405,6 +21429,24 @@ async function getNangoCatalog() {
|
|
|
21405
21429
|
const actionTools = cleanTools(
|
|
21406
21430
|
row.actionTools ?? row.action_tools ?? row.writeTools ?? row.write_tools
|
|
21407
21431
|
).filter((tool) => !disabledTools?.has(tool));
|
|
21432
|
+
const requiredPermissionsByTool = {};
|
|
21433
|
+
const rawRequiredPermissions = row.requiredPermissionsByTool ?? row.required_permissions_by_tool;
|
|
21434
|
+
if (isRecord2(rawRequiredPermissions)) {
|
|
21435
|
+
for (const [tool, permissions] of Object.entries(rawRequiredPermissions).slice(0, 500)) {
|
|
21436
|
+
const cleanTool = cleanString(tool, 200);
|
|
21437
|
+
if (!cleanTool || !safeDefaultAllowedTools.includes(cleanTool) && !actionTools.includes(cleanTool)) continue;
|
|
21438
|
+
requiredPermissionsByTool[cleanTool] = cleanStringArray(permissions, 32, 200);
|
|
21439
|
+
}
|
|
21440
|
+
}
|
|
21441
|
+
const requiredFeaturesByTool = {};
|
|
21442
|
+
const rawRequiredFeatures = row.requiredFeaturesByTool ?? row.required_features_by_tool;
|
|
21443
|
+
if (isRecord2(rawRequiredFeatures)) {
|
|
21444
|
+
for (const [tool, features] of Object.entries(rawRequiredFeatures).slice(0, 500)) {
|
|
21445
|
+
const cleanTool = cleanString(tool, 200);
|
|
21446
|
+
if (!cleanTool) continue;
|
|
21447
|
+
requiredFeaturesByTool[cleanTool] = cleanStringArray(features, 32, 200);
|
|
21448
|
+
}
|
|
21449
|
+
}
|
|
21408
21450
|
const platformSetupStatus = firstString(row, [
|
|
21409
21451
|
"platformSetupStatus",
|
|
21410
21452
|
"platform_setup_status",
|
|
@@ -21414,6 +21456,7 @@ async function getNangoCatalog() {
|
|
|
21414
21456
|
const appReviewStatus = row.appReviewLimited === true || row.app_review_limited === true ? "limited" : firstString(row, ["appReviewStatus", "app_review_status", "reviewStatus", "review_status"], 100);
|
|
21415
21457
|
const appReviewNote = firstString(row, ["appReviewNote", "app_review_note", "reviewNote", "review_note"], 500);
|
|
21416
21458
|
const connectionSyncRequiredTools = [...CONNECTION_SYNC_REQUIRED_TOOLS[providerConfigKey] ?? []];
|
|
21459
|
+
const connectionSyncOptionalTools = [...CONNECTION_SYNC_OPTIONAL_TOOLS[providerConfigKey] ?? []];
|
|
21417
21460
|
result.push({
|
|
21418
21461
|
providerConfigKey,
|
|
21419
21462
|
provider,
|
|
@@ -21425,8 +21468,12 @@ async function getNangoCatalog() {
|
|
|
21425
21468
|
categories,
|
|
21426
21469
|
safeDefaultAllowedTools,
|
|
21427
21470
|
actionTools,
|
|
21471
|
+
requiredPermissionsByTool,
|
|
21472
|
+
requiredFeaturesByTool,
|
|
21473
|
+
enabledFeatures: cleanStringArray(row.enabledFeatures ?? row.enabled_features, 32, 200),
|
|
21428
21474
|
connectionSyncSupported: connectionSyncRequiredTools.length > 0,
|
|
21429
21475
|
connectionSyncRequiredTools,
|
|
21476
|
+
connectionSyncOptionalTools,
|
|
21430
21477
|
platformSetupStatus,
|
|
21431
21478
|
appReviewStatus,
|
|
21432
21479
|
appReviewNote
|
|
@@ -21448,6 +21495,30 @@ async function getNangoConnections(identity) {
|
|
|
21448
21495
|
const label = firstString(row, ["label", "accountLabel", "account_label", "displayName", "display_name", "endUserEmail", "end_user_email"]) || providerConfigKey;
|
|
21449
21496
|
const rawStatus = firstString(row, ["status"], 64) || "connected";
|
|
21450
21497
|
const reconnectRequired = row.reconnectRequired === true || row.reconnect_required === true || rawStatus === "needs_reauth" || rawStatus === "reauth_required" || rawStatus === "invalid" || rawStatus === "error" || rawStatus === "revoked" || rawStatus === "disabled";
|
|
21498
|
+
const toolCapabilities = [];
|
|
21499
|
+
const rawToolCapabilities = row.toolCapabilities ?? row.tool_capabilities;
|
|
21500
|
+
if (Array.isArray(rawToolCapabilities)) {
|
|
21501
|
+
for (const value of rawToolCapabilities.slice(0, 500)) {
|
|
21502
|
+
if (!isRecord2(value)) continue;
|
|
21503
|
+
const name = cleanString(value.name, 200);
|
|
21504
|
+
const classification = value.classification;
|
|
21505
|
+
const blockedValue = Object.prototype.hasOwnProperty.call(value, "blockedReason") ? value.blockedReason : value.blocked_reason;
|
|
21506
|
+
const blockedReason = blockedValue === null ? null : blockedValue === "missing_permission" || blockedValue === "permission_policy_missing" || blockedValue === "permission_verification_unavailable" || blockedValue === "missing_app_feature" ? blockedValue : void 0;
|
|
21507
|
+
if (!name || classification !== "read" && classification !== "action" || typeof value.available !== "boolean" || blockedReason === void 0) continue;
|
|
21508
|
+
toolCapabilities.push({
|
|
21509
|
+
name,
|
|
21510
|
+
classification,
|
|
21511
|
+
requiredPermissions: cleanStringArray(value.requiredPermissions ?? value.required_permissions, 32, 200),
|
|
21512
|
+
requiredFeatures: cleanStringArray(value.requiredFeatures ?? value.required_features, 32, 200),
|
|
21513
|
+
available: value.available,
|
|
21514
|
+
blockedReason,
|
|
21515
|
+
missingPermissions: cleanStringArray(value.missingPermissions ?? value.missing_permissions, 32, 200),
|
|
21516
|
+
missingFeatures: cleanStringArray(value.missingFeatures ?? value.missing_features, 32, 200)
|
|
21517
|
+
});
|
|
21518
|
+
}
|
|
21519
|
+
}
|
|
21520
|
+
const permissionVerificationValue = row.permissionVerification ?? row.permission_verification;
|
|
21521
|
+
const permissionVerification = permissionVerificationValue === "verified" || permissionVerificationValue === "unavailable" ? permissionVerificationValue : null;
|
|
21451
21522
|
result.push({
|
|
21452
21523
|
connectionId,
|
|
21453
21524
|
providerConfigKey,
|
|
@@ -21458,6 +21529,10 @@ async function getNangoConnections(identity) {
|
|
|
21458
21529
|
actionsEnabled: row.actionsEnabled === true || row.actions_enabled === true,
|
|
21459
21530
|
readTools: cleanTools(row.readTools ?? row.read_tools),
|
|
21460
21531
|
actionTools: cleanTools(row.actionTools ?? row.action_tools ?? row.writeTools ?? row.write_tools),
|
|
21532
|
+
toolCapabilities,
|
|
21533
|
+
grantedPermissions: cleanStringArray(row.grantedPermissions ?? row.granted_permissions, 64, 200),
|
|
21534
|
+
enabledFeatures: cleanStringArray(row.enabledFeatures ?? row.enabled_features, 32, 200),
|
|
21535
|
+
permissionVerification,
|
|
21461
21536
|
mcpEndpoint: null,
|
|
21462
21537
|
schemaDiscovery: "compatibility_describe",
|
|
21463
21538
|
toolRevision: firstString(row, ["toolRevision", "tool_revision"], 200),
|
|
@@ -21685,7 +21760,7 @@ async function describeNangoTool(identity, connectionId, tool, fresh) {
|
|
|
21685
21760
|
const outputSchemaValue = rawTool.outputSchema ?? rawTool.output_schema;
|
|
21686
21761
|
const outputSchema = outputSchemaValue === void 0 ? void 0 : sanitizeToolSchema(outputSchemaValue);
|
|
21687
21762
|
const blockedReasonValue = rawTool.blockedReason ?? rawTool.blocked_reason;
|
|
21688
|
-
const blockedReason = blockedReasonValue === null || blockedReasonValue === void 0 ? null : blockedReasonValue === "actions_disabled" || blockedReasonValue === "inactive_connection" ? blockedReasonValue : void 0;
|
|
21763
|
+
const blockedReason = blockedReasonValue === null || blockedReasonValue === void 0 ? null : blockedReasonValue === "actions_disabled" || blockedReasonValue === "inactive_connection" || blockedReasonValue === "missing_permission" || blockedReasonValue === "missing_app_feature" || blockedReasonValue === "permission_policy_missing" || blockedReasonValue === "permission_verification_unavailable" ? blockedReasonValue : void 0;
|
|
21689
21764
|
if (serializedBytes > 256 * 1024 || !name || name !== tool || classification !== "read" && classification !== "action" || typeof rawTool.callable !== "boolean" || blockedReason === void 0 || transport !== "nango" && transport !== "remote_mcp" || !providerConfigKey || schemaSource !== "live_tools_list" || !upstreamSchemaHash || !/^[a-f0-9]{64}$/i.test(upstreamSchemaHash) || !fetchedAt || Number.isNaN(Date.parse(fetchedAt)) || !inputSchema || outputSchemaValue !== void 0 && !outputSchema) {
|
|
21690
21765
|
throw new NangoControlError(
|
|
21691
21766
|
"The connection service returned an invalid live tool description.",
|
|
@@ -21702,6 +21777,10 @@ async function describeNangoTool(identity, connectionId, tool, fresh) {
|
|
|
21702
21777
|
const annotations = sanitizeToolAnnotations(rawTool.annotations);
|
|
21703
21778
|
const icons = sanitizeToolIcons(rawTool.icons);
|
|
21704
21779
|
const execution = taskSupport ? { taskSupport } : void 0;
|
|
21780
|
+
const requiredPermissions = cleanStringArray(rawTool.requiredPermissions ?? rawTool.required_permissions, 32, 200);
|
|
21781
|
+
const missingPermissions = cleanStringArray(rawTool.missingPermissions ?? rawTool.missing_permissions, 32, 200);
|
|
21782
|
+
const requiredFeatures = cleanStringArray(rawTool.requiredFeatures ?? rawTool.required_features, 32, 200);
|
|
21783
|
+
const missingFeatures = cleanStringArray(rawTool.missingFeatures ?? rawTool.missing_features, 32, 200);
|
|
21705
21784
|
const schemaHash = projectedToolSchemaHash({
|
|
21706
21785
|
name,
|
|
21707
21786
|
inputSchema,
|
|
@@ -21719,6 +21798,10 @@ async function describeNangoTool(identity, connectionId, tool, fresh) {
|
|
|
21719
21798
|
classification,
|
|
21720
21799
|
callable: rawTool.callable,
|
|
21721
21800
|
blockedReason,
|
|
21801
|
+
requiredPermissions,
|
|
21802
|
+
missingPermissions,
|
|
21803
|
+
requiredFeatures,
|
|
21804
|
+
missingFeatures,
|
|
21722
21805
|
transport,
|
|
21723
21806
|
providerConfigKey,
|
|
21724
21807
|
protocolVersion,
|
|
@@ -22768,7 +22851,34 @@ async function combinedConnections(identity) {
|
|
|
22768
22851
|
if (nango.status === "rejected" && resend.status === "rejected") throw nango.reason;
|
|
22769
22852
|
return [
|
|
22770
22853
|
...nango.status === "fulfilled" ? nango.value.map((connection) => ({ ...connection, transport: "nango", adminBlockedTools: [] })) : [],
|
|
22771
|
-
...resend.status === "fulfilled" ? resend.value
|
|
22854
|
+
...resend.status === "fulfilled" ? resend.value.map((connection) => ({
|
|
22855
|
+
...connection,
|
|
22856
|
+
toolCapabilities: [
|
|
22857
|
+
...connection.readTools.map((name) => ({
|
|
22858
|
+
name,
|
|
22859
|
+
classification: "read",
|
|
22860
|
+
requiredPermissions: [],
|
|
22861
|
+
requiredFeatures: [],
|
|
22862
|
+
available: true,
|
|
22863
|
+
blockedReason: null,
|
|
22864
|
+
missingPermissions: [],
|
|
22865
|
+
missingFeatures: []
|
|
22866
|
+
})),
|
|
22867
|
+
...connection.actionTools.map((name) => ({
|
|
22868
|
+
name,
|
|
22869
|
+
classification: "action",
|
|
22870
|
+
requiredPermissions: [],
|
|
22871
|
+
requiredFeatures: [],
|
|
22872
|
+
available: true,
|
|
22873
|
+
blockedReason: null,
|
|
22874
|
+
missingPermissions: [],
|
|
22875
|
+
missingFeatures: []
|
|
22876
|
+
}))
|
|
22877
|
+
],
|
|
22878
|
+
grantedPermissions: [],
|
|
22879
|
+
enabledFeatures: [],
|
|
22880
|
+
permissionVerification: null
|
|
22881
|
+
})) : []
|
|
22772
22882
|
];
|
|
22773
22883
|
}
|
|
22774
22884
|
async function isResendConnection(identity, connectionId, providerHint) {
|
|
@@ -24160,7 +24270,7 @@ app.get("/cron/tick", async (c) => {
|
|
|
24160
24270
|
if (!process.env.CRON_SECRET || secret2 !== `Bearer ${process.env.CRON_SECRET}`) {
|
|
24161
24271
|
return c.json({ error: "Unauthorized" }, 401);
|
|
24162
24272
|
}
|
|
24163
|
-
const { drainQueue } = await import("./worker-
|
|
24273
|
+
const { drainQueue } = await import("./worker-AM2DHUWG.js");
|
|
24164
24274
|
const budget = { maxJobs: 10, deadlineMs: Date.now() + 28e4 };
|
|
24165
24275
|
const workflowDispatchResult = await dispatchDueWorkflowSchedules(`${new URL(c.req.url).protocol}//${new URL(c.req.url).host}`);
|
|
24166
24276
|
const [results, sweepResult, reapResult, expiredResult, blobCleanup, connectedDataArtifactCleanup] = await Promise.all([
|
|
@@ -24361,4 +24471,4 @@ app.get("/blog/:slug/", (c) => {
|
|
|
24361
24471
|
export {
|
|
24362
24472
|
app
|
|
24363
24473
|
};
|
|
24364
|
-
//# sourceMappingURL=server-
|
|
24474
|
+
//# sourceMappingURL=server-VWXDE64Y.js.map
|