mcp-scraper 0.11.0 → 0.12.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/dist/bin/api-server.cjs +85 -11
- package/dist/bin/api-server.cjs.map +1 -1
- package/dist/bin/api-server.js +1 -1
- package/dist/bin/mcp-scraper-cli.cjs +1 -1
- package/dist/bin/mcp-scraper-cli.cjs.map +1 -1
- package/dist/bin/mcp-scraper-cli.js +1 -1
- package/dist/bin/mcp-scraper-install.cjs +1 -1
- package/dist/bin/mcp-scraper-install.cjs.map +1 -1
- package/dist/bin/mcp-scraper-install.js +1 -1
- package/dist/bin/mcp-stdio-server.cjs +6 -3
- package/dist/bin/mcp-stdio-server.cjs.map +1 -1
- package/dist/bin/mcp-stdio-server.js +2 -2
- package/dist/{chunk-QJBWER2Q.js → chunk-HXNE5GLG.js} +7 -4
- package/dist/{chunk-QJBWER2Q.js.map → chunk-HXNE5GLG.js.map} +1 -1
- package/dist/chunk-YA364G53.js +7 -0
- package/dist/chunk-YA364G53.js.map +1 -0
- package/dist/{server-LRWHUBUN.js → server-TFWBW24U.js} +81 -10
- package/dist/{server-LRWHUBUN.js.map → server-TFWBW24U.js.map} +1 -1
- package/docs/mcp-tool-manifest.generated.json +29 -4
- package/package.json +2 -2
- package/dist/chunk-P2WZLJKQ.js +0 -7
- package/dist/chunk-P2WZLJKQ.js.map +0 -1
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"sources":["../src/version.ts"],"sourcesContent":["export const PACKAGE_VERSION = '0.12.0'\n"],"mappings":";AAAO,IAAM,kBAAkB;","names":[]}
|
|
@@ -34,7 +34,7 @@ import {
|
|
|
34
34
|
registerSerpIntelligenceCaptureTools,
|
|
35
35
|
sanitizeAttempts,
|
|
36
36
|
sanitizeHarvestResult
|
|
37
|
-
} from "./chunk-
|
|
37
|
+
} from "./chunk-HXNE5GLG.js";
|
|
38
38
|
import {
|
|
39
39
|
auditImages,
|
|
40
40
|
buildLinkReport,
|
|
@@ -70,7 +70,7 @@ import {
|
|
|
70
70
|
RawMapsOverviewSchema,
|
|
71
71
|
RawMapsReviewStatsSchema
|
|
72
72
|
} from "./chunk-XGIPATLV.js";
|
|
73
|
-
import "./chunk-
|
|
73
|
+
import "./chunk-YA364G53.js";
|
|
74
74
|
import {
|
|
75
75
|
completeExtractJob,
|
|
76
76
|
countSuccessfulPages,
|
|
@@ -20399,6 +20399,54 @@ var DISABLED_NANGO_TOOLS = {
|
|
|
20399
20399
|
"google-mail": /* @__PURE__ */ new Set(["list-filters"]),
|
|
20400
20400
|
slack: /* @__PURE__ */ new Set(["search-files", "search-messages"])
|
|
20401
20401
|
};
|
|
20402
|
+
var CONNECTION_SYNC_REQUIRED_TOOLS = {
|
|
20403
|
+
"google-mail": ["list-messages", "get-message"],
|
|
20404
|
+
"google-calendar": ["list-events", "get-event"],
|
|
20405
|
+
zoom: [
|
|
20406
|
+
"list-meetings",
|
|
20407
|
+
"get-meeting",
|
|
20408
|
+
"list-recordings",
|
|
20409
|
+
"get-recording",
|
|
20410
|
+
"get-meeting-transcript"
|
|
20411
|
+
]
|
|
20412
|
+
};
|
|
20413
|
+
function connectionSyncPolicyIssues(selections) {
|
|
20414
|
+
const issues = [];
|
|
20415
|
+
for (const selection of selections) {
|
|
20416
|
+
const required = CONNECTION_SYNC_REQUIRED_TOOLS[selection.providerConfigKey];
|
|
20417
|
+
if (!required) {
|
|
20418
|
+
issues.push({ providerConfigKey: selection.providerConfigKey, code: "unsupported_provider" });
|
|
20419
|
+
continue;
|
|
20420
|
+
}
|
|
20421
|
+
const allowed = new Set(selection.allowedTools);
|
|
20422
|
+
const missingTools = required.filter((tool) => !allowed.has(tool));
|
|
20423
|
+
if (missingTools.length > 0) {
|
|
20424
|
+
issues.push({ providerConfigKey: selection.providerConfigKey, code: "missing_required_tools", missingTools });
|
|
20425
|
+
}
|
|
20426
|
+
const requiredSet = new Set(required);
|
|
20427
|
+
const unexpectedTools = [...allowed].filter((tool) => !requiredSet.has(tool));
|
|
20428
|
+
if (unexpectedTools.length > 0) {
|
|
20429
|
+
issues.push({ providerConfigKey: selection.providerConfigKey, code: "unexpected_tools", unexpectedTools });
|
|
20430
|
+
}
|
|
20431
|
+
}
|
|
20432
|
+
return issues;
|
|
20433
|
+
}
|
|
20434
|
+
function connectionSyncSelectionError(connections) {
|
|
20435
|
+
if (connections.length === 0) {
|
|
20436
|
+
return "Data sync requires at least one bound service connection with approved read-only tools.";
|
|
20437
|
+
}
|
|
20438
|
+
const issues = connectionSyncPolicyIssues(connections);
|
|
20439
|
+
const unsupported = [...new Set(issues.filter((issue) => issue.code === "unsupported_provider").map((issue) => issue.providerConfigKey))];
|
|
20440
|
+
if (unsupported.length > 0) {
|
|
20441
|
+
return `Data sync currently supports only ${Object.keys(CONNECTION_SYNC_REQUIRED_TOOLS).join(", ")}. Unsupported providerConfigKey: ${unsupported.join(", ")}.`;
|
|
20442
|
+
}
|
|
20443
|
+
const missing = issues.filter((issue) => issue.code === "missing_required_tools").map((issue) => `${issue.providerConfigKey}: ${(issue.missingTools ?? []).join(", ")}`);
|
|
20444
|
+
if (missing.length > 0) {
|
|
20445
|
+
return `Data sync is missing required approved read tools (${missing.join("; ")}).`;
|
|
20446
|
+
}
|
|
20447
|
+
const unexpected = issues.filter((issue) => issue.code === "unexpected_tools").map((issue) => `${issue.providerConfigKey}: ${(issue.unexpectedTools ?? []).join(", ")}`);
|
|
20448
|
+
return unexpected.length > 0 ? `Data sync permits only its exact read-only tool set; remove unexpected tools (${unexpected.join("; ")}).` : null;
|
|
20449
|
+
}
|
|
20402
20450
|
var NangoControlError = class extends Error {
|
|
20403
20451
|
status;
|
|
20404
20452
|
constructor(message, status = 502) {
|
|
@@ -20542,6 +20590,7 @@ async function getNangoCatalog() {
|
|
|
20542
20590
|
], 100);
|
|
20543
20591
|
const appReviewStatus = row.appReviewLimited === true || row.app_review_limited === true ? "limited" : firstString(row, ["appReviewStatus", "app_review_status", "reviewStatus", "review_status"], 100);
|
|
20544
20592
|
const appReviewNote = firstString(row, ["appReviewNote", "app_review_note", "reviewNote", "review_note"], 500);
|
|
20593
|
+
const connectionSyncRequiredTools = [...CONNECTION_SYNC_REQUIRED_TOOLS[providerConfigKey] ?? []];
|
|
20545
20594
|
result.push({
|
|
20546
20595
|
providerConfigKey,
|
|
20547
20596
|
provider,
|
|
@@ -20552,6 +20601,8 @@ async function getNangoCatalog() {
|
|
|
20552
20601
|
authMode,
|
|
20553
20602
|
categories,
|
|
20554
20603
|
safeDefaultAllowedTools,
|
|
20604
|
+
connectionSyncSupported: connectionSyncRequiredTools.length > 0,
|
|
20605
|
+
connectionSyncRequiredTools,
|
|
20555
20606
|
platformSetupStatus,
|
|
20556
20607
|
appReviewStatus,
|
|
20557
20608
|
appReviewNote
|
|
@@ -21530,9 +21581,10 @@ app.get("/schedule-actions", auth2, async (c) => {
|
|
|
21530
21581
|
try {
|
|
21531
21582
|
const actions = await Promise.all(r.actions.map(async (action) => {
|
|
21532
21583
|
const id = typeof action.id === "string" ? action.id : "";
|
|
21533
|
-
|
|
21584
|
+
const executionMode = action.executionMode === "connection_sync" ? "connection_sync" : "agent";
|
|
21585
|
+
if (!id) return { ...action, executionMode, connections: [] };
|
|
21534
21586
|
const bindings = await getScheduleConnectionBindings(user.email, id);
|
|
21535
|
-
return { ...action, connections: bindingsForSchedule(bindings, id) };
|
|
21587
|
+
return { ...action, executionMode, connections: bindingsForSchedule(bindings, id) };
|
|
21536
21588
|
}));
|
|
21537
21589
|
return c.json({
|
|
21538
21590
|
...r,
|
|
@@ -21542,7 +21594,7 @@ app.get("/schedule-actions", auth2, async (c) => {
|
|
|
21542
21594
|
console.warn("[schedule-actions] connection bindings unavailable:", err instanceof Error ? err.message : String(err));
|
|
21543
21595
|
return c.json({
|
|
21544
21596
|
...r,
|
|
21545
|
-
actions: r.actions.map((action) => ({ ...action, connections: [] })),
|
|
21597
|
+
actions: r.actions.map((action) => ({ ...action, executionMode: action.executionMode === "connection_sync" ? "connection_sync" : "agent", connections: [] })),
|
|
21546
21598
|
connectionBindingsUnavailable: true
|
|
21547
21599
|
});
|
|
21548
21600
|
}
|
|
@@ -21552,6 +21604,10 @@ app.post("/schedule-actions", auth2, async (c) => {
|
|
|
21552
21604
|
const { key, error } = await getOrCreateUserMemoryKey(user);
|
|
21553
21605
|
if (!key) return c.json({ ok: false, error: error ?? "memory unavailable" }, 503);
|
|
21554
21606
|
const body = await c.req.json().catch(() => ({}));
|
|
21607
|
+
const executionMode = body.executionMode ?? "agent";
|
|
21608
|
+
if (executionMode !== "agent" && executionMode !== "connection_sync") {
|
|
21609
|
+
return c.json({ ok: false, error: 'executionMode must be "agent" or "connection_sync".' }, 400);
|
|
21610
|
+
}
|
|
21555
21611
|
const hasConnections = Object.prototype.hasOwnProperty.call(body, "connections");
|
|
21556
21612
|
let connections;
|
|
21557
21613
|
try {
|
|
@@ -21559,18 +21615,23 @@ app.post("/schedule-actions", auth2, async (c) => {
|
|
|
21559
21615
|
} catch (err) {
|
|
21560
21616
|
return scheduleConnectionError(c, err, "Invalid service connections.");
|
|
21561
21617
|
}
|
|
21562
|
-
|
|
21618
|
+
if (executionMode === "connection_sync") {
|
|
21619
|
+
const policyError = connectionSyncSelectionError(connections);
|
|
21620
|
+
if (policyError) return c.json({ ok: false, error: policyError }, 400);
|
|
21621
|
+
}
|
|
21622
|
+
const actionBody = { ...body, executionMode };
|
|
21563
21623
|
delete actionBody.connections;
|
|
21564
21624
|
const r = await memoryCall("createScheduledActionTool", actionBody, key);
|
|
21565
|
-
|
|
21566
|
-
if (
|
|
21625
|
+
const createResponse = { ...r, executionMode };
|
|
21626
|
+
if (!r.ok || !hasConnections) return c.json(createResponse);
|
|
21627
|
+
if (connections.length === 0) return c.json({ ...createResponse, connections: [] });
|
|
21567
21628
|
if (!r.id) {
|
|
21568
21629
|
console.error("[schedule-actions] create returned no id while service connections were requested");
|
|
21569
21630
|
return c.json({ ok: false, error: "The scheduled action was created without a usable id; service connections were not attached." }, 502);
|
|
21570
21631
|
}
|
|
21571
21632
|
try {
|
|
21572
21633
|
const bound = await putScheduleConnectionBindings(user.email, r.id, connections);
|
|
21573
|
-
return c.json({ ...
|
|
21634
|
+
return c.json({ ...createResponse, connections: bound });
|
|
21574
21635
|
} catch (err) {
|
|
21575
21636
|
const rollback = await memoryCall("deleteScheduledActionTool", { id: r.id }, key).catch(() => ({ ok: false }));
|
|
21576
21637
|
if (!rollback.ok) console.error("[schedule-actions] failed to compensate action after connection binding failure", r.id);
|
|
@@ -21608,6 +21669,16 @@ app.put("/schedule-actions/:id/connections", auth2, async (c) => {
|
|
|
21608
21669
|
}
|
|
21609
21670
|
try {
|
|
21610
21671
|
const scheduleActionId = c.req.param("id");
|
|
21672
|
+
const { key, error } = await getOrCreateUserMemoryKey(user);
|
|
21673
|
+
if (!key) return c.json({ ok: false, error: error ?? "memory unavailable" }, 503);
|
|
21674
|
+
const schedules = await memoryCall("listScheduledActionsTool", {}, key);
|
|
21675
|
+
if (!schedules.ok || !Array.isArray(schedules.actions)) return c.json(schedules, 502);
|
|
21676
|
+
const schedule = schedules.actions.find((action) => action.id === scheduleActionId);
|
|
21677
|
+
if (!schedule) return c.json({ ok: false, error: "scheduled action not found" }, 404);
|
|
21678
|
+
if (schedule.executionMode === "connection_sync") {
|
|
21679
|
+
const policyError = connectionSyncSelectionError(connections);
|
|
21680
|
+
if (policyError) return c.json({ ok: false, error: policyError }, 400);
|
|
21681
|
+
}
|
|
21611
21682
|
const bindings = await putScheduleConnectionBindings(user.email, scheduleActionId, connections);
|
|
21612
21683
|
return c.json({ ok: true, connections: bindings });
|
|
21613
21684
|
} catch (err) {
|
|
@@ -22694,4 +22765,4 @@ app.get("/blog/:slug/", (c) => {
|
|
|
22694
22765
|
export {
|
|
22695
22766
|
app
|
|
22696
22767
|
};
|
|
22697
|
-
//# sourceMappingURL=server-
|
|
22768
|
+
//# sourceMappingURL=server-TFWBW24U.js.map
|