mcp-scraper 0.11.0 → 0.12.0

package/README.md

@@ -88,7 +88,7 @@ Build the branded one-click bundle:
 npm run build:mcpb
 ```
 
-The generated bundle is written to `build/mcpb/mcp-scraper-<version>.mcpb` and copied to `public/downloads/` for the hosted download. The current public bundle is `https://mcpscraper.dev/downloads/mcp-scraper.mcpb` (`0.11.0`, SHA-256 `ed9f00c3f5ea47cc79b5e6e0205c2f85fecb9aa0eed5dcd55fee9e4e309ecfac`). Install it by opening or dragging it into Claude Desktop. Claude displays the `MCP Scraper` install card, icon, and API-key configuration field from the bundle manifest.
+The generated bundle is written to `build/mcpb/mcp-scraper-<version>.mcpb` and copied to `public/downloads/` for the hosted download. The current public bundle is `https://mcpscraper.dev/downloads/mcp-scraper.mcpb` (`0.12.0`, SHA-256 `2a0dfdc53e93d48588ef672c6daf0ec0e6194b3f938add02b106bf5e86c20ef3`). Install it by opening or dragging it into Claude Desktop. Claude displays the `MCP Scraper` install card, icon, and API-key configuration field from the bundle manifest.
 
 The MCPB install exposes every tool — web-intelligence plus all `browser_*` tools — through the one `mcp-scraper` server.
 

package/dist/bin/api-server.cjs

@@ -27452,7 +27452,7 @@ var PACKAGE_VERSION;
 var init_version = __esm({
   "src/version.ts"() {
     "use strict";
-    PACKAGE_VERSION = "0.11.0";
+    PACKAGE_VERSION = "0.12.0";
   }
 });
 
@@ -32869,11 +32869,12 @@ var init_memory_tool_schemas = __esm({
     CreateScheduledActionSchema = {
       id: "create-scheduled-action",
       upstreamName: "createScheduledActionTool",
-      description: "Create a scheduled action: on the given cadence, an agent reads the description, does what it asks, and writes the result into the target vault. Cadence 'once' runs a single time then completes permanently \u2014 use it for one-off tasks, not recurring ones. Requires an active scheduling subscription and write access to the target vault.",
+      description: "Create a scheduled action in agent mode (default) or connection_sync mode. Agent mode asks an agent to follow the description and write a result into the target vault. connection_sync deterministically runs the approved read-only tools on bound service connections and ingests their data; it requires at least one connection to be bound before execution. Cadence 'once' runs a single time then completes permanently. Requires an active scheduling subscription and write access to the target vault.",
       input: {
         description: import_zod35.z.string().min(1).describe("Free-text description of what this action should do each time it runs."),
         vault: import_zod35.z.string().min(1).describe("The vault this action writes its results into. You must already have write access to it."),
         cadence: import_zod35.z.enum(["once", "daily", "weekly", "monthly"]).describe('How often this action runs. "once" fires a single time and then completes.'),
+        executionMode: import_zod35.z.enum(["agent", "connection_sync"]).default("agent").describe(`How to execute each run. "agent" (default) lets an agent follow the description. "connection_sync" deterministically ingests data from the schedule's bound connections using only their approved read-only tools; bind at least one connection before it runs.`),
         timeOfDay: import_zod35.z.string().regex(/^([01]\d|2[0-3]):([0-5]\d)$/).optional().describe("24-hour HH:MM clock time to run at, in the given timezone. Optional \u2014 omit to run at any time during the period (matches prior default behavior)."),
         timezone: import_zod35.z.string().optional().describe('IANA timezone name, e.g. "America/Denver". Only meaningful together with timeOfDay. Defaults to UTC.'),
         deployDate: import_zod35.z.string().regex(/^\d{4}-\d{2}-\d{2}$/).optional().describe('Calendar date (YYYY-MM-DD, in the given timezone) this action should first become eligible to run \u2014 its deployment/start date. For recurring cadences, the first occurrence lands on or after this date; every later occurrence still follows the normal cadence. For cadence "once", this (combined with timeOfDay if given) is exactly what day it fires. Omit to start immediately.')
@@ -32882,6 +32883,7 @@ var init_memory_tool_schemas = __esm({
         ok: import_zod35.z.boolean().describe("True when the scheduled action was created."),
         id: import_zod35.z.string().optional().describe("The new scheduled action id."),
         nextRunAt: import_zod35.z.string().optional().describe("When it will first run."),
+        executionMode: import_zod35.z.enum(["agent", "connection_sync"]).optional().describe("The stored execution mode. Defaults to agent when omitted from the request."),
         error: import_zod35.z.string().optional().describe("Human-readable failure reason when ok is false."),
         code: import_zod35.z.string().optional().describe("Machine-readable denial code: not_enabled when no scheduling subscription is active.")
       },
@@ -32956,7 +32958,7 @@ var init_memory_tool_schemas = __esm({
     ListScheduledActionsSchema = {
       id: "list-scheduled-actions",
       upstreamName: "listScheduledActionsTool",
-      description: "List every scheduled action you own \u2014 active, paused, and completed one-time actions \u2014 with cadence, next run time, and last run status.",
+      description: "List every scheduled action you own \u2014 active, paused, and completed one-time actions \u2014 with execution mode, cadence, next run time, and last run status. connection_sync means deterministic read-only ingestion from bound service connections.",
       input: {},
       output: {
         ok: import_zod35.z.boolean(),
@@ -32965,6 +32967,7 @@ var init_memory_tool_schemas = __esm({
           description: import_zod35.z.string(),
           vault: import_zod35.z.string(),
           cadence: import_zod35.z.enum(["once", "daily", "weekly", "monthly"]),
+          executionMode: import_zod35.z.enum(["agent", "connection_sync"]),
           timeOfDay: import_zod35.z.string().nullable(),
           timezone: import_zod35.z.string(),
           status: import_zod35.z.enum(["active", "paused", "completed"]),
@@ -38884,6 +38887,43 @@ var init_memory_db = __esm({
 });
 
 // src/api/nango-control.ts
+function connectionSyncPolicyIssues(selections) {
+  const issues = [];
+  for (const selection of selections) {
+    const required = CONNECTION_SYNC_REQUIRED_TOOLS[selection.providerConfigKey];
+    if (!required) {
+      issues.push({ providerConfigKey: selection.providerConfigKey, code: "unsupported_provider" });
+      continue;
+    }
+    const allowed = new Set(selection.allowedTools);
+    const missingTools = required.filter((tool) => !allowed.has(tool));
+    if (missingTools.length > 0) {
+      issues.push({ providerConfigKey: selection.providerConfigKey, code: "missing_required_tools", missingTools });
+    }
+    const requiredSet = new Set(required);
+    const unexpectedTools = [...allowed].filter((tool) => !requiredSet.has(tool));
+    if (unexpectedTools.length > 0) {
+      issues.push({ providerConfigKey: selection.providerConfigKey, code: "unexpected_tools", unexpectedTools });
+    }
+  }
+  return issues;
+}
+function connectionSyncSelectionError(connections) {
+  if (connections.length === 0) {
+    return "Data sync requires at least one bound service connection with approved read-only tools.";
+  }
+  const issues = connectionSyncPolicyIssues(connections);
+  const unsupported = [...new Set(issues.filter((issue) => issue.code === "unsupported_provider").map((issue) => issue.providerConfigKey))];
+  if (unsupported.length > 0) {
+    return `Data sync currently supports only ${Object.keys(CONNECTION_SYNC_REQUIRED_TOOLS).join(", ")}. Unsupported providerConfigKey: ${unsupported.join(", ")}.`;
+  }
+  const missing = issues.filter((issue) => issue.code === "missing_required_tools").map((issue) => `${issue.providerConfigKey}: ${(issue.missingTools ?? []).join(", ")}`);
+  if (missing.length > 0) {
+    return `Data sync is missing required approved read tools (${missing.join("; ")}).`;
+  }
+  const unexpected = issues.filter((issue) => issue.code === "unexpected_tools").map((issue) => `${issue.providerConfigKey}: ${(issue.unexpectedTools ?? []).join(", ")}`);
+  return unexpected.length > 0 ? `Data sync permits only its exact read-only tool set; remove unexpected tools (${unexpected.join("; ")}).` : null;
+}
 function controlBaseUrl() {
   return (process.env.NANGO_CONTROL_URL?.trim() || DEFAULT_NANGO_CONTROL_URL).replace(/\/$/, "");
 }
@@ -39013,6 +39053,7 @@ async function getNangoCatalog() {
     ], 100);
     const appReviewStatus = row.appReviewLimited === true || row.app_review_limited === true ? "limited" : firstString(row, ["appReviewStatus", "app_review_status", "reviewStatus", "review_status"], 100);
     const appReviewNote = firstString(row, ["appReviewNote", "app_review_note", "reviewNote", "review_note"], 500);
+    const connectionSyncRequiredTools = [...CONNECTION_SYNC_REQUIRED_TOOLS[providerConfigKey] ?? []];
     result.push({
       providerConfigKey,
       provider,
@@ -39023,6 +39064,8 @@ async function getNangoCatalog() {
       authMode,
       categories,
       safeDefaultAllowedTools,
+      connectionSyncSupported: connectionSyncRequiredTools.length > 0,
+      connectionSyncRequiredTools,
       platformSetupStatus,
       appReviewStatus,
       appReviewNote
@@ -39186,7 +39229,7 @@ async function callScheduleConnectionRead(identity, connectionId, tool, args) {
   const data = unwrapData(body);
   return isRecord(data) ? data.result ?? data : data;
 }
-var DEFAULT_NANGO_CONTROL_URL, DISABLED_NANGO_TOOLS, NangoControlError, ScheduleConnectionValidationError;
+var DEFAULT_NANGO_CONTROL_URL, DISABLED_NANGO_TOOLS, CONNECTION_SYNC_REQUIRED_TOOLS, NangoControlError, ScheduleConnectionValidationError;
 var init_nango_control = __esm({
   "src/api/nango-control.ts"() {
     "use strict";
@@ -39195,6 +39238,17 @@ var init_nango_control = __esm({
       "google-mail": /* @__PURE__ */ new Set(["list-filters"]),
       slack: /* @__PURE__ */ new Set(["search-files", "search-messages"])
     };
+    CONNECTION_SYNC_REQUIRED_TOOLS = {
+      "google-mail": ["list-messages", "get-message"],
+      "google-calendar": ["list-events", "get-event"],
+      zoom: [
+        "list-meetings",
+        "get-meeting",
+        "list-recordings",
+        "get-recording",
+        "get-meeting-transcript"
+      ]
+    };
     NangoControlError = class extends Error {
       status;
       constructor(message, status = 502) {
@@ -40266,9 +40320,10 @@ var init_server = __esm({
       try {
         const actions = await Promise.all(r.actions.map(async (action) => {
           const id = typeof action.id === "string" ? action.id : "";
-          if (!id) return { ...action, connections: [] };
+          const executionMode = action.executionMode === "connection_sync" ? "connection_sync" : "agent";
+          if (!id) return { ...action, executionMode, connections: [] };
           const bindings = await getScheduleConnectionBindings(user.email, id);
-          return { ...action, connections: bindingsForSchedule(bindings, id) };
+          return { ...action, executionMode, connections: bindingsForSchedule(bindings, id) };
         }));
         return c.json({
           ...r,
@@ -40278,7 +40333,7 @@ var init_server = __esm({
         console.warn("[schedule-actions] connection bindings unavailable:", err instanceof Error ? err.message : String(err));
         return c.json({
           ...r,
-          actions: r.actions.map((action) => ({ ...action, connections: [] })),
+          actions: r.actions.map((action) => ({ ...action, executionMode: action.executionMode === "connection_sync" ? "connection_sync" : "agent", connections: [] })),
           connectionBindingsUnavailable: true
         });
       }
@@ -40288,6 +40343,10 @@ var init_server = __esm({
       const { key, error } = await getOrCreateUserMemoryKey(user);
       if (!key) return c.json({ ok: false, error: error ?? "memory unavailable" }, 503);
       const body = await c.req.json().catch(() => ({}));
+      const executionMode = body.executionMode ?? "agent";
+      if (executionMode !== "agent" && executionMode !== "connection_sync") {
+        return c.json({ ok: false, error: 'executionMode must be "agent" or "connection_sync".' }, 400);
+      }
       const hasConnections = Object.prototype.hasOwnProperty.call(body, "connections");
       let connections;
       try {
@@ -40295,18 +40354,23 @@ var init_server = __esm({
       } catch (err) {
         return scheduleConnectionError(c, err, "Invalid service connections.");
       }
-      const actionBody = { ...body };
+      if (executionMode === "connection_sync") {
+        const policyError = connectionSyncSelectionError(connections);
+        if (policyError) return c.json({ ok: false, error: policyError }, 400);
+      }
+      const actionBody = { ...body, executionMode };
       delete actionBody.connections;
       const r = await memoryCall("createScheduledActionTool", actionBody, key);
-      if (!r.ok || !hasConnections) return c.json(r);
-      if (connections.length === 0) return c.json({ ...r, connections: [] });
+      const createResponse = { ...r, executionMode };
+      if (!r.ok || !hasConnections) return c.json(createResponse);
+      if (connections.length === 0) return c.json({ ...createResponse, connections: [] });
       if (!r.id) {
         console.error("[schedule-actions] create returned no id while service connections were requested");
         return c.json({ ok: false, error: "The scheduled action was created without a usable id; service connections were not attached." }, 502);
       }
       try {
         const bound = await putScheduleConnectionBindings(user.email, r.id, connections);
-        return c.json({ ...r, connections: bound });
+        return c.json({ ...createResponse, connections: bound });
       } catch (err) {
         const rollback = await memoryCall("deleteScheduledActionTool", { id: r.id }, key).catch(() => ({ ok: false }));
         if (!rollback.ok) console.error("[schedule-actions] failed to compensate action after connection binding failure", r.id);
@@ -40344,6 +40408,16 @@ var init_server = __esm({
       }
       try {
         const scheduleActionId = c.req.param("id");
+        const { key, error } = await getOrCreateUserMemoryKey(user);
+        if (!key) return c.json({ ok: false, error: error ?? "memory unavailable" }, 503);
+        const schedules = await memoryCall("listScheduledActionsTool", {}, key);
+        if (!schedules.ok || !Array.isArray(schedules.actions)) return c.json(schedules, 502);
+        const schedule = schedules.actions.find((action) => action.id === scheduleActionId);
+        if (!schedule) return c.json({ ok: false, error: "scheduled action not found" }, 404);
+        if (schedule.executionMode === "connection_sync") {
+          const policyError = connectionSyncSelectionError(connections);
+          if (policyError) return c.json({ ok: false, error: policyError }, 400);
+        }
         const bindings = await putScheduleConnectionBindings(user.email, scheduleActionId, connections);
         return c.json({ ok: true, connections: bindings });
       } catch (err) {