mcp-scraper 0.21.0 → 0.21.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (36) hide show
  1. package/README.md +4 -2
  2. package/dist/bin/api-server.cjs +368 -224
  3. package/dist/bin/api-server.cjs.map +1 -1
  4. package/dist/bin/api-server.js +2 -2
  5. package/dist/bin/mcp-scraper-cli.cjs +1 -1
  6. package/dist/bin/mcp-scraper-cli.cjs.map +1 -1
  7. package/dist/bin/mcp-scraper-cli.js +1 -1
  8. package/dist/bin/mcp-scraper-install.cjs +2 -2
  9. package/dist/bin/mcp-scraper-install.cjs.map +1 -1
  10. package/dist/bin/mcp-scraper-install.js +2 -2
  11. package/dist/bin/mcp-stdio-server.cjs +12 -8
  12. package/dist/bin/mcp-stdio-server.cjs.map +1 -1
  13. package/dist/bin/mcp-stdio-server.js +4 -4
  14. package/dist/{chunk-HXGZMMW5.js → chunk-2FF4GBQ4.js} +1 -5
  15. package/dist/{chunk-HXGZMMW5.js.map → chunk-2FF4GBQ4.js.map} +1 -1
  16. package/dist/chunk-NBF4F4UT.js +7 -0
  17. package/dist/chunk-NBF4F4UT.js.map +1 -0
  18. package/dist/{chunk-TUSLATZY.js → chunk-X3NIXHHG.js} +2 -2
  19. package/dist/{chunk-Q44DN6T2.js → chunk-YNHPZU6B.js} +2 -2
  20. package/dist/{chunk-Q44DN6T2.js.map → chunk-YNHPZU6B.js.map} +1 -1
  21. package/dist/{chunk-XHJJQRNQ.js → chunk-ZNNYPE2U.js} +13 -9
  22. package/dist/{chunk-XHJJQRNQ.js.map → chunk-ZNNYPE2U.js.map} +1 -1
  23. package/dist/{server-PH3VJ74X.js → server-CHTX6YQA.js} +336 -207
  24. package/dist/server-CHTX6YQA.js.map +1 -0
  25. package/dist/{site-extract-repository-YNPXEZCB.js → site-extract-repository-5V5EEQ72.js} +3 -3
  26. package/dist/{worker-45WEWCMQ.js → worker-GE5JMHE6.js} +2 -2
  27. package/docs/mcp-tool-manifest.generated.json +28 -10
  28. package/docs/specs/google-drive-bulk-access-and-mcp-schema-passthrough-spec.md +16 -7
  29. package/docs/specs/unified-credit-and-scheduled-execution-billing-spec.md +96 -117
  30. package/package.json +2 -2
  31. package/dist/chunk-L4LMSXLO.js +0 -7
  32. package/dist/chunk-L4LMSXLO.js.map +0 -1
  33. package/dist/server-PH3VJ74X.js.map +0 -1
  34. /package/dist/{chunk-TUSLATZY.js.map → chunk-X3NIXHHG.js.map} +0 -0
  35. /package/dist/{site-extract-repository-YNPXEZCB.js.map → site-extract-repository-5V5EEQ72.js.map} +0 -0
  36. /package/dist/{worker-45WEWCMQ.js.map → worker-GE5JMHE6.js.map} +0 -0
@@ -35,7 +35,7 @@ import {
35
35
  sanitizeAttempts,
36
36
  sanitizeHarvestResult,
37
37
  transcribeMediaUrl
38
- } from "./chunk-XHJJQRNQ.js";
38
+ } from "./chunk-ZNNYPE2U.js";
39
39
  import {
40
40
  auditImages,
41
41
  buildLinkReport,
@@ -77,7 +77,7 @@ import {
77
77
  RawMapsOverviewSchema,
78
78
  RawMapsReviewStatsSchema
79
79
  } from "./chunk-XGIPATLV.js";
80
- import "./chunk-L4LMSXLO.js";
80
+ import "./chunk-NBF4F4UT.js";
81
81
  import {
82
82
  completeExtractJob,
83
83
  countSuccessfulPages,
@@ -88,7 +88,7 @@ import {
88
88
  saveExtractPages,
89
89
  setExtractJobTotal,
90
90
  settleExtractJob
91
- } from "./chunk-TUSLATZY.js";
91
+ } from "./chunk-X3NIXHHG.js";
92
92
  import {
93
93
  BROWSER_OPEN_MIN_BALANCE_MC,
94
94
  CONCURRENCY_PRICE_ID,
@@ -108,7 +108,6 @@ import {
108
108
  SCHEDULED_RUN_BASE_CREDITS,
109
109
  SCHEDULED_RUN_BASE_MC,
110
110
  SCHEDULED_RUN_DEFAULT_MAX_CREDITS,
111
- SCHEDULING_PLANS,
112
111
  SITE_AUDIT_LLM_MARGIN_MULTIPLE,
113
112
  SUBSCRIPTION_TIERS,
114
113
  SUBSCRIPTION_TIER_BY_KEY,
@@ -118,7 +117,7 @@ import {
118
117
  concurrencySlotBillingInfo,
119
118
  insufficientBalanceResponse,
120
119
  vendorCostUsdToMc
121
- } from "./chunk-HXGZMMW5.js";
120
+ } from "./chunk-2FF4GBQ4.js";
122
121
  import {
123
122
  CaptchaError,
124
123
  RECAPTCHA_INSTRUCTIONS,
@@ -7985,6 +7984,46 @@ import { z as z10 } from "zod";
7985
7984
 
7986
7985
  // src/api/unified-billing.ts
7987
7986
  import { createHash as createHash2, createHmac, randomUUID, timingSafeEqual } from "crypto";
7987
+
7988
+ // src/api/scheduling-access.ts
7989
+ var PAID_PLAN_REQUIRED_CODE = "paid_plan_required";
7990
+ var PAID_PLAN_REQUIRED_MESSAGE = "Scheduled Actions requires an active MCP Scraper Starter plan or higher.";
7991
+ function hasPaidSchedulingAccess(user) {
7992
+ return typeof user.subscription_tier === "string" && user.subscription_tier.trim().length > 0;
7993
+ }
7994
+ function paidPlanRequiredPayload() {
7995
+ return {
7996
+ ok: false,
7997
+ enabled: false,
7998
+ code: PAID_PLAN_REQUIRED_CODE,
7999
+ error: PAID_PLAN_REQUIRED_MESSAGE,
8000
+ billingMode: "credits",
8001
+ runBaseCredits: SCHEDULED_RUN_BASE_CREDITS,
8002
+ llmCostMultiplier: SCHEDULED_AGENT_LLM_MARKUP
8003
+ };
8004
+ }
8005
+ var PLAN_GATED_MEMORY_SCHEDULE_TOOLS = /* @__PURE__ */ new Set([
8006
+ "createScheduledActionTool",
8007
+ "getScheduleLinkTool",
8008
+ "resumeScheduledActionTool"
8009
+ ]);
8010
+ function memoryScheduleAccessPayload(upstreamName, user) {
8011
+ if (hasPaidSchedulingAccess(user)) return null;
8012
+ if (upstreamName === "getScheduleStatusTool") return paidPlanRequiredPayload();
8013
+ if (PLAN_GATED_MEMORY_SCHEDULE_TOOLS.has(upstreamName)) {
8014
+ return {
8015
+ ok: false,
8016
+ code: PAID_PLAN_REQUIRED_CODE,
8017
+ error: PAID_PLAN_REQUIRED_MESSAGE
8018
+ };
8019
+ }
8020
+ return null;
8021
+ }
8022
+ function memoryScheduleToolNeedsCredentials(upstreamName) {
8023
+ return upstreamName === "createScheduledActionTool" || upstreamName === "getScheduleLinkTool";
8024
+ }
8025
+
8026
+ // src/api/unified-billing.ts
7988
8027
  var SCHEDULED_RUN_BILLING_CLASS = "scheduled_agent_run";
7989
8028
  var SCHEDULED_RUN_SOURCE_SURFACE = "mastra_scheduler";
7990
8029
  var AUTHORIZATION_TTL_MS = 30 * 60 * 1e3;
@@ -8012,6 +8051,19 @@ function normalizeIdentity(identity) {
8012
8051
  }
8013
8052
  return normalized;
8014
8053
  }
8054
+ async function resolveSchedulingUser(identity) {
8055
+ const user = await getUserByEmail(normalizeIdentity(identity));
8056
+ if (!user) throw new UnifiedBillingError("user_not_found", "billing identity was not found", 404);
8057
+ return user;
8058
+ }
8059
+ async function getSchedulingAccess(identity) {
8060
+ const user = await getUserByEmail(normalizeIdentity(identity));
8061
+ if (!user) return { allowed: false, subscriptionTier: null };
8062
+ return {
8063
+ allowed: hasPaidSchedulingAccess(user),
8064
+ subscriptionTier: user.subscription_tier
8065
+ };
8066
+ }
8015
8067
  function normalizeIdempotencyKey(value) {
8016
8068
  const key = value.trim();
8017
8069
  if (!key || key.length > 500) {
@@ -8093,8 +8145,10 @@ function mapEvent(row, balanceMc, duplicate) {
8093
8145
  async function authorizeScheduledRun(args) {
8094
8146
  const identity = normalizeIdentity(args.identity);
8095
8147
  const idempotencyKey = normalizeIdempotencyKey(args.idempotencyKey);
8096
- const user = await getUserByEmail(identity);
8097
- if (!user) throw new UnifiedBillingError("user_not_found", "billing identity was not found", 404);
8148
+ const user = await resolveSchedulingUser(identity);
8149
+ if (!hasPaidSchedulingAccess(user)) {
8150
+ throw new UnifiedBillingError(PAID_PLAN_REQUIRED_CODE, PAID_PLAN_REQUIRED_MESSAGE, 403);
8151
+ }
8098
8152
  const pending = await getDb().execute({
8099
8153
  sql: "SELECT id FROM billing_events WHERE user_id = ? AND billing_class = ? AND status = 'cost_pending' LIMIT 1",
8100
8154
  args: [user.id, SCHEDULED_RUN_BILLING_CLASS]
@@ -8356,9 +8410,187 @@ async function voidScheduledRunAuthorization(args) {
8356
8410
  return { ok: true, status: authorization.status };
8357
8411
  }
8358
8412
 
8413
+ // src/api/memory.ts
8414
+ import { createCipheriv, createDecipheriv, randomBytes, scryptSync } from "crypto";
8415
+
8416
+ // src/api/session.ts
8417
+ import { createHmac as createHmac2, timingSafeEqual as timingSafeEqual2 } from "crypto";
8418
+ var isProduction = () => process.env.NODE_ENV === "production" || process.env.VERCEL === "1";
8419
+ function getSessionSecret() {
8420
+ const configured = process.env.SESSION_SECRET?.trim();
8421
+ if (configured) return configured;
8422
+ if (isProduction()) throw new Error("SESSION_SECRET is not set \u2014 add it to your Vercel env vars (see src/api/env.ts)");
8423
+ return "dev-secret-change-me";
8424
+ }
8425
+ var secret = () => getSessionSecret();
8426
+ function safeEqualHex(a, b) {
8427
+ if (a.length !== b.length) return false;
8428
+ try {
8429
+ return timingSafeEqual2(Buffer.from(a, "hex"), Buffer.from(b, "hex"));
8430
+ } catch {
8431
+ return false;
8432
+ }
8433
+ }
8434
+ function signSession(userId) {
8435
+ const payload = String(userId);
8436
+ const sig = createHmac2("sha256", secret()).update(payload).digest("hex");
8437
+ return `${payload}.${sig}`;
8438
+ }
8439
+ function verifySession(token) {
8440
+ const dot = token.lastIndexOf(".");
8441
+ if (dot === -1) return null;
8442
+ const payload = token.slice(0, dot);
8443
+ const sig = token.slice(dot + 1);
8444
+ const expected = createHmac2("sha256", secret()).update(payload).digest("hex");
8445
+ if (!safeEqualHex(sig, expected)) return null;
8446
+ const id = parseInt(payload);
8447
+ return isNaN(id) ? null : id;
8448
+ }
8449
+
8450
+ // src/api/memory.ts
8451
+ import { provisionDefaultsTool } from "mcpscraper-memory-tools/tools/vaults/provision-defaults";
8452
+ import { setScheduleEntitlementTool } from "mcpscraper-memory-tools/tools/schedule/set-schedule-entitlement";
8453
+ var MEMORY_BASE_URL = () => (process.env.MCP_MEMORY_BASE_URL ?? "https://memory.mcpscraper.dev").replace(/\/$/, "");
8454
+ var ADMIN_KEY = () => process.env.MCP_MEMORY_ADMIN_KEY?.trim() ?? "";
8455
+ function parseMemoryRpcBody(text) {
8456
+ const dataLine = text.split("\n").find((line) => line.startsWith("data:"));
8457
+ const payload = dataLine ? dataLine.slice(5).trim() : text;
8458
+ return JSON.parse(payload);
8459
+ }
8460
+ function isMemoryOperator(email) {
8461
+ if (!email) return false;
8462
+ const ops = (process.env.MEMORY_OPERATOR_IDENTITIES ?? "").split(",").map((s) => s.trim().toLowerCase()).filter(Boolean);
8463
+ return ops.includes(email.trim().toLowerCase());
8464
+ }
8465
+ function encKey() {
8466
+ return scryptSync(getSessionSecret(), "mcp-memory-key-v1", 32);
8467
+ }
8468
+ function encryptMemoryKey(secret2) {
8469
+ const iv = randomBytes(12);
8470
+ const cipher = createCipheriv("aes-256-gcm", encKey(), iv);
8471
+ const enc = Buffer.concat([cipher.update(secret2, "utf8"), cipher.final()]);
8472
+ const tag = cipher.getAuthTag();
8473
+ return `${iv.toString("base64")}:${tag.toString("base64")}:${enc.toString("base64")}`;
8474
+ }
8475
+ function decryptMemoryKey(stored) {
8476
+ try {
8477
+ const [ivB, tagB, dataB] = stored.split(":");
8478
+ const decipher = createDecipheriv("aes-256-gcm", encKey(), Buffer.from(ivB, "base64"));
8479
+ decipher.setAuthTag(Buffer.from(tagB, "base64"));
8480
+ return Buffer.concat([decipher.update(Buffer.from(dataB, "base64")), decipher.final()]).toString("utf8");
8481
+ } catch {
8482
+ return null;
8483
+ }
8484
+ }
8485
+ async function memoryCall(toolName, args, userMemoryKey) {
8486
+ try {
8487
+ const res = await fetch(`${MEMORY_BASE_URL()}/mcp`, {
8488
+ method: "POST",
8489
+ headers: {
8490
+ "content-type": "application/json",
8491
+ accept: "application/json, text/event-stream",
8492
+ authorization: `Bearer ${userMemoryKey}`
8493
+ },
8494
+ body: JSON.stringify({
8495
+ jsonrpc: "2.0",
8496
+ id: 1,
8497
+ method: "tools/call",
8498
+ params: { name: toolName, arguments: args }
8499
+ })
8500
+ });
8501
+ const text = await res.text();
8502
+ if (!res.ok) return { ok: false, error: `memory ${toolName} failed (${res.status})` };
8503
+ const body = parseMemoryRpcBody(text);
8504
+ if (body.error) return { ok: false, error: body.error.message ?? `memory ${toolName} failed` };
8505
+ if (body.result?.structuredContent) return body.result.structuredContent;
8506
+ const textContent = body.result?.content?.find((c) => c.type === "text")?.text;
8507
+ if (textContent) {
8508
+ try {
8509
+ return JSON.parse(textContent);
8510
+ } catch {
8511
+ return { ok: false, error: textContent };
8512
+ }
8513
+ }
8514
+ return { ok: false, error: `memory ${toolName} returned no result` };
8515
+ } catch (err) {
8516
+ return { ok: false, error: err?.message ?? "memory call failed" };
8517
+ }
8518
+ }
8519
+ function memoryIdentity(user) {
8520
+ return user.email;
8521
+ }
8522
+ function resolveMemoryIdentity(user) {
8523
+ return memoryIdentity(user);
8524
+ }
8525
+ function memoryPlanForReads(user) {
8526
+ if (isMemoryOperator(user.email)) return "unlimited";
8527
+ if (user.memory_plan === "pro" || user.memory_plan === "team") return user.memory_plan;
8528
+ return "free";
8529
+ }
8530
+ async function provisionMemoryForUser(user) {
8531
+ try {
8532
+ const { key } = await getOrCreateUserMemoryKey(user);
8533
+ if (key) await setMemoryProvisioned(user.id, true);
8534
+ } catch {
8535
+ }
8536
+ }
8537
+ function resolveEffectiveMemoryPlan(user) {
8538
+ return user.memory_plan === "pro" || user.memory_plan === "team" ? user.memory_plan : "free";
8539
+ }
8540
+ async function getOrCreateUserMemoryKey(user) {
8541
+ const creds = await getUserMemoryCreds(user.id);
8542
+ if (creds.memory_key) {
8543
+ const decrypted = decryptMemoryKey(creds.memory_key);
8544
+ if (decrypted) return { key: decrypted };
8545
+ }
8546
+ const admin = ADMIN_KEY();
8547
+ if (!admin) return { key: null, error: "memory not configured (MCP_MEMORY_ADMIN_KEY missing)" };
8548
+ const identity = memoryIdentity(user);
8549
+ const plan = isMemoryOperator(user.email) ? "unlimited" : (await getMemoryPlan(user.id)).plan;
8550
+ const provisioned = await provisionDefaultsTool.execute(
8551
+ { apiKey: admin, granteeIdentity: identity, issueKey: true, plan },
8552
+ {}
8553
+ );
8554
+ if (!provisioned.ok || !provisioned.secret) {
8555
+ return { key: null, error: provisioned.error ?? "failed to provision default vaults" };
8556
+ }
8557
+ await setUserMemoryCreds(user.id, encryptMemoryKey(provisioned.secret), identity);
8558
+ return { key: provisioned.secret };
8559
+ }
8560
+ async function syncMemoryKeyPlan(user, plan) {
8561
+ const effectivePlan = isMemoryOperator(user.email) ? "unlimited" : plan;
8562
+ const { key, error } = await getOrCreateUserMemoryKey(user);
8563
+ if (!key) return { ok: false, error: error ?? "memory unavailable" };
8564
+ const listed = await memoryCall(
8565
+ "listKeysTool",
8566
+ {},
8567
+ key
8568
+ );
8569
+ if (!listed.ok || !listed.keys?.length) return { ok: false, error: listed.error ?? "no memory keys found" };
8570
+ const target = listed.keys[0];
8571
+ if (target.plan === effectivePlan) return { ok: true };
8572
+ const res = await memoryCall("setScopeTool", { keyId: target.keyId, plan: effectivePlan }, key);
8573
+ return { ok: res.ok, error: res.error };
8574
+ }
8575
+ async function syncScheduledActionCredentials(user) {
8576
+ const admin = ADMIN_KEY();
8577
+ if (!admin) return { ok: false, error: "memory not configured (MCP_MEMORY_ADMIN_KEY missing)" };
8578
+ const identity = resolveMemoryIdentity(user);
8579
+ const res = await setScheduleEntitlementTool.execute({
8580
+ apiKey: admin,
8581
+ granteeIdentity: identity,
8582
+ mcpScraperApiKey: user.api_key
8583
+ }, {});
8584
+ return { ok: res.ok, error: res.error };
8585
+ }
8586
+
8359
8587
  // src/api/internal-billing-routes.ts
8360
8588
  var internalBillingApp = new Hono3();
8361
8589
  var metadataSchema = z10.record(z10.unknown()).optional();
8590
+ var schedulingAccessSchema = z10.object({
8591
+ identity: z10.string().min(1).max(320),
8592
+ provisionCredentials: z10.boolean().optional()
8593
+ });
8362
8594
  var authorizeSchema = z10.object({
8363
8595
  identity: z10.string().min(1).max(320),
8364
8596
  idempotencyKey: z10.string().min(1).max(500),
@@ -8424,6 +8656,41 @@ internalBillingApp.post("/scheduled-run/authorize", async (c) => {
8424
8656
  return c.json(response.body, response.status);
8425
8657
  }
8426
8658
  });
8659
+ internalBillingApp.post("/scheduling-access", async (c) => {
8660
+ const parsed = schedulingAccessSchema.safeParse(await c.req.json().catch(() => null));
8661
+ if (!parsed.success) {
8662
+ return c.json({ ok: false, error_code: "invalid_request", error: "invalid scheduling access request" }, 400);
8663
+ }
8664
+ try {
8665
+ const access2 = await getSchedulingAccess(parsed.data.identity);
8666
+ if (access2.allowed && parsed.data.provisionCredentials) {
8667
+ const user = await getUserByEmail(parsed.data.identity.trim().toLowerCase());
8668
+ if (!user) {
8669
+ return c.json({ ok: false, error_code: "user_not_found", error: "billing identity was not found" }, 404);
8670
+ }
8671
+ const credentials = await syncScheduledActionCredentials(user);
8672
+ if (!credentials.ok) {
8673
+ return c.json({
8674
+ ok: false,
8675
+ error_code: "schedule_credentials_unavailable",
8676
+ error: credentials.error ?? "Scheduled Actions credentials are unavailable."
8677
+ }, 503);
8678
+ }
8679
+ }
8680
+ return c.json({
8681
+ ok: true,
8682
+ allowed: access2.allowed,
8683
+ subscriptionTier: access2.subscriptionTier,
8684
+ ...access2.allowed ? {} : {
8685
+ error_code: PAID_PLAN_REQUIRED_CODE,
8686
+ error: PAID_PLAN_REQUIRED_MESSAGE
8687
+ }
8688
+ });
8689
+ } catch (error) {
8690
+ const response = billingError(error);
8691
+ return c.json(response.body, response.status);
8692
+ }
8693
+ });
8427
8694
  internalBillingApp.post("/scheduled-run/settle", async (c) => {
8428
8695
  const parsed = settleSchema.safeParse(await c.req.json().catch(() => null));
8429
8696
  if (!parsed.success) {
@@ -12198,184 +12465,6 @@ redditApp.post("/thread", createApiKeyAuth(), async (c) => {
12198
12465
  // src/api/video-routes.ts
12199
12466
  import { Hono as Hono10 } from "hono";
12200
12467
  import { z as z18 } from "zod";
12201
-
12202
- // src/api/memory.ts
12203
- import { createCipheriv, createDecipheriv, randomBytes, scryptSync } from "crypto";
12204
-
12205
- // src/api/session.ts
12206
- import { createHmac as createHmac2, timingSafeEqual as timingSafeEqual2 } from "crypto";
12207
- var isProduction = () => process.env.NODE_ENV === "production" || process.env.VERCEL === "1";
12208
- function getSessionSecret() {
12209
- const configured = process.env.SESSION_SECRET?.trim();
12210
- if (configured) return configured;
12211
- if (isProduction()) throw new Error("SESSION_SECRET is not set \u2014 add it to your Vercel env vars (see src/api/env.ts)");
12212
- return "dev-secret-change-me";
12213
- }
12214
- var secret = () => getSessionSecret();
12215
- function safeEqualHex(a, b) {
12216
- if (a.length !== b.length) return false;
12217
- try {
12218
- return timingSafeEqual2(Buffer.from(a, "hex"), Buffer.from(b, "hex"));
12219
- } catch {
12220
- return false;
12221
- }
12222
- }
12223
- function signSession(userId) {
12224
- const payload = String(userId);
12225
- const sig = createHmac2("sha256", secret()).update(payload).digest("hex");
12226
- return `${payload}.${sig}`;
12227
- }
12228
- function verifySession(token) {
12229
- const dot = token.lastIndexOf(".");
12230
- if (dot === -1) return null;
12231
- const payload = token.slice(0, dot);
12232
- const sig = token.slice(dot + 1);
12233
- const expected = createHmac2("sha256", secret()).update(payload).digest("hex");
12234
- if (!safeEqualHex(sig, expected)) return null;
12235
- const id = parseInt(payload);
12236
- return isNaN(id) ? null : id;
12237
- }
12238
-
12239
- // src/api/memory.ts
12240
- import { provisionDefaultsTool } from "mcpscraper-memory-tools/tools/vaults/provision-defaults";
12241
- import { setScheduleEntitlementTool } from "mcpscraper-memory-tools/tools/schedule/set-schedule-entitlement";
12242
- var MEMORY_BASE_URL = () => (process.env.MCP_MEMORY_BASE_URL ?? "https://memory.mcpscraper.dev").replace(/\/$/, "");
12243
- var ADMIN_KEY = () => process.env.MCP_MEMORY_ADMIN_KEY?.trim() ?? "";
12244
- function parseMemoryRpcBody(text) {
12245
- const dataLine = text.split("\n").find((line) => line.startsWith("data:"));
12246
- const payload = dataLine ? dataLine.slice(5).trim() : text;
12247
- return JSON.parse(payload);
12248
- }
12249
- function isMemoryOperator(email) {
12250
- if (!email) return false;
12251
- const ops = (process.env.MEMORY_OPERATOR_IDENTITIES ?? "").split(",").map((s) => s.trim().toLowerCase()).filter(Boolean);
12252
- return ops.includes(email.trim().toLowerCase());
12253
- }
12254
- function encKey() {
12255
- return scryptSync(getSessionSecret(), "mcp-memory-key-v1", 32);
12256
- }
12257
- function encryptMemoryKey(secret2) {
12258
- const iv = randomBytes(12);
12259
- const cipher = createCipheriv("aes-256-gcm", encKey(), iv);
12260
- const enc = Buffer.concat([cipher.update(secret2, "utf8"), cipher.final()]);
12261
- const tag = cipher.getAuthTag();
12262
- return `${iv.toString("base64")}:${tag.toString("base64")}:${enc.toString("base64")}`;
12263
- }
12264
- function decryptMemoryKey(stored) {
12265
- try {
12266
- const [ivB, tagB, dataB] = stored.split(":");
12267
- const decipher = createDecipheriv("aes-256-gcm", encKey(), Buffer.from(ivB, "base64"));
12268
- decipher.setAuthTag(Buffer.from(tagB, "base64"));
12269
- return Buffer.concat([decipher.update(Buffer.from(dataB, "base64")), decipher.final()]).toString("utf8");
12270
- } catch {
12271
- return null;
12272
- }
12273
- }
12274
- async function memoryCall(toolName, args, userMemoryKey) {
12275
- try {
12276
- const res = await fetch(`${MEMORY_BASE_URL()}/mcp`, {
12277
- method: "POST",
12278
- headers: {
12279
- "content-type": "application/json",
12280
- accept: "application/json, text/event-stream",
12281
- authorization: `Bearer ${userMemoryKey}`
12282
- },
12283
- body: JSON.stringify({
12284
- jsonrpc: "2.0",
12285
- id: 1,
12286
- method: "tools/call",
12287
- params: { name: toolName, arguments: args }
12288
- })
12289
- });
12290
- const text = await res.text();
12291
- if (!res.ok) return { ok: false, error: `memory ${toolName} failed (${res.status})` };
12292
- const body = parseMemoryRpcBody(text);
12293
- if (body.error) return { ok: false, error: body.error.message ?? `memory ${toolName} failed` };
12294
- if (body.result?.structuredContent) return body.result.structuredContent;
12295
- const textContent = body.result?.content?.find((c) => c.type === "text")?.text;
12296
- if (textContent) {
12297
- try {
12298
- return JSON.parse(textContent);
12299
- } catch {
12300
- return { ok: false, error: textContent };
12301
- }
12302
- }
12303
- return { ok: false, error: `memory ${toolName} returned no result` };
12304
- } catch (err) {
12305
- return { ok: false, error: err?.message ?? "memory call failed" };
12306
- }
12307
- }
12308
- function memoryIdentity(user) {
12309
- return user.email;
12310
- }
12311
- function resolveMemoryIdentity(user) {
12312
- return memoryIdentity(user);
12313
- }
12314
- function memoryPlanForReads(user) {
12315
- if (isMemoryOperator(user.email)) return "unlimited";
12316
- if (user.memory_plan === "pro" || user.memory_plan === "team") return user.memory_plan;
12317
- return "free";
12318
- }
12319
- async function provisionMemoryForUser(user) {
12320
- try {
12321
- const { key } = await getOrCreateUserMemoryKey(user);
12322
- if (key) await setMemoryProvisioned(user.id, true);
12323
- } catch {
12324
- }
12325
- }
12326
- function resolveEffectiveMemoryPlan(user) {
12327
- return user.memory_plan === "pro" || user.memory_plan === "team" ? user.memory_plan : "free";
12328
- }
12329
- async function getOrCreateUserMemoryKey(user) {
12330
- const creds = await getUserMemoryCreds(user.id);
12331
- if (creds.memory_key) {
12332
- const decrypted = decryptMemoryKey(creds.memory_key);
12333
- if (decrypted) return { key: decrypted };
12334
- }
12335
- const admin = ADMIN_KEY();
12336
- if (!admin) return { key: null, error: "memory not configured (MCP_MEMORY_ADMIN_KEY missing)" };
12337
- const identity = memoryIdentity(user);
12338
- const plan = isMemoryOperator(user.email) ? "unlimited" : (await getMemoryPlan(user.id)).plan;
12339
- const provisioned = await provisionDefaultsTool.execute(
12340
- { apiKey: admin, granteeIdentity: identity, issueKey: true, plan },
12341
- {}
12342
- );
12343
- if (!provisioned.ok || !provisioned.secret) {
12344
- return { key: null, error: provisioned.error ?? "failed to provision default vaults" };
12345
- }
12346
- await setUserMemoryCreds(user.id, encryptMemoryKey(provisioned.secret), identity);
12347
- return { key: provisioned.secret };
12348
- }
12349
- async function syncMemoryKeyPlan(user, plan) {
12350
- const effectivePlan = isMemoryOperator(user.email) ? "unlimited" : plan;
12351
- const { key, error } = await getOrCreateUserMemoryKey(user);
12352
- if (!key) return { ok: false, error: error ?? "memory unavailable" };
12353
- const listed = await memoryCall(
12354
- "listKeysTool",
12355
- {},
12356
- key
12357
- );
12358
- if (!listed.ok || !listed.keys?.length) return { ok: false, error: listed.error ?? "no memory keys found" };
12359
- const target = listed.keys[0];
12360
- if (target.plan === effectivePlan) return { ok: true };
12361
- const res = await memoryCall("setScopeTool", { keyId: target.keyId, plan: effectivePlan }, key);
12362
- return { ok: res.ok, error: res.error };
12363
- }
12364
- async function syncScheduleEntitlement(user, enabled, quotaPerPeriod) {
12365
- const admin = ADMIN_KEY();
12366
- if (!admin) return { ok: false, error: "memory not configured (MCP_MEMORY_ADMIN_KEY missing)" };
12367
- const identity = resolveMemoryIdentity(user);
12368
- const res = await setScheduleEntitlementTool.execute({
12369
- apiKey: admin,
12370
- granteeIdentity: identity,
12371
- enabled,
12372
- quotaPerPeriod,
12373
- mcpScraperApiKey: user.api_key
12374
- }, {});
12375
- return { ok: res.ok, error: res.error };
12376
- }
12377
-
12378
- // src/api/video-routes.ts
12379
12468
  var videoApp = new Hono10();
12380
12469
  var AnalyzeBodySchema = z18.object({
12381
12470
  sourceUrl: z18.string().trim().url("sourceUrl must be a direct video file URL"),
@@ -15816,6 +15905,29 @@ var HybridMemoryToolExecutor = class {
15816
15905
  return this.mkKeyPromise;
15817
15906
  }
15818
15907
  async callMemoryTool(upstreamName, args) {
15908
+ const accessDenied = memoryScheduleAccessPayload(upstreamName, this.user);
15909
+ if (accessDenied) {
15910
+ return {
15911
+ content: [{ type: "text", text: JSON.stringify(accessDenied) }],
15912
+ structuredContent: accessDenied,
15913
+ isError: false
15914
+ };
15915
+ }
15916
+ if (memoryScheduleToolNeedsCredentials(upstreamName)) {
15917
+ const credentials = await syncScheduledActionCredentials(this.user);
15918
+ if (!credentials.ok) {
15919
+ const result = {
15920
+ ok: false,
15921
+ code: "schedule_credentials_unavailable",
15922
+ error: credentials.error ?? "Scheduled Actions credentials are unavailable."
15923
+ };
15924
+ return {
15925
+ content: [{ type: "text", text: JSON.stringify(result) }],
15926
+ structuredContent: result,
15927
+ isError: false
15928
+ };
15929
+ }
15930
+ }
15819
15931
  const handler = CUTOVER.get(upstreamName);
15820
15932
  if (!handler) return this.httpExecutor.callMemoryTool(upstreamName, args);
15821
15933
  const mkKey = await this.resolveMkKey();
@@ -18692,6 +18804,8 @@ stripeApp.post("/webhooks", async (c) => {
18692
18804
  if (user && invoice.id && !await ledgerExistsForStripePI(invoice.id)) {
18693
18805
  await creditMc(user.id, tier.credits_mc, LedgerOperation.SUBSCRIPTION, `${tier.label} subscription credits`, invoice.id);
18694
18806
  await setSubscriptionTier(user.id, tier.tier, tier.concurrency, invoice.subscription ?? user.subscription_id);
18807
+ const credentials = await syncScheduledActionCredentials(user);
18808
+ if (!credentials.ok) console.warn("[stripe] scheduled-action credential sync failed:", credentials.error ?? "unknown error");
18695
18809
  }
18696
18810
  }
18697
18811
  }
@@ -18709,15 +18823,6 @@ stripeApp.post("/webhooks", async (c) => {
18709
18823
  }
18710
18824
  return c.json({ received: true });
18711
18825
  }
18712
- if (priceId && priceId in SCHEDULING_PLANS) {
18713
- const sched = SCHEDULING_PLANS[priceId];
18714
- const user2 = await resolveUser(sub.customer, sub.customer_email);
18715
- if (user2) {
18716
- const live = sub.status === "active" || sub.status === "trialing";
18717
- await syncScheduleEntitlement(user2, live, sched.quota_per_period);
18718
- }
18719
- return c.json({ received: true });
18720
- }
18721
18826
  const user = await getUserByStripeCustomerId(sub.customer);
18722
18827
  if (!user) return c.json({ received: true });
18723
18828
  if (priceId && priceId in SUBSCRIPTION_TIERS) {
@@ -18725,6 +18830,10 @@ stripeApp.post("/webhooks", async (c) => {
18725
18830
  const live = sub.status === "active" || sub.status === "trialing";
18726
18831
  const newTier = live ? tier.tier : null;
18727
18832
  await setSubscriptionTier(user.id, newTier, live ? tier.concurrency : 0, live ? sub.id : null);
18833
+ if (live) {
18834
+ const credentials = await syncScheduledActionCredentials(user);
18835
+ if (!credentials.ok) console.warn("[stripe] scheduled-action credential sync failed:", credentials.error ?? "unknown error");
18836
+ }
18728
18837
  } else if (priceId === CONCURRENCY_PRICE_ID && event.type === "customer.subscription.created") {
18729
18838
  await setExtraConcurrencySlots(user.id, user.extra_concurrency_slots + 1);
18730
18839
  await setConcurrencySubId(user.id, sub.id);
@@ -18741,11 +18850,6 @@ stripeApp.post("/webhooks", async (c) => {
18741
18850
  }
18742
18851
  return c.json({ received: true });
18743
18852
  }
18744
- if (priceId && priceId in SCHEDULING_PLANS) {
18745
- const user2 = await getUserByStripeCustomerId(sub.customer);
18746
- if (user2) await syncScheduleEntitlement(user2, false);
18747
- return c.json({ received: true });
18748
- }
18749
18853
  const user = await getUserByStripeCustomerId(sub.customer);
18750
18854
  if (!user) return c.json({ received: true });
18751
18855
  if (priceId && priceId in SUBSCRIPTION_TIERS) {
@@ -22463,6 +22567,11 @@ var requireIntegrationsTier = createMiddleware3(async (c, next) => {
22463
22567
  }
22464
22568
  return next();
22465
22569
  });
22570
+ var requirePaidSchedulingTier = createMiddleware3(async (c, next) => {
22571
+ const user = c.get("user");
22572
+ if (!hasPaidSchedulingAccess(user)) return c.json(paidPlanRequiredPayload(), 403);
22573
+ return next();
22574
+ });
22466
22575
  var app = new Hono24();
22467
22576
  var STRIPE_API_VERSION = "2026-02-25.clover";
22468
22577
  function requireStripeSecret() {
@@ -22773,6 +22882,18 @@ app.post("/memory/mcp-call", auth2, async (c) => {
22773
22882
  const user = c.get("user");
22774
22883
  const body = await c.req.json().catch(() => ({}));
22775
22884
  if (!body.toolName) return c.json({ ok: false, error: "toolName is required" }, 400);
22885
+ const accessDenied = memoryScheduleAccessPayload(body.toolName, user);
22886
+ if (accessDenied) return c.json(accessDenied);
22887
+ if (memoryScheduleToolNeedsCredentials(body.toolName)) {
22888
+ const credentials = await syncScheduledActionCredentials(user);
22889
+ if (!credentials.ok) {
22890
+ return c.json({
22891
+ ok: false,
22892
+ code: "schedule_credentials_unavailable",
22893
+ error: credentials.error ?? "Scheduled Actions credentials are unavailable."
22894
+ }, 503);
22895
+ }
22896
+ }
22776
22897
  const { key, error } = await getOrCreateUserMemoryKey(user);
22777
22898
  if (!key) return c.json({ ok: false, error: error ?? "memory unavailable" }, 502);
22778
22899
  const result = await memoryCall(body.toolName, body.args ?? {}, key);
@@ -23078,7 +23199,7 @@ function renderResendOAuthResult(ok) {
23078
23199
  const message = ok ? "Your Resend account is connected privately to MCP Scraper. You can close this tab." : "The connection could not be completed. Close this tab and try again from Integrations.";
23079
23200
  return `<!doctype html><html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1"><meta name="referrer" content="no-referrer"><title>${title}</title><style>:root{color-scheme:light dark}*{box-sizing:border-box}body{margin:0;min-height:100vh;display:grid;place-items:center;padding:24px;background:#f5f2ee;color:#26211d;font-family:ui-sans-serif,system-ui,-apple-system,sans-serif}.card{width:min(100%,440px);padding:32px;border:1px solid #d8d0c8;border-radius:16px;background:#fff;box-shadow:0 18px 55px rgba(31,24,18,.1)}.mark{width:38px;height:38px;display:grid;place-items:center;border-radius:10px;background:#26211d;color:#fff;font-size:20px;font-weight:700}h1{margin:18px 0 8px;font-size:24px;line-height:1.15}p{margin:0;color:#71675f;line-height:1.55}a{display:inline-flex;margin-top:20px;color:#b44d36;font-weight:650;text-underline-offset:3px}@media(prefers-color-scheme:dark){body{background:#171411;color:#f4eee8}.card{background:#211d19;border-color:#3b342d}.mark{background:#f4eee8;color:#211d19}p{color:#b9aea5}}</style></head><body><main class="card"><div class="mark" aria-hidden="true">R</div><h1>${title}</h1><p>${message}</p><a href="/dashboard/integrations">Return to Integrations</a></main><script>if(window.opener){try{window.opener.postMessage({type:'mcp-scraper:resend-oauth',ok:${ok}},window.location.origin)}catch(e){}}setTimeout(function(){window.close()},1200)</script></body></html>`;
23080
23201
  }
23081
- app.get("/schedule-status", auth2, async (c) => {
23202
+ app.get("/schedule-status", auth2, requirePaidSchedulingTier, async (c) => {
23082
23203
  try {
23083
23204
  const user = c.get("user");
23084
23205
  const { key, error } = await getOrCreateUserMemoryKey(user);
@@ -23445,9 +23566,13 @@ app.post("/schedule-connections/actions/call", auth2, requireIntegrationsTier, a
23445
23566
  return scheduleConnectionError(c, err, "Unable to run this connection action.");
23446
23567
  }
23447
23568
  });
23448
- app.post("/schedule-link", auth2, async (c) => {
23569
+ app.post("/schedule-link", auth2, requirePaidSchedulingTier, async (c) => {
23449
23570
  try {
23450
23571
  const user = c.get("user");
23572
+ const credentials = await syncScheduledActionCredentials(user);
23573
+ if (!credentials.ok) {
23574
+ return c.json({ ok: false, code: "schedule_credentials_unavailable", error: credentials.error ?? "Scheduled Actions credentials are unavailable." }, 503);
23575
+ }
23451
23576
  const { key, error } = await getOrCreateUserMemoryKey(user);
23452
23577
  if (!key) return c.json({ error: error ?? "memory unavailable" }, 503);
23453
23578
  const r = await memoryCall("getScheduleLinkTool", {}, key);
@@ -23485,8 +23610,12 @@ app.get("/schedule-actions", auth2, async (c) => {
23485
23610
  });
23486
23611
  }
23487
23612
  });
23488
- app.post("/schedule-actions", auth2, async (c) => {
23613
+ app.post("/schedule-actions", auth2, requirePaidSchedulingTier, async (c) => {
23489
23614
  const user = c.get("user");
23615
+ const credentials = await syncScheduledActionCredentials(user);
23616
+ if (!credentials.ok) {
23617
+ return c.json({ ok: false, code: "schedule_credentials_unavailable", error: credentials.error ?? "Scheduled Actions credentials are unavailable." }, 503);
23618
+ }
23490
23619
  const { key, error } = await getOrCreateUserMemoryKey(user);
23491
23620
  if (!key) return c.json({ ok: false, error: error ?? "memory unavailable" }, 503);
23492
23621
  const body = await c.req.json().catch(() => ({}));
@@ -23578,7 +23707,7 @@ app.post("/schedule-actions/:id/pause", auth2, async (c) => {
23578
23707
  const r = await memoryCall("pauseScheduledActionTool", { id: c.req.param("id") }, key);
23579
23708
  return c.json(r);
23580
23709
  });
23581
- app.post("/schedule-actions/:id/resume", auth2, async (c) => {
23710
+ app.post("/schedule-actions/:id/resume", auth2, requirePaidSchedulingTier, async (c) => {
23582
23711
  const user = c.get("user");
23583
23712
  const { key, error } = await getOrCreateUserMemoryKey(user);
23584
23713
  if (!key) return c.json({ ok: false, error: error ?? "memory unavailable" }, 503);
@@ -24461,7 +24590,7 @@ app.get("/cron/tick", async (c) => {
24461
24590
  if (!process.env.CRON_SECRET || secret2 !== `Bearer ${process.env.CRON_SECRET}`) {
24462
24591
  return c.json({ error: "Unauthorized" }, 401);
24463
24592
  }
24464
- const { drainQueue } = await import("./worker-45WEWCMQ.js");
24593
+ const { drainQueue } = await import("./worker-GE5JMHE6.js");
24465
24594
  const budget = { maxJobs: 10, deadlineMs: Date.now() + 28e4 };
24466
24595
  const workflowDispatchResult = await dispatchDueWorkflowSchedules(`${new URL(c.req.url).protocol}//${new URL(c.req.url).host}`);
24467
24596
  const [results, sweepResult, reapResult, expiredResult, blobCleanup, connectedDataArtifactCleanup] = await Promise.all([
@@ -24489,7 +24618,7 @@ app.post("/api/internal/extract-refinalize/:id", async (c) => {
24489
24618
  return c.json({ error: "Unauthorized" }, 401);
24490
24619
  }
24491
24620
  const jobId = c.req.param("id");
24492
- const { getExtractJob: getExtractJob2, completeExtractJob: completeExtractJob2 } = await import("./site-extract-repository-YNPXEZCB.js");
24621
+ const { getExtractJob: getExtractJob2, completeExtractJob: completeExtractJob2 } = await import("./site-extract-repository-5V5EEQ72.js");
24493
24622
  const { assembleExtractArtifacts } = await import("./extract-bundle-HMCYUVBJ.js");
24494
24623
  const job = await getExtractJob2(jobId);
24495
24624
  if (!job) return c.json({ error: "job not found" }, 404);
@@ -24497,7 +24626,7 @@ app.post("/api/internal/extract-refinalize/:id", async (c) => {
24497
24626
  await completeExtractJob2(jobId, stored);
24498
24627
  let settlement = "already_settled_or_refunded";
24499
24628
  if (job.billedMc == null && job.userId != null) {
24500
- const { settleExtractJob: settleExtractJob2, countSuccessfulPages: countSuccessfulPages2 } = await import("./site-extract-repository-YNPXEZCB.js");
24629
+ const { settleExtractJob: settleExtractJob2, countSuccessfulPages: countSuccessfulPages2 } = await import("./site-extract-repository-5V5EEQ72.js");
24501
24630
  const heldMc = Number(job.options.heldMc ?? 0);
24502
24631
  const successful = await countSuccessfulPages2(jobId);
24503
24632
  const usedMc = Math.min(successful * MC_COSTS.page_scrape, heldMc);
@@ -24663,4 +24792,4 @@ app.get("/blog/:slug/", (c) => {
24663
24792
  export {
24664
24793
  app
24665
24794
  };
24666
- //# sourceMappingURL=server-PH3VJ74X.js.map
24795
+ //# sourceMappingURL=server-CHTX6YQA.js.map