mcp-scraper 0.2.13 → 0.2.15

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (47) hide show
  1. package/README.md +12 -2
  2. package/dist/bin/api-server.cjs +977 -200
  3. package/dist/bin/api-server.cjs.map +1 -1
  4. package/dist/bin/api-server.js +3 -3
  5. package/dist/bin/browser-agent-stdio-server.cjs +1 -1
  6. package/dist/bin/browser-agent-stdio-server.cjs.map +1 -1
  7. package/dist/bin/browser-agent-stdio-server.js +2 -2
  8. package/dist/bin/mcp-scraper-cli.cjs +46 -3
  9. package/dist/bin/mcp-scraper-cli.cjs.map +1 -1
  10. package/dist/bin/mcp-scraper-cli.js +42 -2
  11. package/dist/bin/mcp-scraper-cli.js.map +1 -1
  12. package/dist/bin/mcp-scraper-combined-stdio-server.cjs +135 -5
  13. package/dist/bin/mcp-scraper-combined-stdio-server.cjs.map +1 -1
  14. package/dist/bin/mcp-scraper-combined-stdio-server.js +3 -3
  15. package/dist/bin/mcp-scraper-install.cjs +4 -3
  16. package/dist/bin/mcp-scraper-install.cjs.map +1 -1
  17. package/dist/bin/mcp-scraper-install.js +4 -3
  18. package/dist/bin/mcp-scraper-install.js.map +1 -1
  19. package/dist/bin/mcp-stdio-server.cjs +135 -5
  20. package/dist/bin/mcp-stdio-server.cjs.map +1 -1
  21. package/dist/bin/mcp-stdio-server.js +2 -2
  22. package/dist/chunk-2CQXHSWC.js +7 -0
  23. package/dist/chunk-2CQXHSWC.js.map +1 -0
  24. package/dist/{chunk-ROS67BNV.js → chunk-BSYPATSM.js} +21 -2
  25. package/dist/chunk-BSYPATSM.js.map +1 -0
  26. package/dist/{chunk-AYTOCZBS.js → chunk-F44RBOJ5.js} +6 -3
  27. package/dist/chunk-F44RBOJ5.js.map +1 -0
  28. package/dist/{chunk-DKJ2XCY7.js → chunk-Q4DFONIK.js} +136 -6
  29. package/dist/chunk-Q4DFONIK.js.map +1 -0
  30. package/dist/{chunk-BQTWXY6G.js → chunk-TIPUIEJN.js} +2 -2
  31. package/dist/{chunk-7GCCOT3M.js → chunk-TL7YTFLH.js} +15 -1
  32. package/dist/chunk-TL7YTFLH.js.map +1 -0
  33. package/dist/{db-BVHYI57K.js → db-P5X6UQ3E.js} +2 -2
  34. package/dist/{server-LUNOI26E.js → server-VD2TD3AD.js} +773 -184
  35. package/dist/server-VD2TD3AD.js.map +1 -0
  36. package/dist/{worker-TDJQ6TH3.js → worker-OZSWIS3F.js} +3 -3
  37. package/package.json +1 -1
  38. package/dist/chunk-7GCCOT3M.js.map +0 -1
  39. package/dist/chunk-AYTOCZBS.js.map +0 -1
  40. package/dist/chunk-DKJ2XCY7.js.map +0 -1
  41. package/dist/chunk-RDROUQ4E.js +0 -7
  42. package/dist/chunk-RDROUQ4E.js.map +0 -1
  43. package/dist/chunk-ROS67BNV.js.map +0 -1
  44. package/dist/server-LUNOI26E.js.map +0 -1
  45. /package/dist/{chunk-BQTWXY6G.js.map → chunk-TIPUIEJN.js.map} +0 -0
  46. /package/dist/{db-BVHYI57K.js.map → db-P5X6UQ3E.js.map} +0 -0
  47. /package/dist/{worker-TDJQ6TH3.js.map → worker-OZSWIS3F.js.map} +0 -0
@@ -7099,6 +7099,20 @@ async function migrate() {
7099
7099
  await db.execute(`ALTER TABLE users ADD COLUMN concurrency_stripe_sub_id TEXT`);
7100
7100
  } catch {
7101
7101
  }
7102
+ await db.execute(`
7103
+ CREATE TABLE IF NOT EXISTS concurrency_locks (
7104
+ id TEXT PRIMARY KEY,
7105
+ user_id INTEGER NOT NULL REFERENCES users(id),
7106
+ operation TEXT NOT NULL,
7107
+ status TEXT NOT NULL DEFAULT 'active',
7108
+ acquired_at TEXT NOT NULL DEFAULT (datetime('now')),
7109
+ expires_at TEXT NOT NULL,
7110
+ released_at TEXT,
7111
+ metadata TEXT
7112
+ )
7113
+ `);
7114
+ await db.execute(`CREATE INDEX IF NOT EXISTS concurrency_locks_user_status ON concurrency_locks(user_id, status, expires_at)`);
7115
+ await db.execute(`CREATE INDEX IF NOT EXISTS concurrency_locks_operation ON concurrency_locks(operation, status, expires_at)`);
7102
7116
  await db.execute(`
7103
7117
  CREATE TABLE IF NOT EXISTS ledger (
7104
7118
  id INTEGER PRIMARY KEY AUTOINCREMENT,
@@ -8672,6 +8686,19 @@ var init_site_audit_routes = __esm({
8672
8686
  function browserActiveCostMc(activeMs) {
8673
8687
  return Math.round(activeMs * MC_PER_BROWSER_MS);
8674
8688
  }
8689
+ function concurrencySlotBillingInfo() {
8690
+ const billingUrl = process.env.CONCURRENCY_BILLING_URL ?? "https://mcpscraper.dev/billing";
8691
+ return {
8692
+ product: "Extra concurrency slot",
8693
+ price_label: CONCURRENCY_SLOT_PRICE_LABEL,
8694
+ unit_amount_usd: CONCURRENCY_SLOT_PRICE_USD,
8695
+ currency: CONCURRENCY_SLOT_PRICE_CURRENCY,
8696
+ interval: CONCURRENCY_SLOT_PRICE_INTERVAL,
8697
+ billing_url: billingUrl,
8698
+ terminal_command: CONCURRENCY_SLOT_TERMINAL_COMMAND,
8699
+ terminal_command_with_api_key_env: `MCP_SCRAPER_API_KEY=sk_live_your_key ${CONCURRENCY_SLOT_TERMINAL_COMMAND}`
8700
+ };
8701
+ }
8675
8702
  function mcToCredits(mc) {
8676
8703
  return mc / MC_PER_CREDIT;
8677
8704
  }
@@ -8688,7 +8715,7 @@ function insufficientBalanceResponse(balanceMc, requiredMc) {
8688
8715
  topup_url: topupUrl
8689
8716
  };
8690
8717
  }
8691
- var MC_COSTS, MC_PER_BROWSER_MS, BROWSER_OPEN_MIN_BALANCE_MC, MC_PER_CREDIT, CREDIT_COST_CATALOG, CONCURRENCY_PRICE_ID, FREE_SIGNUP_MC, FREE_MONTHLY_REFRESH_MC, BALANCE_PRICE_IDS, BALANCE_PACK_LABELS, LedgerOperation;
8718
+ var MC_COSTS, MC_PER_BROWSER_MS, BROWSER_OPEN_MIN_BALANCE_MC, MC_PER_CREDIT, CREDIT_COST_CATALOG, CONCURRENCY_PRICE_ID, CONCURRENCY_SLOT_PRICE_USD, CONCURRENCY_SLOT_PRICE_CURRENCY, CONCURRENCY_SLOT_PRICE_INTERVAL, CONCURRENCY_SLOT_PRICE_LABEL, CONCURRENCY_SLOT_TERMINAL_COMMAND, FREE_SIGNUP_MC, FREE_MONTHLY_REFRESH_MC, BALANCE_PRICE_IDS, BALANCE_PACK_LABELS, LedgerOperation;
8692
8719
  var init_rates = __esm({
8693
8720
  "src/api/rates.ts"() {
8694
8721
  "use strict";
@@ -8815,6 +8842,11 @@ var init_rates = __esm({
8815
8842
  }
8816
8843
  ];
8817
8844
  CONCURRENCY_PRICE_ID = "price_1Ta1NRS8aAcsk3TGwsRnYbix";
8845
+ CONCURRENCY_SLOT_PRICE_USD = 5;
8846
+ CONCURRENCY_SLOT_PRICE_CURRENCY = "usd";
8847
+ CONCURRENCY_SLOT_PRICE_INTERVAL = "month";
8848
+ CONCURRENCY_SLOT_PRICE_LABEL = "$5/month";
8849
+ CONCURRENCY_SLOT_TERMINAL_COMMAND = "npx -y -p mcp-scraper@latest mcp-scraper-cli billing concurrency checkout";
8818
8850
  FREE_SIGNUP_MC = 5e5;
8819
8851
  FREE_MONTHLY_REFRESH_MC = 25e4;
8820
8852
  BALANCE_PRICE_IDS = {
@@ -10383,6 +10415,187 @@ var init_server_schemas = __esm({
10383
10415
  }
10384
10416
  });
10385
10417
 
10418
+ // src/api/concurrency-gates.ts
10419
+ function concurrencyLimitForUser(user) {
10420
+ return 1 + Math.max(0, Number(user.extra_concurrency_slots ?? 0));
10421
+ }
10422
+ function concurrencyLimitExceededResponse(gate) {
10423
+ const upgrade = concurrencySlotBillingInfo();
10424
+ const message = `You have ${gate.active} active operation${gate.active !== 1 ? "s" : ""}. Your account allows ${gate.limit} concurrent operation${gate.limit !== 1 ? "s" : ""}. Wait for one to finish, or add an extra concurrency slot for ${upgrade.price_label} at ${upgrade.billing_url}. Terminal upgrade: ${upgrade.terminal_command}`;
10425
+ return {
10426
+ error: message,
10427
+ message,
10428
+ error_code: "concurrency_limit_exceeded",
10429
+ error_type: "concurrency_limit",
10430
+ retryable: true,
10431
+ active: gate.active,
10432
+ limit: gate.limit,
10433
+ operation: gate.operation,
10434
+ upgrade,
10435
+ upgrade_url: upgrade.billing_url,
10436
+ upgrade_command: upgrade.terminal_command
10437
+ };
10438
+ }
10439
+ function isConcurrencyLimitBypassed(user) {
10440
+ const raw = process.env.CONCURRENCY_LIMIT_BYPASS_EMAILS ?? process.env.HARVEST_LIMIT_BYPASS_EMAILS ?? "";
10441
+ return raw.split(",").map((email) => email.trim()).filter(Boolean).includes(user.email);
10442
+ }
10443
+ function lockTtlModifier(ttlSeconds) {
10444
+ const ttl = Number.isFinite(ttlSeconds) ? Math.max(30, Math.min(Math.trunc(Number(ttlSeconds)), MAX_LOCK_TTL_SECONDS)) : DEFAULT_LOCK_TTL_SECONDS;
10445
+ return `+${ttl} seconds`;
10446
+ }
10447
+ function safeMetadata(metadata) {
10448
+ if (!metadata) return null;
10449
+ try {
10450
+ return JSON.stringify(metadata).slice(0, 4e3);
10451
+ } catch {
10452
+ return null;
10453
+ }
10454
+ }
10455
+ async function ensureConcurrencyGateSchema() {
10456
+ if (!schemaReady) {
10457
+ schemaReady = (async () => {
10458
+ const db = getDb();
10459
+ await db.execute(`
10460
+ CREATE TABLE IF NOT EXISTS concurrency_locks (
10461
+ id TEXT PRIMARY KEY,
10462
+ user_id INTEGER NOT NULL REFERENCES users(id),
10463
+ operation TEXT NOT NULL,
10464
+ status TEXT NOT NULL DEFAULT 'active',
10465
+ acquired_at TEXT NOT NULL DEFAULT (datetime('now')),
10466
+ expires_at TEXT NOT NULL,
10467
+ released_at TEXT,
10468
+ metadata TEXT
10469
+ )
10470
+ `);
10471
+ await db.execute(`CREATE INDEX IF NOT EXISTS concurrency_locks_user_status ON concurrency_locks(user_id, status, expires_at)`);
10472
+ await db.execute(`CREATE INDEX IF NOT EXISTS concurrency_locks_operation ON concurrency_locks(operation, status, expires_at)`);
10473
+ })();
10474
+ }
10475
+ return schemaReady;
10476
+ }
10477
+ async function expireConcurrencyLocksForUser(userId) {
10478
+ await ensureConcurrencyGateSchema();
10479
+ await getDb().execute({
10480
+ sql: `UPDATE concurrency_locks
10481
+ SET status = 'expired', released_at = datetime('now')
10482
+ WHERE user_id = ? AND status = 'active' AND expires_at <= datetime('now')`,
10483
+ args: [userId]
10484
+ });
10485
+ }
10486
+ async function countActiveUsageForUser(userId) {
10487
+ await ensureConcurrencyGateSchema();
10488
+ await expireConcurrencyLocksForUser(userId);
10489
+ const res = await getDb().execute({
10490
+ sql: `SELECT
10491
+ (
10492
+ SELECT COUNT(*)
10493
+ FROM jobs
10494
+ WHERE user_id = ?
10495
+ AND (status = 'pending' OR (status = 'running' AND started_at > datetime('now', '-10 minutes')))
10496
+ ) +
10497
+ (
10498
+ SELECT COUNT(*)
10499
+ FROM concurrency_locks
10500
+ WHERE user_id = ? AND status = 'active' AND expires_at > datetime('now')
10501
+ ) AS n`,
10502
+ args: [userId, userId]
10503
+ });
10504
+ return Number(res.rows[0]?.n ?? 0);
10505
+ }
10506
+ async function reuseExistingConcurrencyGate(userId, lockId, ttlSeconds) {
10507
+ if (!/^cl_[a-f0-9]{24,32}$/.test(lockId)) return false;
10508
+ await ensureConcurrencyGateSchema();
10509
+ await expireConcurrencyLocksForUser(userId);
10510
+ const res = await getDb().execute({
10511
+ sql: `UPDATE concurrency_locks
10512
+ SET expires_at = datetime('now', ?)
10513
+ WHERE id = ?
10514
+ AND user_id = ?
10515
+ AND status = 'active'
10516
+ AND expires_at > datetime('now')`,
10517
+ args: [lockTtlModifier(ttlSeconds), lockId, userId]
10518
+ });
10519
+ return Number(res.rowsAffected ?? 0) > 0;
10520
+ }
10521
+ async function acquireConcurrencyGate(user, operation, options = {}) {
10522
+ await ensureConcurrencyGateSchema();
10523
+ const limit = concurrencyLimitForUser(user);
10524
+ if (isConcurrencyLimitBypassed(user)) {
10525
+ return { ok: true, lockId: null, active: 0, limit, operation, bypassed: true };
10526
+ }
10527
+ if (options.reuseLockId && await reuseExistingConcurrencyGate(user.id, options.reuseLockId, options.ttlSeconds)) {
10528
+ return { ok: true, lockId: null, active: await countActiveUsageForUser(user.id), limit, operation, reused: true };
10529
+ }
10530
+ await expireConcurrencyLocksForUser(user.id);
10531
+ const lockId = `cl_${(0, import_node_crypto2.randomUUID)().replace(/-/g, "").slice(0, 24)}`;
10532
+ const res = await getDb().execute({
10533
+ sql: `INSERT INTO concurrency_locks (id, user_id, operation, status, expires_at, metadata)
10534
+ SELECT ?, ?, ?, 'active', datetime('now', ?), ?
10535
+ WHERE (
10536
+ (
10537
+ SELECT COUNT(*)
10538
+ FROM jobs
10539
+ WHERE user_id = ?
10540
+ AND (status = 'pending' OR (status = 'running' AND started_at > datetime('now', '-10 minutes')))
10541
+ ) +
10542
+ (
10543
+ SELECT COUNT(*)
10544
+ FROM concurrency_locks
10545
+ WHERE user_id = ? AND status = 'active' AND expires_at > datetime('now')
10546
+ )
10547
+ ) < ?`,
10548
+ args: [
10549
+ lockId,
10550
+ user.id,
10551
+ operation,
10552
+ lockTtlModifier(options.ttlSeconds),
10553
+ safeMetadata(options.metadata),
10554
+ user.id,
10555
+ user.id,
10556
+ limit
10557
+ ]
10558
+ });
10559
+ const active = await countActiveUsageForUser(user.id);
10560
+ if (Number(res.rowsAffected ?? 0) > 0) {
10561
+ return { ok: true, lockId, active, limit, operation };
10562
+ }
10563
+ return { ok: false, active, limit, operation, retryAfterSeconds: DEFAULT_RETRY_AFTER_SECONDS };
10564
+ }
10565
+ async function releaseConcurrencyGate(lockId) {
10566
+ if (!lockId) return;
10567
+ await ensureConcurrencyGateSchema();
10568
+ await getDb().execute({
10569
+ sql: `UPDATE concurrency_locks
10570
+ SET status = 'released', released_at = datetime('now')
10571
+ WHERE id = ? AND status = 'active'`,
10572
+ args: [lockId]
10573
+ });
10574
+ }
10575
+ async function extendConcurrencyGate(lockId, ttlSeconds = DEFAULT_LOCK_TTL_SECONDS) {
10576
+ if (!lockId) return;
10577
+ await ensureConcurrencyGateSchema();
10578
+ await getDb().execute({
10579
+ sql: `UPDATE concurrency_locks
10580
+ SET expires_at = datetime('now', ?)
10581
+ WHERE id = ? AND status = 'active' AND expires_at > datetime('now')`,
10582
+ args: [lockTtlModifier(ttlSeconds), lockId]
10583
+ });
10584
+ }
10585
+ var import_node_crypto2, DEFAULT_LOCK_TTL_SECONDS, DEFAULT_RETRY_AFTER_SECONDS, MAX_LOCK_TTL_SECONDS, schemaReady;
10586
+ var init_concurrency_gates = __esm({
10587
+ "src/api/concurrency-gates.ts"() {
10588
+ "use strict";
10589
+ import_node_crypto2 = require("crypto");
10590
+ init_db();
10591
+ init_rates();
10592
+ DEFAULT_LOCK_TTL_SECONDS = 15 * 60;
10593
+ DEFAULT_RETRY_AFTER_SECONDS = 30;
10594
+ MAX_LOCK_TTL_SECONDS = 24 * 60 * 60;
10595
+ schemaReady = null;
10596
+ }
10597
+ });
10598
+
10386
10599
  // src/api/youtube-routes.ts
10387
10600
  function buildTranscriptMarkdown(result) {
10388
10601
  const lines = [];
@@ -10444,6 +10657,7 @@ var init_youtube_routes = __esm({
10444
10657
  init_errors();
10445
10658
  init_api_auth();
10446
10659
  init_server_schemas();
10660
+ init_concurrency_gates();
10447
10661
  youtubeApp = new import_hono2.Hono();
10448
10662
  youtubeApp.post("/harvest", createApiKeyAuth(), async (c) => {
10449
10663
  const raw = await c.req.json().catch(() => ({}));
@@ -10457,9 +10671,16 @@ var init_youtube_routes = __esm({
10457
10671
  return c.json({ error: "channel mode requires channelHandle (@handle or UC... ID)" }, 400);
10458
10672
  }
10459
10673
  const user = c.get("user");
10460
- const { ok: ytOk, balance_mc: ytBal } = await debitMc(user.id, MC_COSTS.yt_channel, LedgerOperation.YT_CHANNEL, query ?? channelHandle ?? "");
10461
- if (!ytOk) return c.json(insufficientBalanceResponse(ytBal, MC_COSTS.yt_channel), 402);
10674
+ const gate = await acquireConcurrencyGate(user, "youtube_harvest", {
10675
+ reuseLockId: c.req.header("x-mcp-scraper-concurrency-lock"),
10676
+ metadata: { mode, query, channelHandle }
10677
+ });
10678
+ if (!gate.ok) return c.json(concurrencyLimitExceededResponse(gate), 429, { "Retry-After": String(gate.retryAfterSeconds) });
10679
+ let debited = false;
10462
10680
  try {
10681
+ const { ok: ytOk, balance_mc: ytBal } = await debitMc(user.id, MC_COSTS.yt_channel, LedgerOperation.YT_CHANNEL, query ?? channelHandle ?? "");
10682
+ if (!ytOk) return c.json(insufficientBalanceResponse(ytBal, MC_COSTS.yt_channel), 402);
10683
+ debited = true;
10463
10684
  const normalizedChannelUrl = mode === "channel" ? buildYouTubeChannelVideosUrl(channelHandle.trim()) : void 0;
10464
10685
  const result = await ytHarvest({
10465
10686
  mode,
@@ -10484,13 +10705,15 @@ var init_youtube_routes = __esm({
10484
10705
  return c.json({ error: msg }, 400);
10485
10706
  }
10486
10707
  if (err instanceof CaptchaError || msg.includes("CAPTCHA") || msg.includes("blocked")) {
10487
- await creditMc(user.id, MC_COSTS.yt_channel, LedgerOperation.YT_CHANNEL_REFUND, "failed call");
10708
+ if (debited) await creditMc(user.id, MC_COSTS.yt_channel, LedgerOperation.YT_CHANNEL_REFUND, "failed call");
10488
10709
  await logRequestEvent({ userId: user.id, source: "youtube_harvest", status: "failed", query: mode === "search" ? query?.trim() ?? "" : channelHandle?.trim() ?? "", error: msg });
10489
10710
  return c.json({ error: msg }, 503);
10490
10711
  }
10491
- await creditMc(user.id, MC_COSTS.yt_channel, LedgerOperation.YT_CHANNEL_REFUND, "failed call");
10712
+ if (debited) await creditMc(user.id, MC_COSTS.yt_channel, LedgerOperation.YT_CHANNEL_REFUND, "failed call");
10492
10713
  await logRequestEvent({ userId: user.id, source: "youtube_harvest", status: "failed", query: mode === "search" ? query?.trim() ?? "" : channelHandle?.trim() ?? "", error: msg });
10493
10714
  return c.json({ error: msg }, 500);
10715
+ } finally {
10716
+ await releaseConcurrencyGate(gate.lockId);
10494
10717
  }
10495
10718
  });
10496
10719
  youtubeApp.post("/transcribe", createApiKeyAuth(), async (c) => {
@@ -10502,9 +10725,16 @@ var init_youtube_routes = __esm({
10502
10725
  const user = c.get("user");
10503
10726
  const holdMins = 5;
10504
10727
  const holdMc = MC_COSTS.yt_transcription * holdMins;
10505
- const { ok: trOk, balance_mc: trBal } = await debitMc(user.id, holdMc, LedgerOperation.TRANSCRIPTION_HOLD, id);
10506
- if (!trOk) return c.json(insufficientBalanceResponse(trBal, holdMc), 402);
10728
+ const gate = await acquireConcurrencyGate(user, "youtube_transcribe", {
10729
+ reuseLockId: c.req.header("x-mcp-scraper-concurrency-lock"),
10730
+ metadata: { videoId: id }
10731
+ });
10732
+ if (!gate.ok) return c.json(concurrencyLimitExceededResponse(gate), 429, { "Retry-After": String(gate.retryAfterSeconds) });
10733
+ let debited = false;
10507
10734
  try {
10735
+ const { ok: trOk, balance_mc: trBal } = await debitMc(user.id, holdMc, LedgerOperation.TRANSCRIPTION_HOLD, id);
10736
+ if (!trOk) return c.json(insufficientBalanceResponse(trBal, holdMc), 402);
10737
+ debited = true;
10508
10738
  const result = await fetchCaptions(id);
10509
10739
  const lastChunk = result.chunks?.[result.chunks.length - 1];
10510
10740
  const chunkSecs = lastChunk && Number.isFinite(lastChunk.timestamp[1]) ? lastChunk.timestamp[1] : 0;
@@ -10529,10 +10759,12 @@ var init_youtube_routes = __esm({
10529
10759
  html: buildTranscriptHtml(result)
10530
10760
  });
10531
10761
  } catch (err) {
10532
- await creditMc(user.id, holdMc, LedgerOperation.TRANSCRIPTION_REFUND, "failed call");
10762
+ if (debited) await creditMc(user.id, holdMc, LedgerOperation.TRANSCRIPTION_REFUND, "failed call");
10533
10763
  const msg = err instanceof Error ? err.message : String(err);
10534
10764
  await logRequestEvent({ userId: user.id, source: "youtube_transcribe", status: "failed", query: id, error: msg });
10535
10765
  return c.json({ error: msg }, 500);
10766
+ } finally {
10767
+ await releaseConcurrencyGate(gate.lockId);
10536
10768
  }
10537
10769
  });
10538
10770
  }
@@ -10549,6 +10781,7 @@ var init_screenshot_routes = __esm({
10549
10781
  init_screenshot();
10550
10782
  init_api_auth();
10551
10783
  init_url_utils();
10784
+ init_concurrency_gates();
10552
10785
  ScreenshotBodySchema = import_zod14.z.object({
10553
10786
  url: import_zod14.z.string().trim().min(1, "url is required"),
10554
10787
  device: import_zod14.z.string().trim().optional(),
@@ -10564,6 +10797,7 @@ var init_screenshot_routes = __esm({
10564
10797
  }
10565
10798
  const body = parsed.data;
10566
10799
  const urlCheck = await validatePublicHttpUrl(body.url, { field: "url", requireHttps: false });
10800
+ let targetUrl = urlCheck.parsed?.href;
10567
10801
  if (urlCheck.error) {
10568
10802
  if (!body.allowLocal) {
10569
10803
  return c.json({ error_code: "invalid_request", message: urlCheck.error }, 400);
@@ -10577,25 +10811,17 @@ var init_screenshot_routes = __esm({
10577
10811
  if (parsedFallback.protocol !== "http:" && parsedFallback.protocol !== "https:") {
10578
10812
  return c.json({ error: "url must use http or https" }, 400);
10579
10813
  }
10580
- const device2 = body.device === "mobile" ? "mobile" : "desktop";
10581
- try {
10582
- const buf = await captureScreenshot(parsedFallback.href, browserServiceApiKey(), device2);
10583
- return new Response(new Uint8Array(buf), {
10584
- status: 200,
10585
- headers: {
10586
- "Content-Type": "image/png",
10587
- "Content-Length": String(buf.length),
10588
- "Cache-Control": "no-store"
10589
- }
10590
- });
10591
- } catch (err) {
10592
- const msg = err instanceof Error ? err.message : String(err);
10593
- return c.json({ error: msg }, 502);
10594
- }
10814
+ targetUrl = parsedFallback.href;
10595
10815
  }
10596
10816
  const device = body.device === "mobile" ? "mobile" : "desktop";
10817
+ const user = c.get("user");
10818
+ const gate = await acquireConcurrencyGate(user, "screenshot", {
10819
+ reuseLockId: c.req.header("x-mcp-scraper-concurrency-lock"),
10820
+ metadata: { url: targetUrl, device }
10821
+ });
10822
+ if (!gate.ok) return c.json(concurrencyLimitExceededResponse(gate), 429, { "Retry-After": String(gate.retryAfterSeconds) });
10597
10823
  try {
10598
- const buf = await captureScreenshot(urlCheck.parsed.href, browserServiceApiKey(), device);
10824
+ const buf = await captureScreenshot(targetUrl, browserServiceApiKey(), device);
10599
10825
  return new Response(new Uint8Array(buf), {
10600
10826
  status: 200,
10601
10827
  headers: {
@@ -10607,6 +10833,8 @@ var init_screenshot_routes = __esm({
10607
10833
  } catch (err) {
10608
10834
  const msg = err instanceof Error ? err.message : String(err);
10609
10835
  return c.json({ error: msg }, 502);
10836
+ } finally {
10837
+ await releaseConcurrencyGate(gate.lockId);
10610
10838
  }
10611
10839
  });
10612
10840
  }
@@ -11390,6 +11618,132 @@ var init_FacebookAdGraphql = __esm({
11390
11618
  }
11391
11619
  });
11392
11620
 
11621
+ // src/extractor/FacebookOrganicVideoExtractor.ts
11622
+ function htmlDecode(value) {
11623
+ return value.replace(/&amp;/g, "&").replace(/&quot;/g, '"').replace(/&#039;/g, "'").replace(/&lt;/g, "<").replace(/&gt;/g, ">");
11624
+ }
11625
+ function decodeEscapedValue(raw) {
11626
+ let out = raw;
11627
+ try {
11628
+ out = JSON.parse(`"${raw.replace(/"/g, '\\"')}"`);
11629
+ } catch {
11630
+ out = raw.replace(/\\u0025/g, "%").replace(/\\u0026/g, "&").replace(/\\u003D/g, "=").replace(/\\u003d/g, "=").replace(/\\"/g, '"').replace(/\\\//g, "/");
11631
+ }
11632
+ return htmlDecode(out);
11633
+ }
11634
+ function facebookVideoIdFromUrl(url) {
11635
+ try {
11636
+ const parsed = new URL(url);
11637
+ const path6 = parsed.pathname;
11638
+ const direct = path6.match(/\/(?:reel|videos|watch)\/(?:[^/]+\/)?(\d{8,})/i);
11639
+ if (direct) return direct[1];
11640
+ const queryVideoId = parsed.searchParams.get("v") ?? parsed.searchParams.get("video_id");
11641
+ if (queryVideoId && /^\d{8,}$/.test(queryVideoId)) return queryVideoId;
11642
+ const numericTail = path6.match(/\/(\d{8,})\/?$/);
11643
+ if (numericTail) return numericTail[1];
11644
+ } catch {
11645
+ }
11646
+ return null;
11647
+ }
11648
+ function parseEfg(url) {
11649
+ try {
11650
+ const efg = new URL(url).searchParams.get("efg");
11651
+ if (!efg) return null;
11652
+ return JSON.parse(Buffer.from(efg, "base64").toString("utf8"));
11653
+ } catch {
11654
+ return null;
11655
+ }
11656
+ }
11657
+ function manifestDurationSec(htmlWindow) {
11658
+ const decoded = decodeEscapedValue(htmlWindow);
11659
+ const match = decoded.match(/mediaPresentationDuration="PT([\d.]+)S"/);
11660
+ return match ? Number(match[1]) : null;
11661
+ }
11662
+ function metadataContent(html, key) {
11663
+ const escaped = key.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
11664
+ const re = new RegExp(`<meta\\s+[^>]*(?:property|name)=["']${escaped}["'][^>]*content=["']([^"']+)["'][^>]*>`, "i");
11665
+ const m = html.match(re);
11666
+ return m ? htmlDecode(m[1]).trim() : null;
11667
+ }
11668
+ function ownerNameFromHtml(html) {
11669
+ const ogTitle = metadataContent(html, "og:title");
11670
+ const pipe = ogTitle?.split("|").map((s) => s.trim()).filter(Boolean);
11671
+ if (pipe && pipe.length > 1) return pipe[pipe.length - 1] ?? null;
11672
+ return null;
11673
+ }
11674
+ function extractFacebookOrganicVideo(html, pageUrl, sourceUrl = pageUrl, quality = "best") {
11675
+ const videoId = facebookVideoIdFromUrl(pageUrl) ?? facebookVideoIdFromUrl(sourceUrl);
11676
+ const manifestIndex = videoId ? html.indexOf(`dash_mpd_debug.mpd?v=${videoId}`) : -1;
11677
+ const searchWindow = manifestIndex >= 0 ? html.slice(Math.max(0, manifestIndex - 5e4), Math.min(html.length, manifestIndex + 5e4)) : html;
11678
+ const targetDuration = manifestDurationSec(searchWindow);
11679
+ const candidates = [];
11680
+ const seen = /* @__PURE__ */ new Set();
11681
+ const re = /browser_native_(sd|hd)_url\\?"\s*:\s*\\?"([^"<]+?)\\?"/g;
11682
+ let match;
11683
+ while ((match = re.exec(searchWindow)) !== null) {
11684
+ const url = decodeEscapedValue(match[2]);
11685
+ if (!url || seen.has(url)) continue;
11686
+ seen.add(url);
11687
+ const parsed = parseEfg(url);
11688
+ let bitrate = null;
11689
+ try {
11690
+ const value = new URL(url).searchParams.get("bitrate");
11691
+ bitrate = value ? Number(value) : null;
11692
+ } catch {
11693
+ }
11694
+ candidates.push({
11695
+ url,
11696
+ quality: match[1],
11697
+ bitrate: Number.isFinite(bitrate) ? bitrate : null,
11698
+ durationSec: typeof parsed?.duration_s === "number" ? parsed.duration_s : null,
11699
+ assetId: typeof parsed?.xpv_asset_id === "number" ? parsed.xpv_asset_id : null,
11700
+ vencodeTag: typeof parsed?.vencode_tag === "string" ? parsed.vencode_tag : null
11701
+ });
11702
+ }
11703
+ const targetCandidates = candidates.filter((candidate) => {
11704
+ if (!targetDuration || candidate.durationSec == null) return true;
11705
+ return Math.abs(candidate.durationSec - targetDuration) <= 2;
11706
+ });
11707
+ const selectable = targetCandidates.length ? targetCandidates : candidates;
11708
+ const selected = [...selectable].filter((candidate) => quality === "best" || candidate.quality === quality).sort((a, b) => {
11709
+ const aq = a.quality === "hd" ? 1 : 0;
11710
+ const bq = b.quality === "hd" ? 1 : 0;
11711
+ if (quality === "best" && aq !== bq) return bq - aq;
11712
+ return (b.bitrate ?? 0) - (a.bitrate ?? 0);
11713
+ })[0];
11714
+ if (!selected) {
11715
+ throw new Error("No progressive Facebook video URL was found in the rendered page state");
11716
+ }
11717
+ return {
11718
+ sourceUrl,
11719
+ pageUrl,
11720
+ videoId,
11721
+ title: metadataContent(html, "og:title") ?? metadataContent(html, "twitter:title"),
11722
+ description: metadataContent(html, "og:description") ?? metadataContent(html, "description") ?? metadataContent(html, "twitter:description"),
11723
+ ownerName: ownerNameFromHtml(html),
11724
+ videoUrl: selected.url,
11725
+ selectedQuality: selected.quality,
11726
+ bitrate: selected.bitrate,
11727
+ durationSec: targetDuration ?? selected.durationSec,
11728
+ assetId: selected.assetId,
11729
+ vencodeTag: selected.vencodeTag,
11730
+ candidates: selectable,
11731
+ extractedAt: (/* @__PURE__ */ new Date()).toISOString()
11732
+ };
11733
+ }
11734
+ async function extractFacebookOrganicVideoFromPage(page, sourceUrl, quality = "best") {
11735
+ await page.waitForLoadState("domcontentloaded", { timeout: 15e3 }).catch(() => {
11736
+ });
11737
+ await page.waitForTimeout(1500);
11738
+ const html = await page.content();
11739
+ return extractFacebookOrganicVideo(html, page.url(), sourceUrl, quality);
11740
+ }
11741
+ var init_FacebookOrganicVideoExtractor = __esm({
11742
+ "src/extractor/FacebookOrganicVideoExtractor.ts"() {
11743
+ "use strict";
11744
+ }
11745
+ });
11746
+
11393
11747
  // src/locations.ts
11394
11748
  var LOCATIONS;
11395
11749
  var init_locations = __esm({
@@ -11933,6 +12287,39 @@ var init_kernel_proxy_resolver = __esm({
11933
12287
  function invalidRequest(message) {
11934
12288
  return { error_code: "invalid_request", message };
11935
12289
  }
12290
+ function isAllowedFacebookOrganicUrl(url) {
12291
+ const hostname = url.hostname.toLowerCase().replace(/^www\./, "").replace(/^m\./, "");
12292
+ return hostname === "facebook.com" || hostname === "fb.watch";
12293
+ }
12294
+ function facebookTranscriptMarkdown(title, text, chunks, durationMs) {
12295
+ const fmtTs2 = (s) => `${Math.floor(s / 60)}:${String(Math.floor(s % 60)).padStart(2, "0")}`;
12296
+ const lines = [title, "", `*Transcribed in ${(durationMs / 1e3).toFixed(1)}s*`, "", "## Full Text", "", text, ""];
12297
+ if (chunks.length) {
12298
+ lines.push("## Timestamped Segments", "");
12299
+ for (const ch of chunks) {
12300
+ lines.push(`**[${fmtTs2(ch.timestamp[0])} -> ${fmtTs2(ch.timestamp[1])}]** ${ch.text.trim()}`, "");
12301
+ }
12302
+ }
12303
+ return lines.join("\n");
12304
+ }
12305
+ async function transcribeFacebookVideoUrl(videoUrl, markdownTitle = "# Facebook Video Transcript") {
12306
+ const startMs = Date.now();
12307
+ const result = await import_client3.fal.subscribe("fal-ai/wizper", {
12308
+ input: { audio_url: videoUrl, task: "transcribe", language: "en" },
12309
+ logs: false,
12310
+ pollInterval: 3e3
12311
+ });
12312
+ const data = result.data;
12313
+ const text = data.text ?? "";
12314
+ const chunks = data.chunks ?? [];
12315
+ const durationMs = Date.now() - startMs;
12316
+ return {
12317
+ text,
12318
+ chunks,
12319
+ durationMs,
12320
+ markdown: facebookTranscriptMarkdown(markdownTitle, text, chunks, durationMs)
12321
+ };
12322
+ }
11936
12323
  async function detectSoftBlock(driver) {
11937
12324
  const page = driver.getPage();
11938
12325
  const bodyText = await page.evaluate(() => document.body?.innerText ?? "").catch(() => "");
@@ -11962,7 +12349,7 @@ async function kernelLaunchOptsResidential() {
11962
12349
  }
11963
12350
  return { headless: true, kernelApiKey: browserServiceApiKey(), kernelProxyId: proxyId, viewport: { width: 1280, height: 900 }, locale: "en-US" };
11964
12351
  }
11965
- var import_hono4, import_zod15, import_client3, FacebookAdBodySchema, FacebookPageIntelBodySchema, FacebookTranscribeBodySchema, FacebookSearchBodySchema, FacebookMediaBodySchema, facebookAdApp, ALLOWED_MEDIA_HOSTS;
12352
+ var import_hono4, import_zod15, import_client3, FacebookAdBodySchema, FacebookPageIntelBodySchema, FacebookTranscribeBodySchema, FacebookVideoTranscribeBodySchema, FacebookSearchBodySchema, FacebookMediaBodySchema, facebookAdApp, ALLOWED_MEDIA_HOSTS;
11966
12353
  var init_facebook_ad_routes = __esm({
11967
12354
  "src/api/facebook-ad-routes.ts"() {
11968
12355
  "use strict";
@@ -11974,10 +12361,12 @@ var init_facebook_ad_routes = __esm({
11974
12361
  init_BrowserDriver();
11975
12362
  init_FacebookAdExtractor();
11976
12363
  init_FacebookAdGraphql();
12364
+ init_FacebookOrganicVideoExtractor();
11977
12365
  init_kernel_proxy_resolver();
11978
12366
  import_client3 = require("@fal-ai/client");
11979
12367
  init_api_auth();
11980
12368
  init_url_utils();
12369
+ init_concurrency_gates();
11981
12370
  FacebookAdBodySchema = import_zod15.z.object({
11982
12371
  url: import_zod15.z.string().trim().optional(),
11983
12372
  libraryId: import_zod15.z.string().trim().optional(),
@@ -11995,6 +12384,10 @@ var init_facebook_ad_routes = __esm({
11995
12384
  FacebookTranscribeBodySchema = import_zod15.z.object({
11996
12385
  videoUrl: import_zod15.z.string().trim().min(1, "videoUrl is required")
11997
12386
  });
12387
+ FacebookVideoTranscribeBodySchema = import_zod15.z.object({
12388
+ url: import_zod15.z.string().trim().min(1, "url is required"),
12389
+ quality: import_zod15.z.enum(["best", "hd", "sd"]).default("best")
12390
+ });
11998
12391
  FacebookSearchBodySchema = import_zod15.z.object({
11999
12392
  query: import_zod15.z.string().trim().min(1, "query is required"),
12000
12393
  country: import_zod15.z.string().trim().toUpperCase().optional(),
@@ -12016,10 +12409,17 @@ var init_facebook_ad_routes = __esm({
12016
12409
  const libraryId = FacebookAdExtractor.resolveLibraryId(raw2);
12017
12410
  if (!libraryId) return c.json({ error: "Could not resolve a valid Facebook Ad Library ID from the provided input" }, 400);
12018
12411
  const fbUser = c.get("user");
12019
- const { ok: adOk, balance_mc: adBal } = await debitMc(fbUser.id, MC_COSTS.fb_ad, LedgerOperation.FB_AD, raw2);
12020
- if (!adOk) return c.json(insufficientBalanceResponse(adBal, MC_COSTS.fb_ad), 402);
12412
+ const gate = await acquireConcurrencyGate(fbUser, "facebook_ad", {
12413
+ reuseLockId: c.req.header("x-mcp-scraper-concurrency-lock"),
12414
+ metadata: { libraryId }
12415
+ });
12416
+ if (!gate.ok) return c.json(concurrencyLimitExceededResponse(gate), 429, { "Retry-After": String(gate.retryAfterSeconds) });
12021
12417
  const driver = new BrowserDriver();
12418
+ let debited = false;
12022
12419
  try {
12420
+ const { ok: adOk, balance_mc: adBal } = await debitMc(fbUser.id, MC_COSTS.fb_ad, LedgerOperation.FB_AD, raw2);
12421
+ if (!adOk) return c.json(insufficientBalanceResponse(adBal, MC_COSTS.fb_ad), 402);
12422
+ debited = true;
12023
12423
  await driver.launch(kernelLaunchOpts());
12024
12424
  const extractor = new FacebookAdExtractor(driver);
12025
12425
  const result = await extractor.extract(libraryId, { openModal: body.openModal !== false });
@@ -12033,7 +12433,7 @@ var init_facebook_ad_routes = __esm({
12033
12433
  });
12034
12434
  return c.json(result);
12035
12435
  } catch (err) {
12036
- await creditMc(fbUser.id, MC_COSTS.fb_ad, LedgerOperation.FB_AD_REFUND, "failed call");
12436
+ if (debited) await creditMc(fbUser.id, MC_COSTS.fb_ad, LedgerOperation.FB_AD_REFUND, "failed call");
12037
12437
  const msg = err instanceof Error ? err.message : String(err);
12038
12438
  await logRequestEvent({ userId: fbUser.id, source: "facebook_ad", status: "failed", query: raw2, error: msg });
12039
12439
  if (msg.toLowerCase().includes("blocked") || msg.toLowerCase().includes("captcha")) {
@@ -12042,24 +12442,26 @@ var init_facebook_ad_routes = __esm({
12042
12442
  return c.json({ error: msg }, 500);
12043
12443
  } finally {
12044
12444
  await driver.close();
12445
+ await releaseConcurrencyGate(gate.lockId);
12045
12446
  }
12046
12447
  });
12047
12448
  facebookAdApp.post("/page-intel", createApiKeyAuth(), async (c) => {
12048
12449
  const raw = await c.req.json().catch(() => ({}));
12049
12450
  const parsed = FacebookPageIntelBodySchema.safeParse(raw);
12050
- if (!parsed.success) {
12051
- return c.json(invalidRequest(parsed.error.issues[0]?.message ?? "Invalid request"), 400);
12052
- }
12451
+ if (!parsed.success) return c.json(invalidRequest(parsed.error.issues[0]?.message ?? "Invalid request"), 400);
12053
12452
  const body = parsed.data;
12054
12453
  const maxAds = Math.min(200, Math.max(1, body.maxAds ?? 50));
12055
12454
  const country = body.country?.trim().toUpperCase() ?? "US";
12056
12455
  const listingUrl = buildPageIntelUrl(body, country);
12057
12456
  const fbUser = c.get("user");
12058
- const { ok: fbOk, balance_mc: fbBal } = await debitMc(fbUser.id, MC_COSTS.fb_ad, LedgerOperation.FB_AD, body.pageId ?? body.query ?? body.libraryId ?? "");
12059
- if (!fbOk) return c.json(insufficientBalanceResponse(fbBal, MC_COSTS.fb_ad), 402);
12457
+ const gate = await acquireConcurrencyGate(fbUser, "facebook_page_intel", { reuseLockId: c.req.header("x-mcp-scraper-concurrency-lock"), metadata: { pageId: body.pageId, query: body.query, libraryId: body.libraryId, country } });
12458
+ if (!gate.ok) return c.json(concurrencyLimitExceededResponse(gate), 429, { "Retry-After": String(gate.retryAfterSeconds) });
12060
12459
  const driver = new BrowserDriver();
12061
- let refunded = false;
12460
+ let refunded = false, debited = false;
12062
12461
  try {
12462
+ const { ok: fbOk, balance_mc: fbBal } = await debitMc(fbUser.id, MC_COSTS.fb_ad, LedgerOperation.FB_AD, body.pageId ?? body.query ?? body.libraryId ?? "");
12463
+ if (!fbOk) return c.json(insufficientBalanceResponse(fbBal, MC_COSTS.fb_ad), 402);
12464
+ debited = true;
12063
12465
  await driver.launch(await kernelLaunchOptsResidential());
12064
12466
  await driver.navigateTo(listingUrl);
12065
12467
  const extractor = new FacebookAdExtractor(driver);
@@ -12074,7 +12476,7 @@ var init_facebook_ad_routes = __esm({
12074
12476
  return c.json(result);
12075
12477
  } catch (err) {
12076
12478
  const msg = err instanceof Error ? err.message : String(err);
12077
- if (!refunded) await creditMc(fbUser.id, MC_COSTS.fb_ad, LedgerOperation.FB_AD_REFUND, "failed call");
12479
+ if (debited && !refunded) await creditMc(fbUser.id, MC_COSTS.fb_ad, LedgerOperation.FB_AD_REFUND, "failed call");
12078
12480
  await logRequestEvent({ userId: fbUser.id, source: "facebook_page_intel", status: "failed", query: body.pageId ?? body.query ?? body.libraryId ?? "", error: msg });
12079
12481
  if (msg.toLowerCase().includes("blocked") || msg.toLowerCase().includes("captcha")) {
12080
12482
  return c.json({ error: msg }, 503);
@@ -12082,6 +12484,7 @@ var init_facebook_ad_routes = __esm({
12082
12484
  return c.json({ error: msg }, 500);
12083
12485
  } finally {
12084
12486
  await driver.close();
12487
+ await releaseConcurrencyGate(gate.lockId);
12085
12488
  }
12086
12489
  });
12087
12490
  facebookAdApp.post("/transcribe", createApiKeyAuth(), async (c) => {
@@ -12097,35 +12500,83 @@ var init_facebook_ad_routes = __esm({
12097
12500
  }
12098
12501
  const videoUrl = urlCheck.parsed.href;
12099
12502
  const fbUser = c.get("user");
12100
- const { ok, balance_mc } = await debitMc(fbUser.id, MC_COSTS.fb_transcribe, LedgerOperation.FB_TRANSCRIBE, videoUrl);
12101
- if (!ok) return c.json(insufficientBalanceResponse(balance_mc, MC_COSTS.fb_transcribe), 402);
12503
+ const gate = await acquireConcurrencyGate(fbUser, "facebook_transcribe", {
12504
+ reuseLockId: c.req.header("x-mcp-scraper-concurrency-lock"),
12505
+ metadata: { videoUrl }
12506
+ });
12507
+ if (!gate.ok) return c.json(concurrencyLimitExceededResponse(gate), 429, { "Retry-After": String(gate.retryAfterSeconds) });
12102
12508
  import_client3.fal.config({ credentials: process.env.FAL_KEY });
12509
+ let debited = false;
12103
12510
  try {
12104
- const startMs = Date.now();
12105
- const result = await import_client3.fal.subscribe("fal-ai/wizper", {
12106
- input: { audio_url: videoUrl, task: "transcribe", language: "en" },
12107
- logs: false,
12108
- pollInterval: 3e3
12109
- });
12110
- const data = result.data;
12111
- const text = data.text ?? "";
12112
- const chunks = data.chunks ?? [];
12113
- const durationMs = Date.now() - startMs;
12114
- const fmtTs2 = (s) => `${Math.floor(s / 60)}:${String(Math.floor(s % 60)).padStart(2, "0")}`;
12115
- const lines = ["# Facebook Ad Transcript", "", `*Transcribed in ${(durationMs / 1e3).toFixed(1)}s*`, "", "## Full Text", "", text, ""];
12116
- if (chunks.length) {
12117
- lines.push("## Timestamped Segments", "");
12118
- for (const ch of chunks) {
12119
- lines.push(`**[${fmtTs2(ch.timestamp[0])} \u2192 ${fmtTs2(ch.timestamp[1])}]** ${ch.text.trim()}`, "");
12120
- }
12121
- }
12122
- await logRequestEvent({ userId: fbUser.id, source: "facebook_transcribe", status: "done", query: videoUrl, resultCount: chunks.length, result: { text, chunks, durationMs } });
12123
- return c.json({ text, chunks, durationMs, markdown: lines.join("\n") });
12511
+ const { ok, balance_mc } = await debitMc(fbUser.id, MC_COSTS.fb_transcribe, LedgerOperation.FB_TRANSCRIBE, videoUrl);
12512
+ if (!ok) return c.json(insufficientBalanceResponse(balance_mc, MC_COSTS.fb_transcribe), 402);
12513
+ debited = true;
12514
+ const transcript = await transcribeFacebookVideoUrl(videoUrl, "# Facebook Ad Transcript");
12515
+ await logRequestEvent({ userId: fbUser.id, source: "facebook_transcribe", status: "done", query: videoUrl, resultCount: transcript.chunks.length, result: { text: transcript.text, chunks: transcript.chunks, durationMs: transcript.durationMs } });
12516
+ return c.json(transcript);
12124
12517
  } catch (err) {
12125
12518
  const msg = err instanceof Error ? err.message : String(err);
12126
- await creditMc(fbUser.id, MC_COSTS.fb_transcribe, LedgerOperation.FB_TRANSCRIBE_REFUND, "failed call");
12519
+ if (debited) await creditMc(fbUser.id, MC_COSTS.fb_transcribe, LedgerOperation.FB_TRANSCRIBE_REFUND, "failed call");
12127
12520
  await logRequestEvent({ userId: fbUser.id, source: "facebook_transcribe", status: "failed", query: videoUrl, error: msg });
12128
12521
  return c.json({ error: msg }, 500);
12522
+ } finally {
12523
+ await releaseConcurrencyGate(gate.lockId);
12524
+ }
12525
+ });
12526
+ facebookAdApp.post("/video-transcribe", createApiKeyAuth(), async (c) => {
12527
+ const raw = await c.req.json().catch(() => ({}));
12528
+ const parsed = FacebookVideoTranscribeBodySchema.safeParse(raw);
12529
+ if (!parsed.success) {
12530
+ return c.json(invalidRequest(parsed.error.issues[0]?.message ?? "Invalid request"), 400);
12531
+ }
12532
+ const body = parsed.data;
12533
+ const urlCheck = await validatePublicHttpUrl(body.url, { field: "url", requireHttps: false });
12534
+ if (urlCheck.error) {
12535
+ return c.json(invalidRequest(urlCheck.error), 400);
12536
+ }
12537
+ const sourceUrl = urlCheck.parsed;
12538
+ if (!isAllowedFacebookOrganicUrl(sourceUrl)) {
12539
+ return c.json(invalidRequest("url must be a Facebook organic video, reel, or share URL"), 400);
12540
+ }
12541
+ const fbUser = c.get("user");
12542
+ const gate = await acquireConcurrencyGate(fbUser, "facebook_video_transcribe", {
12543
+ reuseLockId: c.req.header("x-mcp-scraper-concurrency-lock"),
12544
+ metadata: { url: sourceUrl.href }
12545
+ });
12546
+ if (!gate.ok) return c.json(concurrencyLimitExceededResponse(gate), 429, { "Retry-After": String(gate.retryAfterSeconds) });
12547
+ import_client3.fal.config({ credentials: process.env.FAL_KEY });
12548
+ const driver = new BrowserDriver();
12549
+ let debited = false;
12550
+ try {
12551
+ const { ok, balance_mc } = await debitMc(fbUser.id, MC_COSTS.fb_transcribe, LedgerOperation.FB_TRANSCRIBE, sourceUrl.href);
12552
+ if (!ok) return c.json(insufficientBalanceResponse(balance_mc, MC_COSTS.fb_transcribe), 402);
12553
+ debited = true;
12554
+ await driver.launch(await kernelLaunchOptsResidential());
12555
+ await driver.navigateTo(sourceUrl.href);
12556
+ const page = driver.getPage();
12557
+ const video = await extractFacebookOrganicVideoFromPage(page, sourceUrl.href, body.quality);
12558
+ const transcript = await transcribeFacebookVideoUrl(video.videoUrl, "# Facebook Organic Video Transcript");
12559
+ const result = {
12560
+ ...video,
12561
+ videoDurationSec: video.durationSec,
12562
+ text: transcript.text,
12563
+ chunks: transcript.chunks,
12564
+ durationMs: transcript.durationMs,
12565
+ markdown: transcript.markdown
12566
+ };
12567
+ await logRequestEvent({ userId: fbUser.id, source: "facebook_video_transcribe", status: "done", query: sourceUrl.href, resultCount: transcript.chunks.length, result });
12568
+ return c.json(result);
12569
+ } catch (err) {
12570
+ const msg = err instanceof Error ? err.message : String(err);
12571
+ if (debited) await creditMc(fbUser.id, MC_COSTS.fb_transcribe, LedgerOperation.FB_TRANSCRIBE_REFUND, "failed call");
12572
+ await logRequestEvent({ userId: fbUser.id, source: "facebook_video_transcribe", status: "failed", query: sourceUrl.href, error: msg });
12573
+ if (msg.toLowerCase().includes("blocked") || msg.toLowerCase().includes("captcha")) {
12574
+ return c.json({ error: msg }, 503);
12575
+ }
12576
+ return c.json({ error: msg }, 500);
12577
+ } finally {
12578
+ await driver.close();
12579
+ await releaseConcurrencyGate(gate.lockId);
12129
12580
  }
12130
12581
  });
12131
12582
  facebookAdApp.post("/search", createApiKeyAuth(), async (c) => {
@@ -12139,11 +12590,18 @@ var init_facebook_ad_routes = __esm({
12139
12590
  const maxResults = Math.min(20, Math.max(1, body.maxResults ?? 10));
12140
12591
  const searchUrl = `https://www.facebook.com/ads/library/?active_status=all&ad_type=all&country=${country}&q=${encodeURIComponent(body.query.trim())}&search_type=keyword_unordered`;
12141
12592
  const fbUser = c.get("user");
12142
- const { ok, balance_mc } = await debitMc(fbUser.id, MC_COSTS.fb_search, LedgerOperation.FB_SEARCH, body.query.trim());
12143
- if (!ok) return c.json(insufficientBalanceResponse(balance_mc, MC_COSTS.fb_search), 402);
12593
+ const gate = await acquireConcurrencyGate(fbUser, "facebook_search", {
12594
+ reuseLockId: c.req.header("x-mcp-scraper-concurrency-lock"),
12595
+ metadata: { query: body.query.trim(), country }
12596
+ });
12597
+ if (!gate.ok) return c.json(concurrencyLimitExceededResponse(gate), 429, { "Retry-After": String(gate.retryAfterSeconds) });
12144
12598
  const driver = new BrowserDriver();
12145
12599
  let searchRefunded = false;
12600
+ let debited = false;
12146
12601
  try {
12602
+ const { ok, balance_mc } = await debitMc(fbUser.id, MC_COSTS.fb_search, LedgerOperation.FB_SEARCH, body.query.trim());
12603
+ if (!ok) return c.json(insufficientBalanceResponse(balance_mc, MC_COSTS.fb_search), 402);
12604
+ debited = true;
12147
12605
  await driver.launch(await kernelLaunchOptsResidential());
12148
12606
  const page = driver.getPage();
12149
12607
  const collated = await collectAdLibraryResults(page, searchUrl, Math.max(maxResults * 4, 40));
@@ -12204,7 +12662,7 @@ var init_facebook_ad_routes = __esm({
12204
12662
  return c.json(searchResult);
12205
12663
  } catch (err) {
12206
12664
  const msg = err instanceof Error ? err.message : String(err);
12207
- if (!searchRefunded) await creditMc(fbUser.id, MC_COSTS.fb_search, LedgerOperation.FB_SEARCH_REFUND, "failed call");
12665
+ if (debited && !searchRefunded) await creditMc(fbUser.id, MC_COSTS.fb_search, LedgerOperation.FB_SEARCH_REFUND, "failed call");
12208
12666
  await logRequestEvent({ userId: fbUser.id, source: "facebook_search", status: "failed", query: body.query.trim(), error: msg });
12209
12667
  if (msg.toLowerCase().includes("blocked") || msg.toLowerCase().includes("captcha")) {
12210
12668
  return c.json({ error: msg }, 503);
@@ -12212,6 +12670,7 @@ var init_facebook_ad_routes = __esm({
12212
12670
  return c.json({ error: msg }, 500);
12213
12671
  } finally {
12214
12672
  await driver.close();
12673
+ await releaseConcurrencyGate(gate.lockId);
12215
12674
  }
12216
12675
  });
12217
12676
  ALLOWED_MEDIA_HOSTS = ["fbcdn.net", "cdninstagram.com", "scontent.facebook.com", "scontent.cdninstagram.com"];
@@ -13215,6 +13674,7 @@ var init_maps_routes = __esm({
13215
13674
  init_errors();
13216
13675
  init_browser_service_env();
13217
13676
  init_maps_search_rotation();
13677
+ init_concurrency_gates();
13218
13678
  mapsApp = new import_hono5.Hono();
13219
13679
  mapsApp.post("/search", createApiKeyAuth(), async (c) => {
13220
13680
  const user = c.get("user");
@@ -13226,14 +13686,21 @@ var init_maps_routes = __esm({
13226
13686
  if (!parsed.success) {
13227
13687
  return c.json({ error: parsed.error.issues[0]?.message ?? "Invalid request" }, 400);
13228
13688
  }
13229
- const { ok, balance_mc } = await debitMc(
13230
- user.id,
13231
- MC_COSTS.maps_search,
13232
- LedgerOperation.MAPS_SEARCH,
13233
- [parsed.data.query, parsed.data.location].filter(Boolean).join(" ")
13234
- );
13235
- if (!ok) return c.json(insufficientBalanceResponse(balance_mc, MC_COSTS.maps_search), 402);
13689
+ const gate = await acquireConcurrencyGate(user, "maps_search", {
13690
+ reuseLockId: c.req.header("x-mcp-scraper-concurrency-lock"),
13691
+ metadata: { query: parsed.data.query, location: parsed.data.location }
13692
+ });
13693
+ if (!gate.ok) return c.json(concurrencyLimitExceededResponse(gate), 429, { "Retry-After": String(gate.retryAfterSeconds) });
13694
+ let debited = false;
13236
13695
  try {
13696
+ const { ok, balance_mc } = await debitMc(
13697
+ user.id,
13698
+ MC_COSTS.maps_search,
13699
+ LedgerOperation.MAPS_SEARCH,
13700
+ [parsed.data.query, parsed.data.location].filter(Boolean).join(" ")
13701
+ );
13702
+ if (!ok) return c.json(insufficientBalanceResponse(balance_mc, MC_COSTS.maps_search), 402);
13703
+ debited = true;
13237
13704
  const result = await runMapsSearchWithRotation({
13238
13705
  ...parsed.data,
13239
13706
  configuredKernelProxyId: browserServiceProxyId()
@@ -13249,7 +13716,7 @@ var init_maps_routes = __esm({
13249
13716
  });
13250
13717
  return c.json(result);
13251
13718
  } catch (err) {
13252
- await creditMc(user.id, MC_COSTS.maps_search, LedgerOperation.REFUND, "failed maps_search call");
13719
+ if (debited) await creditMc(user.id, MC_COSTS.maps_search, LedgerOperation.REFUND, "failed maps_search call");
13253
13720
  const msg = err instanceof Error ? err.message : String(err);
13254
13721
  await logRequestEvent({
13255
13722
  userId: user.id,
@@ -13261,6 +13728,8 @@ var init_maps_routes = __esm({
13261
13728
  result: err instanceof MapsSearchRotationError ? { attempts: err.attempts } : void 0
13262
13729
  });
13263
13730
  return mapsErrorResponse(c, err, "maps_search_failed");
13731
+ } finally {
13732
+ await releaseConcurrencyGate(gate.lockId);
13264
13733
  }
13265
13734
  });
13266
13735
  mapsApp.post("/place", createApiKeyAuth(), async (c) => {
@@ -13273,17 +13742,24 @@ var init_maps_routes = __esm({
13273
13742
  if (!parsed.success) {
13274
13743
  return c.json({ error: parsed.error.issues[0]?.message ?? "Invalid request" }, 400);
13275
13744
  }
13276
- const { ok, balance_mc } = await debitMc(
13277
- user.id,
13278
- MC_COSTS.maps_place,
13279
- LedgerOperation.MAPS_PLACE,
13280
- `${parsed.data.businessName} ${parsed.data.location}`
13281
- );
13282
- if (!ok) return c.json(insufficientBalanceResponse(balance_mc, MC_COSTS.maps_place), 402);
13745
+ const gate = await acquireConcurrencyGate(user, "maps_place", {
13746
+ reuseLockId: c.req.header("x-mcp-scraper-concurrency-lock"),
13747
+ metadata: { businessName: parsed.data.businessName, location: parsed.data.location }
13748
+ });
13749
+ if (!gate.ok) return c.json(concurrencyLimitExceededResponse(gate), 429, { "Retry-After": String(gate.retryAfterSeconds) });
13283
13750
  const driver = new BrowserDriver();
13284
13751
  const extractor = new MapsExtractor(driver);
13285
13752
  let reviewDebitMc = 0;
13753
+ let debited = false;
13286
13754
  try {
13755
+ const { ok, balance_mc } = await debitMc(
13756
+ user.id,
13757
+ MC_COSTS.maps_place,
13758
+ LedgerOperation.MAPS_PLACE,
13759
+ `${parsed.data.businessName} ${parsed.data.location}`
13760
+ );
13761
+ if (!ok) return c.json(insufficientBalanceResponse(balance_mc, MC_COSTS.maps_place), 402);
13762
+ debited = true;
13287
13763
  const result = await extractor.extract(parsed.data);
13288
13764
  const reviewCount = Array.isArray(result.reviews) ? result.reviews.length : 0;
13289
13765
  if (reviewCount > 0) {
@@ -13311,7 +13787,7 @@ var init_maps_routes = __esm({
13311
13787
  });
13312
13788
  return c.json(result);
13313
13789
  } catch (err) {
13314
- await creditMc(user.id, MC_COSTS.maps_place, LedgerOperation.REFUND, "failed maps_place call");
13790
+ if (debited) await creditMc(user.id, MC_COSTS.maps_place, LedgerOperation.REFUND, "failed maps_place call");
13315
13791
  if (reviewDebitMc > 0) {
13316
13792
  await creditMc(
13317
13793
  user.id,
@@ -13332,6 +13808,7 @@ var init_maps_routes = __esm({
13332
13808
  return mapsErrorResponse(c, msg, "maps_place_failed");
13333
13809
  } finally {
13334
13810
  await driver.close();
13811
+ await releaseConcurrencyGate(gate.lockId);
13335
13812
  }
13336
13813
  });
13337
13814
  }
@@ -13969,6 +14446,8 @@ function formatCreditsInfo(raw, input) {
13969
14446
  const costs = d.costs ?? [];
13970
14447
  const matched = d.matched_cost;
13971
14448
  const ledger = d.ledger ?? [];
14449
+ const concurrencyRaw = d.concurrency;
14450
+ const upgradeRaw = concurrencyRaw?.upgrade;
13972
14451
  const costRows = costs.map((c) => {
13973
14452
  const notes = c.notes ? ` ${c.notes}` : "";
13974
14453
  return `| ${c.label} | ${c.credits} | ${c.unit}${notes} |`;
@@ -13984,10 +14463,20 @@ function formatCreditsInfo(raw, input) {
13984
14463
  ${matched.notes}` : ""}` : input.item ? `
13985
14464
  ## Matched Cost
13986
14465
  No exact cost match found for "${input.item}". See the full cost table below.` : "";
14466
+ const concurrencySection = concurrencyRaw ? [
14467
+ `
14468
+ ## Concurrency`,
14469
+ `**Current limit:** ${concurrencyRaw.current_limit ?? "unknown"} concurrent operation${concurrencyRaw.current_limit === 1 ? "" : "s"}`,
14470
+ `**Extra slots:** ${concurrencyRaw.current_extra_slots ?? "unknown"}`,
14471
+ `**Extra concurrency slot:** ${upgradeRaw?.price_label ?? "$5/month"}`,
14472
+ `**Upgrade in terminal:** \`${upgradeRaw?.terminal_command ?? "npx -y -p mcp-scraper@latest mcp-scraper-cli billing concurrency checkout"}\``,
14473
+ `**Billing URL:** ${upgradeRaw?.billing_url ?? "https://mcpscraper.dev/billing"}`
14474
+ ].join("\n") : "";
13987
14475
  const full = [
13988
14476
  `# Credits`,
13989
14477
  `**Balance:** ${balance ?? "unknown"} credits`,
13990
14478
  matchedSection,
14479
+ concurrencySection,
13991
14480
  costs.length ? `
13992
14481
  ## Cost Table
13993
14482
  | Item | Credits | Unit |
@@ -14016,7 +14505,22 @@ ${ledgerRows}` : ""
14016
14505
  operation: String(row.operation ?? ""),
14017
14506
  credits: row.amount_mc / 1e3,
14018
14507
  description: row.description ?? null
14019
- }))
14508
+ })),
14509
+ concurrency: concurrencyRaw && upgradeRaw ? {
14510
+ currentExtraSlots: Number(concurrencyRaw.current_extra_slots ?? 0),
14511
+ currentLimit: Number(concurrencyRaw.current_limit ?? 1),
14512
+ hasSubscription: concurrencyRaw.has_subscription === true,
14513
+ upgrade: {
14514
+ product: String(upgradeRaw.product ?? "Extra concurrency slot"),
14515
+ priceLabel: String(upgradeRaw.price_label ?? "$5/month"),
14516
+ unitAmountUsd: Number(upgradeRaw.unit_amount_usd ?? 5),
14517
+ currency: String(upgradeRaw.currency ?? "usd"),
14518
+ interval: String(upgradeRaw.interval ?? "month"),
14519
+ billingUrl: String(upgradeRaw.billing_url ?? "https://mcpscraper.dev/billing"),
14520
+ terminalCommand: String(upgradeRaw.terminal_command ?? "npx -y -p mcp-scraper@latest mcp-scraper-cli billing concurrency checkout"),
14521
+ terminalCommandWithApiKeyEnv: String(upgradeRaw.terminal_command_with_api_key_env ?? "MCP_SCRAPER_API_KEY=sk_live_your_key npx -y -p mcp-scraper@latest mcp-scraper-cli billing concurrency checkout")
14522
+ }
14523
+ } : null
14020
14524
  }
14021
14525
  };
14022
14526
  }
@@ -14302,6 +14806,62 @@ ${chunkRows}` : "",
14302
14806
  ].filter(Boolean).join("\n");
14303
14807
  return oneBlock(full);
14304
14808
  }
14809
+ function formatFacebookVideoTranscribe(raw, input) {
14810
+ const parsed = parseData(raw);
14811
+ if ("error" in parsed) return { content: [{ type: "text", text: parsed.error }], isError: true };
14812
+ const d = parsed.data;
14813
+ const text = d.text ?? "";
14814
+ const chunks = d.chunks ?? [];
14815
+ const wordCount = text.trim() ? text.trim().split(/\s+/).length : 0;
14816
+ const durSec = d.durationMs ? (d.durationMs / 1e3).toFixed(0) : "\u2014";
14817
+ const videoDuration = typeof d.videoDurationSec === "number" ? `${Math.round(d.videoDurationSec)}s` : "\u2014";
14818
+ const label = d.videoId ? `Facebook Organic Video \`${d.videoId}\`` : "Facebook Organic Video";
14819
+ const chunkRows = chunks.slice(0, 50).map((c) => {
14820
+ const sec = Number.isFinite(c.timestamp[0]) ? Math.floor(c.timestamp[0]) : 0;
14821
+ const mm = String(Math.floor(sec / 60)).padStart(2, "0");
14822
+ const ss = String(sec % 60).padStart(2, "0");
14823
+ return `| ${mm}:${ss} | ${cell(truncate(c.text, 120))} |`;
14824
+ }).join("\n");
14825
+ const full = [
14826
+ `# ${label} Transcript`,
14827
+ d.ownerName ? `**Owner:** ${d.ownerName}` : "",
14828
+ `**Video duration:** ${videoDuration} \xB7 **Transcribed in:** ${durSec}s \xB7 **${wordCount} words**`,
14829
+ d.pageUrl ? `**Page URL:** ${d.pageUrl}` : `**Page URL:** ${input.url}`,
14830
+ d.videoUrl ? `**Extracted MP4:** \`${d.videoUrl}\`` : "",
14831
+ `
14832
+ ## Full Transcript
14833
+ ${text}`,
14834
+ chunks.length ? `
14835
+ ## Timestamped Chunks
14836
+ | Time | Text |
14837
+ |------|------|
14838
+ ${chunkRows}` : "",
14839
+ `
14840
+ ---
14841
+ \u{1F4A1} To download the extracted MP4, call the Facebook media endpoint with the extracted MP4 URL.`
14842
+ ].filter(Boolean).join("\n");
14843
+ return {
14844
+ ...oneBlock(full),
14845
+ structuredContent: {
14846
+ sourceUrl: d.sourceUrl ?? input.url,
14847
+ pageUrl: d.pageUrl ?? input.url,
14848
+ videoId: d.videoId ?? null,
14849
+ ownerName: d.ownerName ?? null,
14850
+ selectedQuality: d.selectedQuality ?? input.quality ?? "best",
14851
+ bitrate: typeof d.bitrate === "number" ? d.bitrate : null,
14852
+ videoDurationSec: typeof d.videoDurationSec === "number" ? d.videoDurationSec : null,
14853
+ videoUrl: d.videoUrl ?? "",
14854
+ wordCount,
14855
+ chunkCount: chunks.length,
14856
+ transcriptText: text,
14857
+ chunks: chunks.map((c) => ({
14858
+ startSec: Number.isFinite(c.timestamp[0]) ? c.timestamp[0] : 0,
14859
+ endSec: Number.isFinite(c.timestamp[1]) ? c.timestamp[1] : 0,
14860
+ text: c.text
14861
+ }))
14862
+ }
14863
+ };
14864
+ }
14305
14865
  var import_node_fs3, import_node_os3, import_node_path5, reportSavingEnabled;
14306
14866
  var init_mcp_response_formatter = __esm({
14307
14867
  "src/mcp/mcp-response-formatter.ts"() {
@@ -14811,6 +15371,7 @@ var init_directory_routes = __esm({
14811
15371
  init_directory_workflow();
14812
15372
  init_location_db();
14813
15373
  init_browser_service_env();
15374
+ init_concurrency_gates();
14814
15375
  directoryApp = new import_hono6.Hono();
14815
15376
  directoryApp.post("/run", createApiKeyAuth(), async (c) => {
14816
15377
  const user = c.get("user");
@@ -14826,18 +15387,27 @@ var init_directory_routes = __esm({
14826
15387
  if (!kernelApiKey && parsed.data.proxyMode !== "none") {
14827
15388
  return c.json({ error: "Browser service API key is required for directory workflow Maps searches unless proxyMode is none" }, 503);
14828
15389
  }
14829
- const plan = await resolveDirectoryMarkets(parsed.data);
14830
- const requiredMc = plan.markets.length * MC_COSTS.maps_search;
14831
- if (requiredMc > 0) {
14832
- const debit = await debitMc(
14833
- user.id,
14834
- requiredMc,
14835
- LedgerOperation.MAPS_SEARCH,
14836
- `directory_workflow ${parsed.data.query} ${parsed.data.state} ${plan.markets.length} cities`
14837
- );
14838
- if (!debit.ok) return c.json(insufficientBalanceResponse(debit.balance_mc, requiredMc), 402);
14839
- }
15390
+ const gate = await acquireConcurrencyGate(user, "directory_workflow", {
15391
+ reuseLockId: c.req.header("x-mcp-scraper-concurrency-lock"),
15392
+ ttlSeconds: 30 * 60,
15393
+ metadata: { query: parsed.data.query, state: parsed.data.state }
15394
+ });
15395
+ if (!gate.ok) return c.json(concurrencyLimitExceededResponse(gate), 429, { "Retry-After": String(gate.retryAfterSeconds) });
15396
+ let debited = false;
15397
+ let requiredMc = 0;
14840
15398
  try {
15399
+ const plan = await resolveDirectoryMarkets(parsed.data);
15400
+ requiredMc = plan.markets.length * MC_COSTS.maps_search;
15401
+ if (requiredMc > 0) {
15402
+ const debit = await debitMc(
15403
+ user.id,
15404
+ requiredMc,
15405
+ LedgerOperation.MAPS_SEARCH,
15406
+ `directory_workflow ${parsed.data.query} ${parsed.data.state} ${plan.markets.length} cities`
15407
+ );
15408
+ if (!debit.ok) return c.json(insufficientBalanceResponse(debit.balance_mc, requiredMc), 402);
15409
+ debited = true;
15410
+ }
14841
15411
  const result = await runDirectoryWorkflowFromPlan(parsed.data, plan);
14842
15412
  const failedCities = result.cities.filter((city) => city.status === "failed").length;
14843
15413
  if (failedCities > 0) {
@@ -14854,7 +15424,7 @@ var init_directory_routes = __esm({
14854
15424
  });
14855
15425
  return c.json(result);
14856
15426
  } catch (err) {
14857
- if (requiredMc > 0) {
15427
+ if (debited && requiredMc > 0) {
14858
15428
  await creditMc(user.id, requiredMc, LedgerOperation.REFUND, "failed directory_workflow call");
14859
15429
  }
14860
15430
  const message = err instanceof Error ? err.message : String(err);
@@ -14867,6 +15437,8 @@ var init_directory_routes = __esm({
14867
15437
  error: message
14868
15438
  });
14869
15439
  return c.json({ error: message, error_code: "directory_workflow_failed", retryable: true }, 500);
15440
+ } finally {
15441
+ await releaseConcurrencyGate(gate.lockId);
14870
15442
  }
14871
15443
  });
14872
15444
  }
@@ -15015,18 +15587,21 @@ var init_http_client2 = __esm({
15015
15587
  "src/workflows/http-client.ts"() {
15016
15588
  "use strict";
15017
15589
  WorkflowHttpClient = class {
15018
- constructor(apiUrl, apiKey, fetchImpl = fetch) {
15590
+ constructor(apiUrl, apiKey, fetchImpl = fetch, extraHeaders = {}) {
15019
15591
  this.apiUrl = apiUrl;
15020
15592
  this.apiKey = apiKey;
15021
15593
  this.fetchImpl = fetchImpl;
15594
+ this.extraHeaders = extraHeaders;
15022
15595
  }
15023
15596
  apiUrl;
15024
15597
  apiKey;
15025
15598
  fetchImpl;
15599
+ extraHeaders;
15026
15600
  async post(path6, body, timeoutMs = 18e4) {
15027
15601
  const res = await this.fetchImpl(`${this.apiUrl.replace(/\/$/, "")}${path6}`, {
15028
15602
  method: "POST",
15029
15603
  headers: {
15604
+ ...this.extraHeaders,
15030
15605
  "Content-Type": "application/json",
15031
15606
  "x-api-key": this.apiKey
15032
15607
  },
@@ -16374,7 +16949,7 @@ async function runWorkflow(id, rawInput, options = {}) {
16374
16949
  if (!apiKey) throw new Error("MCP_SCRAPER_API_KEY is required for workflow runs. Pass --api-key or set the environment variable.");
16375
16950
  const apiUrl = options.apiUrl?.trim() || process.env.MCP_SCRAPER_API_URL?.trim() || "https://mcpscraper.dev";
16376
16951
  const artifacts = await ArtifactWriter.create(definition.id, definition.title, input, options.outputDir, options.runId);
16377
- const client2 = new WorkflowHttpClient(apiUrl, apiKey, options.fetchImpl);
16952
+ const client2 = new WorkflowHttpClient(apiUrl, apiKey, options.fetchImpl, options.headers);
16378
16953
  try {
16379
16954
  return await definition.run(input, {
16380
16955
  runId: artifacts.runId,
@@ -16473,7 +17048,7 @@ async function readManifestFromSummary(summary) {
16473
17048
  function webhookSignature(body, timestamp2) {
16474
17049
  const secret2 = process.env.MCP_SCRAPER_WEBHOOK_SECRET?.trim();
16475
17050
  if (!secret2) return null;
16476
- return (0, import_node_crypto2.createHmac)("sha256", secret2).update(`${timestamp2}.${body}`).digest("hex");
17051
+ return (0, import_node_crypto3.createHmac)("sha256", secret2).update(`${timestamp2}.${body}`).digest("hex");
16477
17052
  }
16478
17053
  async function deliverWorkflowWebhook(input) {
16479
17054
  if (!input.webhookUrl) return;
@@ -16527,7 +17102,8 @@ async function executeWorkflowRun(input) {
16527
17102
  apiKey: input.user.api_key,
16528
17103
  apiUrl: input.apiUrl,
16529
17104
  outputDir: hostedWorkflowOutputDir(),
16530
- runId: input.runId
17105
+ runId: input.runId,
17106
+ headers: input.concurrencyLockId ? { "x-mcp-scraper-concurrency-lock": input.concurrencyLockId } : void 0
16531
17107
  });
16532
17108
  const manifest = await readManifestFromSummary(summary);
16533
17109
  await completeWorkflowRunRecord(input.runId, summary.status, manifest ?? { summary });
@@ -16556,19 +17132,28 @@ async function dispatchDueWorkflowSchedules(apiUrl, limit = 3) {
16556
17132
  continue;
16557
17133
  }
16558
17134
  const scheduledFor = schedule.next_run_at ?? now;
16559
- const run = await createWorkflowRun({
16560
- userId: user.id,
16561
- scheduleId: schedule.id,
16562
- workflowId: schedule.workflow_id,
16563
- workflowInput: schedule.input,
16564
- idempotencyKey: `${schedule.id}:${scheduledFor}`
17135
+ const gate = await acquireConcurrencyGate(user, "workflow_schedule_dispatch", {
17136
+ ttlSeconds: 60 * 60,
17137
+ metadata: { scheduleId: schedule.id, workflowId: schedule.workflow_id }
16565
17138
  });
16566
- await markWorkflowScheduleRan(schedule.id, now, addCadence(scheduledFor, schedule.cadence));
16567
- if (run.status !== "queued") {
16568
- results.push({ scheduleId: schedule.id, runId: run.id, status: run.status });
17139
+ if (!gate.ok) {
17140
+ results.push({ scheduleId: schedule.id, status: "deferred", error: concurrencyLimitExceededResponse(gate).message });
16569
17141
  continue;
16570
17142
  }
17143
+ let run = null;
16571
17144
  try {
17145
+ run = await createWorkflowRun({
17146
+ userId: user.id,
17147
+ scheduleId: schedule.id,
17148
+ workflowId: schedule.workflow_id,
17149
+ workflowInput: schedule.input,
17150
+ idempotencyKey: `${schedule.id}:${scheduledFor}`
17151
+ });
17152
+ await markWorkflowScheduleRan(schedule.id, now, addCadence(scheduledFor, schedule.cadence));
17153
+ if (run.status !== "queued") {
17154
+ results.push({ scheduleId: schedule.id, runId: run.id, status: run.status });
17155
+ continue;
17156
+ }
16572
17157
  const executed = await executeWorkflowRun({
16573
17158
  runId: run.id,
16574
17159
  user,
@@ -16576,20 +17161,23 @@ async function dispatchDueWorkflowSchedules(apiUrl, limit = 3) {
16576
17161
  workflowInput: schedule.input,
16577
17162
  apiUrl,
16578
17163
  scheduleId: schedule.id,
16579
- webhookUrl: schedule.webhook_url
17164
+ webhookUrl: schedule.webhook_url,
17165
+ concurrencyLockId: gate.lockId
16580
17166
  });
16581
17167
  results.push({ scheduleId: schedule.id, runId: executed.run.id, status: executed.run.status });
16582
17168
  } catch (err) {
16583
- results.push({ scheduleId: schedule.id, runId: run.id, status: "failed", error: err instanceof Error ? err.message : String(err) });
17169
+ results.push({ scheduleId: schedule.id, runId: run?.id, status: "failed", error: err instanceof Error ? err.message : String(err) });
17170
+ } finally {
17171
+ await releaseConcurrencyGate(gate.lockId);
16584
17172
  }
16585
17173
  }
16586
17174
  return { dispatched: results.length, results };
16587
17175
  }
16588
- var import_node_crypto2, import_promises7, import_hono7, import_zod24, workflowApp, WorkflowInputSchema, WorkflowIdSchema, CadenceSchema, ScheduleStatusSchema, RunBodySchema, ScheduleCreateSchema, SchedulePatchSchema;
17176
+ var import_node_crypto3, import_promises7, import_hono7, import_zod24, workflowApp, WorkflowInputSchema, WorkflowIdSchema, CadenceSchema, ScheduleStatusSchema, RunBodySchema, ScheduleCreateSchema, SchedulePatchSchema;
16589
17177
  var init_workflow_routes = __esm({
16590
17178
  "src/api/workflow-routes.ts"() {
16591
17179
  "use strict";
16592
- import_node_crypto2 = require("crypto");
17180
+ import_node_crypto3 = require("crypto");
16593
17181
  import_promises7 = require("fs/promises");
16594
17182
  import_hono7 = require("hono");
16595
17183
  import_zod24 = require("zod");
@@ -16598,6 +17186,7 @@ var init_workflow_routes = __esm({
16598
17186
  init_api_auth();
16599
17187
  init_db();
16600
17188
  init_url_utils();
17189
+ init_concurrency_gates();
16601
17190
  workflowApp = new import_hono7.Hono();
16602
17191
  WorkflowInputSchema = import_zod24.z.record(import_zod24.z.unknown()).default({});
16603
17192
  WorkflowIdSchema = import_zod24.z.string().min(1);
@@ -16641,20 +17230,29 @@ var init_workflow_routes = __esm({
16641
17230
  } catch (err) {
16642
17231
  return c.json({ error: err instanceof Error ? err.message : String(err) }, 400);
16643
17232
  }
16644
- const run = await createWorkflowRun({ userId: user.id, workflowId: parsed.data.workflowId, workflowInput: input });
17233
+ const gate = await acquireConcurrencyGate(user, "workflow_run", {
17234
+ ttlSeconds: 60 * 60,
17235
+ metadata: { workflowId: parsed.data.workflowId }
17236
+ });
17237
+ if (!gate.ok) return c.json(concurrencyLimitExceededResponse(gate), 429, { "Retry-After": String(gate.retryAfterSeconds) });
17238
+ let run = null;
16645
17239
  try {
17240
+ run = await createWorkflowRun({ userId: user.id, workflowId: parsed.data.workflowId, workflowInput: input });
16646
17241
  const executed = await executeWorkflowRun({
16647
17242
  runId: run.id,
16648
17243
  user,
16649
17244
  workflowId: parsed.data.workflowId,
16650
17245
  workflowInput: input,
16651
17246
  apiUrl: originFromUrl(c.req.url),
16652
- webhookUrl
17247
+ webhookUrl,
17248
+ concurrencyLockId: gate.lockId
16653
17249
  });
16654
17250
  return c.json({ run: exposeRun(c, executed.run, executed.artifacts), summary: executed.summary });
16655
17251
  } catch (err) {
16656
- const failed = await getWorkflowRun(run.id, user.id);
17252
+ const failed = run ? await getWorkflowRun(run.id, user.id) : null;
16657
17253
  return c.json({ run: failed, error: err instanceof Error ? err.message : String(err) }, 500);
17254
+ } finally {
17255
+ await releaseConcurrencyGate(gate.lockId);
16658
17256
  }
16659
17257
  });
16660
17258
  workflowApp.get("/runs", createApiKeyAuth(), async (c) => {
@@ -16752,8 +17350,14 @@ var init_workflow_routes = __esm({
16752
17350
  const user = c.get("user");
16753
17351
  const schedule = await getWorkflowSchedule(c.req.param("id"), user.id);
16754
17352
  if (!schedule) return c.json({ error: "Schedule not found" }, 404);
16755
- const run = await createWorkflowRun({ userId: user.id, scheduleId: schedule.id, workflowId: schedule.workflow_id, workflowInput: schedule.input });
17353
+ const gate = await acquireConcurrencyGate(user, "workflow_schedule_run", {
17354
+ ttlSeconds: 60 * 60,
17355
+ metadata: { scheduleId: schedule.id, workflowId: schedule.workflow_id }
17356
+ });
17357
+ if (!gate.ok) return c.json(concurrencyLimitExceededResponse(gate), 429, { "Retry-After": String(gate.retryAfterSeconds) });
17358
+ let run = null;
16756
17359
  try {
17360
+ run = await createWorkflowRun({ userId: user.id, scheduleId: schedule.id, workflowId: schedule.workflow_id, workflowInput: schedule.input });
16757
17361
  const executed = await executeWorkflowRun({
16758
17362
  runId: run.id,
16759
17363
  user,
@@ -16761,12 +17365,15 @@ var init_workflow_routes = __esm({
16761
17365
  workflowInput: schedule.input,
16762
17366
  apiUrl: originFromUrl(c.req.url),
16763
17367
  scheduleId: schedule.id,
16764
- webhookUrl: schedule.webhook_url
17368
+ webhookUrl: schedule.webhook_url,
17369
+ concurrencyLockId: gate.lockId
16765
17370
  });
16766
17371
  return c.json({ run: exposeRun(c, executed.run, executed.artifacts), summary: executed.summary });
16767
17372
  } catch (err) {
16768
- const failed = await getWorkflowRun(run.id, user.id);
17373
+ const failed = run ? await getWorkflowRun(run.id, user.id) : null;
16769
17374
  return c.json({ run: failed, error: err instanceof Error ? err.message : String(err) }, 500);
17375
+ } finally {
17376
+ await releaseConcurrencyGate(gate.lockId);
16770
17377
  }
16771
17378
  });
16772
17379
  workflowApp.post("/cron/dispatch", async (c) => {
@@ -16782,7 +17389,7 @@ var init_workflow_routes = __esm({
16782
17389
  // src/serp-intelligence/page-snapshot-extractor.ts
16783
17390
  function sha256(value) {
16784
17391
  if (!value) return null;
16785
- return (0, import_node_crypto3.createHash)("sha256").update(value).digest("hex");
17392
+ return (0, import_node_crypto4.createHash)("sha256").update(value).digest("hex");
16786
17393
  }
16787
17394
  function countWords(markdown) {
16788
17395
  const matches = markdown.trim().match(/\b[\p{L}\p{N}][\p{L}\p{N}'-]*\b/gu);
@@ -17082,11 +17689,11 @@ async function capturePageSnapshots(targets, options = {}) {
17082
17689
  }
17083
17690
  };
17084
17691
  }
17085
- var import_node_crypto3, import_p_limit3, DEFAULT_TIMEOUT_MS, DEFAULT_MAX_CONCURRENCY, DEFAULT_MAX_CONTENT_CHARS;
17692
+ var import_node_crypto4, import_p_limit3, DEFAULT_TIMEOUT_MS, DEFAULT_MAX_CONCURRENCY, DEFAULT_MAX_CONTENT_CHARS;
17086
17693
  var init_page_snapshot_extractor = __esm({
17087
17694
  "src/serp-intelligence/page-snapshot-extractor.ts"() {
17088
17695
  "use strict";
17089
- import_node_crypto3 = require("crypto");
17696
+ import_node_crypto4 = require("crypto");
17090
17697
  import_p_limit3 = __toESM(require("p-limit"), 1);
17091
17698
  init_kpo_extractor();
17092
17699
  init_url_utils();
@@ -19382,26 +19989,6 @@ async function enforceSerpIntelligenceRateLimit(c, userId) {
19382
19989
  { "Retry-After": String(result.resetSeconds) }
19383
19990
  );
19384
19991
  }
19385
- async function enforceSerpIntelligenceConcurrency(user) {
19386
- const active = await countActiveJobsForUser(user.id);
19387
- const limit = 1 + (user.extra_concurrency_slots ?? 0);
19388
- if (active < limit) return null;
19389
- return new Response(
19390
- JSON.stringify(structuredError({
19391
- error_code: "concurrency_limit_exceeded",
19392
- error_type: "concurrency_limit",
19393
- message: `You have ${active} active job${active !== 1 ? "s" : ""}. Your account allows ${limit} concurrent job${limit !== 1 ? "s" : ""}.`,
19394
- retryable: true
19395
- })),
19396
- {
19397
- status: 429,
19398
- headers: {
19399
- "Content-Type": "application/json",
19400
- "Retry-After": "30"
19401
- }
19402
- }
19403
- );
19404
- }
19405
19992
  function pageSnapshotTargetsFromBody(body) {
19406
19993
  return body.targets?.length ? body.targets : body.urls.map((url) => ({
19407
19994
  url,
@@ -19421,6 +20008,7 @@ var init_serp_intelligence_routes = __esm({
19421
20008
  init_api_auth();
19422
20009
  init_db();
19423
20010
  init_rates();
20011
+ init_concurrency_gates();
19424
20012
  SERP_INTELLIGENCE_RATE_LIMIT = 60;
19425
20013
  SERP_INTELLIGENCE_RATE_WINDOW_SECONDS = 60;
19426
20014
  POST_CAPTURE_ROUTE_LABEL = "POST /capture";
@@ -19435,17 +20023,22 @@ var init_serp_intelligence_routes = __esm({
19435
20023
  if (!parsed.success) return c.json(formatZodError(parsed.error), 400);
19436
20024
  const limited = await enforceSerpIntelligenceRateLimit(c, user.id);
19437
20025
  if (limited) return limited;
19438
- const concurrencyLimited = await enforceSerpIntelligenceConcurrency(user);
19439
- if (concurrencyLimited) return concurrencyLimited;
20026
+ const gate = await acquireConcurrencyGate(user, "serp_intelligence_capture", {
20027
+ reuseLockId: c.req.header("x-mcp-scraper-concurrency-lock"),
20028
+ metadata: { query: parsed.data.query, location: parsed.data.location }
20029
+ });
20030
+ if (!gate.ok) return c.json(concurrencyLimitExceededResponse(gate), 429, { "Retry-After": String(gate.retryAfterSeconds) });
19440
20031
  const cost = MC_COSTS.serp;
19441
- const { ok, balance_mc } = await debitMc(
19442
- user.id,
19443
- cost,
19444
- LedgerOperation.SERP,
19445
- parsed.data.query
19446
- );
19447
- if (!ok) return c.json(insufficientBalanceResponse(balance_mc, cost), 402);
20032
+ let debited = false;
19448
20033
  try {
20034
+ const { ok, balance_mc } = await debitMc(
20035
+ user.id,
20036
+ cost,
20037
+ LedgerOperation.SERP,
20038
+ parsed.data.query
20039
+ );
20040
+ if (!ok) return c.json(insufficientBalanceResponse(balance_mc, cost), 402);
20041
+ debited = true;
19449
20042
  const result = await captureSerpIntelligenceSnapshot(parsed.data, {
19450
20043
  kernelApiKey: browserServiceApiKey(),
19451
20044
  kernelProxyId: browserServiceProxyId(),
@@ -19463,7 +20056,7 @@ var init_serp_intelligence_routes = __esm({
19463
20056
  });
19464
20057
  return c.json(result);
19465
20058
  } catch (error) {
19466
- await creditMc(user.id, cost, LedgerOperation.REFUND, "failed serp intelligence capture");
20059
+ if (debited) await creditMc(user.id, cost, LedgerOperation.REFUND, "failed serp intelligence capture");
19467
20060
  const body = error instanceof SerpIntelligenceCaptureError ? error.toJSON() : structuredError({
19468
20061
  error_code: "capture_failed",
19469
20062
  error_type: "capture_error",
@@ -19480,6 +20073,8 @@ var init_serp_intelligence_routes = __esm({
19480
20073
  });
19481
20074
  const status = error instanceof SerpIntelligenceCaptureError ? error.httpStatus : 500;
19482
20075
  return c.json(body, status);
20076
+ } finally {
20077
+ await releaseConcurrencyGate(gate.lockId);
19483
20078
  }
19484
20079
  });
19485
20080
  serpIntelligenceApp.post("/page-snapshots", async (c) => {
@@ -19490,18 +20085,23 @@ var init_serp_intelligence_routes = __esm({
19490
20085
  if (!parsed.success) return c.json(formatZodError(parsed.error), 400);
19491
20086
  const limited = await enforceSerpIntelligenceRateLimit(c, user.id);
19492
20087
  if (limited) return limited;
19493
- const concurrencyLimited = await enforceSerpIntelligenceConcurrency(user);
19494
- if (concurrencyLimited) return concurrencyLimited;
19495
20088
  const targets = pageSnapshotTargetsFromBody(parsed.data);
20089
+ const gate = await acquireConcurrencyGate(user, "serp_intelligence_page_snapshots", {
20090
+ reuseLockId: c.req.header("x-mcp-scraper-concurrency-lock"),
20091
+ metadata: { targets: targets.length }
20092
+ });
20093
+ if (!gate.ok) return c.json(concurrencyLimitExceededResponse(gate), 429, { "Retry-After": String(gate.retryAfterSeconds) });
19496
20094
  const cost = targets.length * MC_COSTS.page_scrape;
19497
- const { ok, balance_mc } = await debitMc(
19498
- user.id,
19499
- cost,
19500
- LedgerOperation.EXTRACT_URL,
19501
- `serp intelligence page snapshots: ${targets.length}`
19502
- );
19503
- if (!ok) return c.json(insufficientBalanceResponse(balance_mc, cost), 402);
20095
+ let debited = false;
19504
20096
  try {
20097
+ const { ok, balance_mc } = await debitMc(
20098
+ user.id,
20099
+ cost,
20100
+ LedgerOperation.EXTRACT_URL,
20101
+ `serp intelligence page snapshots: ${targets.length}`
20102
+ );
20103
+ if (!ok) return c.json(insufficientBalanceResponse(balance_mc, cost), 402);
20104
+ debited = true;
19505
20105
  const result = await capturePageSnapshots(targets, {
19506
20106
  kernelApiKey: browserServiceApiKey(),
19507
20107
  timeoutMs: parsed.data.timeoutMs,
@@ -19518,7 +20118,7 @@ var init_serp_intelligence_routes = __esm({
19518
20118
  });
19519
20119
  return c.json(result);
19520
20120
  } catch (error) {
19521
- await creditMc(user.id, cost, LedgerOperation.REFUND, "failed serp intelligence page snapshots");
20121
+ if (debited) await creditMc(user.id, cost, LedgerOperation.REFUND, "failed serp intelligence page snapshots");
19522
20122
  const body = structuredError({
19523
20123
  error_code: "page_snapshot_failed",
19524
20124
  error_type: "capture_error",
@@ -19533,6 +20133,8 @@ var init_serp_intelligence_routes = __esm({
19533
20133
  error: body.message
19534
20134
  });
19535
20135
  return c.json(body, 500);
20136
+ } finally {
20137
+ await releaseConcurrencyGate(gate.lockId);
19536
20138
  }
19537
20139
  });
19538
20140
  }
@@ -19543,12 +20145,12 @@ var PACKAGE_VERSION;
19543
20145
  var init_version = __esm({
19544
20146
  "src/version.ts"() {
19545
20147
  "use strict";
19546
- PACKAGE_VERSION = "0.2.13";
20148
+ PACKAGE_VERSION = "0.2.15";
19547
20149
  }
19548
20150
  });
19549
20151
 
19550
20152
  // src/mcp/mcp-tool-schemas.ts
19551
- var import_zod26, HarvestPaaInputSchema, ExtractUrlInputSchema, MapSiteUrlsInputSchema, ExtractSiteInputSchema, YoutubeHarvestInputSchema, YoutubeTranscribeInputSchema, FacebookPageIntelInputSchema, FacebookAdSearchInputSchema, FacebookAdTranscribeInputSchema, MapsPlaceIntelInputSchema, MapsSearchInputSchema, DirectoryWorkflowInputSchema, RankTrackerModeSchema, RankTrackerBlueprintInputSchema, NullableString, MapsSearchAttemptOutput, MapsSearchOutputSchema, DirectoryMapsBusinessOutput, DirectoryWorkflowOutputSchema, RankTrackerToolPlanOutput, RankTrackerTableOutput, RankTrackerCronJobOutput, RankTrackerBlueprintOutputSchema, OrganicResultOutput, AiOverviewOutput, EntityIdsOutput, HarvestPaaOutputSchema, SearchSerpOutputSchema, ExtractUrlOutputSchema, ExtractSiteOutputSchema, MapsPlaceIntelOutputSchema, CreditsInfoOutputSchema, MapSiteUrlsOutputSchema, YoutubeHarvestOutputSchema, FacebookAdSearchOutputSchema, FacebookPageIntelOutputSchema, CreditsInfoInputSchema, SearchSerpInputSchema, CaptureSerpSnapshotInputSchema, ScreenshotInputSchema, CaptureSerpPageSnapshotsInputSchema;
20153
+ var import_zod26, HarvestPaaInputSchema, ExtractUrlInputSchema, MapSiteUrlsInputSchema, ExtractSiteInputSchema, YoutubeHarvestInputSchema, YoutubeTranscribeInputSchema, FacebookPageIntelInputSchema, FacebookAdSearchInputSchema, FacebookAdTranscribeInputSchema, FacebookVideoTranscribeInputSchema, MapsPlaceIntelInputSchema, MapsSearchInputSchema, DirectoryWorkflowInputSchema, RankTrackerModeSchema, RankTrackerBlueprintInputSchema, NullableString, MapsSearchAttemptOutput, MapsSearchOutputSchema, DirectoryMapsBusinessOutput, DirectoryWorkflowOutputSchema, RankTrackerToolPlanOutput, RankTrackerTableOutput, RankTrackerCronJobOutput, RankTrackerBlueprintOutputSchema, OrganicResultOutput, AiOverviewOutput, EntityIdsOutput, HarvestPaaOutputSchema, SearchSerpOutputSchema, ExtractUrlOutputSchema, ExtractSiteOutputSchema, MapsPlaceIntelOutputSchema, CreditsInfoOutputSchema, MapSiteUrlsOutputSchema, YoutubeHarvestOutputSchema, FacebookAdSearchOutputSchema, FacebookPageIntelOutputSchema, FacebookVideoTranscribeOutputSchema, CreditsInfoInputSchema, SearchSerpInputSchema, CaptureSerpSnapshotInputSchema, ScreenshotInputSchema, CaptureSerpPageSnapshotsInputSchema;
19552
20154
  var init_mcp_tool_schemas = __esm({
19553
20155
  "src/mcp/mcp-tool-schemas.ts"() {
19554
20156
  "use strict";
@@ -19605,6 +20207,10 @@ var init_mcp_tool_schemas = __esm({
19605
20207
  FacebookAdTranscribeInputSchema = {
19606
20208
  videoUrl: import_zod26.z.string().url().describe("Facebook CDN video URL from a facebook_page_intel result")
19607
20209
  };
20210
+ FacebookVideoTranscribeInputSchema = {
20211
+ url: import_zod26.z.string().url().describe("Organic Facebook reel, video, watch, or share URL. The tool renders the page, extracts the best public Facebook CDN MP4 URL, then transcribes it."),
20212
+ quality: import_zod26.z.enum(["best", "hd", "sd"]).default("best").describe("Preferred progressive MP4 quality. Use best by default; hd prefers the highest HD progressive URL; sd forces the SD URL.")
20213
+ };
19608
20214
  MapsPlaceIntelInputSchema = {
19609
20215
  businessName: import_zod26.z.string().min(1).describe('Business name only. If user says "Elite Roofing Denver CO", use businessName="Elite Roofing" and location="Denver, CO".'),
19610
20216
  location: import_zod26.z.string().min(1).describe('City/region/country where the business should be searched, e.g. "Denver, CO". Infer from the user request when possible.'),
@@ -19898,7 +20504,22 @@ var init_mcp_tool_schemas = __esm({
19898
20504
  operation: import_zod26.z.string(),
19899
20505
  credits: import_zod26.z.number(),
19900
20506
  description: NullableString
19901
- }))
20507
+ })),
20508
+ concurrency: import_zod26.z.object({
20509
+ currentExtraSlots: import_zod26.z.number().int().min(0),
20510
+ currentLimit: import_zod26.z.number().int().min(1),
20511
+ hasSubscription: import_zod26.z.boolean(),
20512
+ upgrade: import_zod26.z.object({
20513
+ product: import_zod26.z.string(),
20514
+ priceLabel: import_zod26.z.string(),
20515
+ unitAmountUsd: import_zod26.z.number(),
20516
+ currency: import_zod26.z.string(),
20517
+ interval: import_zod26.z.string(),
20518
+ billingUrl: import_zod26.z.string().url(),
20519
+ terminalCommand: import_zod26.z.string(),
20520
+ terminalCommandWithApiKeyEnv: import_zod26.z.string()
20521
+ })
20522
+ }).nullable()
19902
20523
  };
19903
20524
  MapSiteUrlsOutputSchema = {
19904
20525
  startUrl: import_zod26.z.string(),
@@ -19955,8 +20576,26 @@ var init_mcp_tool_schemas = __esm({
19955
20576
  variations: import_zod26.z.number().int().nullable()
19956
20577
  }))
19957
20578
  };
20579
+ FacebookVideoTranscribeOutputSchema = {
20580
+ sourceUrl: import_zod26.z.string().url(),
20581
+ pageUrl: import_zod26.z.string().url(),
20582
+ videoId: NullableString,
20583
+ ownerName: NullableString,
20584
+ selectedQuality: import_zod26.z.string(),
20585
+ bitrate: import_zod26.z.number().int().nullable(),
20586
+ videoDurationSec: import_zod26.z.number().nullable(),
20587
+ videoUrl: import_zod26.z.string().url(),
20588
+ wordCount: import_zod26.z.number().int().min(0),
20589
+ chunkCount: import_zod26.z.number().int().min(0),
20590
+ transcriptText: import_zod26.z.string(),
20591
+ chunks: import_zod26.z.array(import_zod26.z.object({
20592
+ startSec: import_zod26.z.number(),
20593
+ endSec: import_zod26.z.number(),
20594
+ text: import_zod26.z.string()
20595
+ }))
20596
+ };
19958
20597
  CreditsInfoInputSchema = {
19959
- item: import_zod26.z.string().optional().describe('Optional tool, action, or feature to look up, e.g. "maps reviews", "extract_url", or "YouTube transcription"'),
20598
+ item: import_zod26.z.string().optional().describe('Optional tool, action, or feature to look up, e.g. "maps reviews", "extract_url", "YouTube transcription", or "concurrency"'),
19960
20599
  includeLedger: import_zod26.z.boolean().default(false).describe("Whether to include recent credit ledger entries")
19961
20600
  };
19962
20601
  SearchSerpInputSchema = {
@@ -20462,6 +21101,13 @@ function registerPaaExtractorMcpTools(server, executor, options = {}) {
20462
21101
  inputSchema: FacebookAdTranscribeInputSchema,
20463
21102
  annotations: liveWebToolAnnotations("Facebook Ad Transcription")
20464
21103
  }, async (input) => formatFacebookAdTranscribe(await executor.facebookAdTranscribe(input), input));
21104
+ server.registerTool("facebook_video_transcribe", {
21105
+ title: "Facebook Organic Video Transcription",
21106
+ description: withReportNote("Transcribe audio from an organic Facebook reel, video, watch, or share URL. Renders the Facebook page, extracts the best public Facebook CDN MP4 URL from the page state, then returns the full transcript, timestamped chunks, and extracted MP4 URL."),
21107
+ inputSchema: FacebookVideoTranscribeInputSchema,
21108
+ outputSchema: FacebookVideoTranscribeOutputSchema,
21109
+ annotations: liveWebToolAnnotations("Facebook Organic Video Transcription")
21110
+ }, async (input) => formatFacebookVideoTranscribe(await executor.facebookVideoTranscribe(input), input));
20465
21111
  server.registerTool("maps_place_intel", {
20466
21112
  title: "Google Maps Business Profile Details",
20467
21113
  description: withReportNote('Extract Google Maps business intelligence for one known/named business: rating, review count, category, address, phone, website, hours, booking URL, review histogram, review topics, about attributes, entity IDs, and optional review cards. Do not use this for category searches, local market prospect lists, or requests for multiple GMB/GBP profiles; use maps_search first for those. Split business name from location (e.g. "Elite Roofing Denver CO" => businessName "Elite Roofing", location "Denver, CO"). Pass includeReviews true when the user asks for reviews/customer pain.'),
@@ -20492,7 +21138,7 @@ function registerPaaExtractorMcpTools(server, executor, options = {}) {
20492
21138
  }, async (input) => buildRankTrackerBlueprint(input));
20493
21139
  server.registerTool("credits_info", {
20494
21140
  title: "MCP Scraper Credits & Costs",
20495
- description: "Answer questions about MCP Scraper credits: current credit balance, what a specific tool/action costs, the full cost table, and optionally recent credit ledger entries. Does not expose payment methods or credit card information.",
21141
+ description: "Answer questions about MCP Scraper credits, usage limits, and concurrency upgrades: current credit balance, what a specific tool/action costs, the full cost table, current concurrency limit, the extra-slot price, billing URL, and the terminal checkout command. Does not expose payment methods or credit card information.",
20496
21142
  inputSchema: CreditsInfoInputSchema,
20497
21143
  outputSchema: CreditsInfoOutputSchema,
20498
21144
  annotations: {
@@ -20610,6 +21256,9 @@ var init_http_mcp_tool_executor = __esm({
20610
21256
  facebookAdTranscribe(input) {
20611
21257
  return this.call("/facebook/transcribe", input);
20612
21258
  }
21259
+ facebookVideoTranscribe(input) {
21260
+ return this.call("/facebook/video-transcribe", input, this.httpTimeoutOverrideMs ?? 24e4);
21261
+ }
20613
21262
  mapsPlaceIntel(input) {
20614
21263
  return this.call("/maps/place", input);
20615
21264
  }
@@ -20725,6 +21374,7 @@ async function migrateBrowserAgent() {
20725
21374
  status TEXT NOT NULL DEFAULT 'open',
20726
21375
  label TEXT,
20727
21376
  user_id INTEGER,
21377
+ concurrency_lock_id TEXT,
20728
21378
  created_at TEXT NOT NULL DEFAULT (datetime('now')),
20729
21379
  closed_at TEXT,
20730
21380
  last_action_at TEXT,
@@ -20734,6 +21384,11 @@ async function migrateBrowserAgent() {
20734
21384
  `);
20735
21385
  await db.execute(`CREATE INDEX IF NOT EXISTS browser_agent_sessions_status ON browser_agent_sessions(status)`);
20736
21386
  await db.execute(`CREATE INDEX IF NOT EXISTS browser_agent_sessions_user ON browser_agent_sessions(user_id)`);
21387
+ try {
21388
+ await db.execute(`ALTER TABLE browser_agent_sessions ADD COLUMN concurrency_lock_id TEXT`);
21389
+ } catch {
21390
+ }
21391
+ await db.execute(`CREATE INDEX IF NOT EXISTS browser_agent_sessions_concurrency_lock ON browser_agent_sessions(concurrency_lock_id)`);
20737
21392
  await db.execute(`
20738
21393
  CREATE TABLE IF NOT EXISTS browser_agent_actions (
20739
21394
  id TEXT PRIMARY KEY,
@@ -20761,11 +21416,11 @@ async function migrateBrowserAgent() {
20761
21416
  }
20762
21417
  async function createSessionRow(input) {
20763
21418
  const db = getDb();
20764
- const id = `bas_${(0, import_node_crypto4.randomUUID)().replace(/-/g, "").slice(0, 20)}`;
21419
+ const id = `bas_${(0, import_node_crypto5.randomUUID)().replace(/-/g, "").slice(0, 20)}`;
20765
21420
  await db.execute({
20766
- sql: `INSERT INTO browser_agent_sessions (id, runtime_session_id, live_view_url, cdp_ws_url, status, label, user_id, last_action_at)
20767
- VALUES (?, ?, ?, ?, 'open', ?, ?, datetime('now'))`,
20768
- args: [id, input.runtimeSessionId, input.liveViewUrl, input.cdpWsUrl, input.label, input.userId]
21421
+ sql: `INSERT INTO browser_agent_sessions (id, runtime_session_id, live_view_url, cdp_ws_url, status, label, user_id, concurrency_lock_id, last_action_at)
21422
+ VALUES (?, ?, ?, ?, 'open', ?, ?, ?, datetime('now'))`,
21423
+ args: [id, input.runtimeSessionId, input.liveViewUrl, input.cdpWsUrl, input.label, input.userId, input.concurrencyLockId ?? null]
20769
21424
  });
20770
21425
  const row = await getSessionRow(id);
20771
21426
  if (!row) throw new Error("session insert failed");
@@ -20822,7 +21477,7 @@ async function recordAction(input) {
20822
21477
  sql: `INSERT INTO browser_agent_actions (id, session_id, type, params_json, ok, error)
20823
21478
  VALUES (?, ?, ?, ?, ?, ?)`,
20824
21479
  args: [
20825
- `baa_${(0, import_node_crypto4.randomUUID)().replace(/-/g, "").slice(0, 20)}`,
21480
+ `baa_${(0, import_node_crypto5.randomUUID)().replace(/-/g, "").slice(0, 20)}`,
20826
21481
  input.sessionId,
20827
21482
  input.type,
20828
21483
  input.params == null ? null : JSON.stringify(input.params),
@@ -20862,11 +21517,11 @@ async function listReplayRows(sessionId) {
20862
21517
  });
20863
21518
  return res.rows;
20864
21519
  }
20865
- var import_node_crypto4, _ready;
21520
+ var import_node_crypto5, _ready;
20866
21521
  var init_browser_agent_db = __esm({
20867
21522
  "src/api/browser-agent-db.ts"() {
20868
21523
  "use strict";
20869
- import_node_crypto4 = require("crypto");
21524
+ import_node_crypto5 = require("crypto");
20870
21525
  init_db();
20871
21526
  _ready = false;
20872
21527
  }
@@ -21186,6 +21841,12 @@ var init_browser_agent_service = __esm({
21186
21841
  });
21187
21842
 
21188
21843
  // src/api/browser-agent-routes.ts
21844
+ function browserSessionLockTtlSeconds(timeoutSeconds) {
21845
+ if (typeof timeoutSeconds === "number" && Number.isFinite(timeoutSeconds) && timeoutSeconds > 0) {
21846
+ return Math.max(30 * 60, Math.min(Math.trunc(timeoutSeconds) + 5 * 60, 24 * 60 * 60));
21847
+ }
21848
+ return DEFAULT_BROWSER_SESSION_LOCK_TTL_SECONDS;
21849
+ }
21189
21850
  async function charge(sessionId, userId, startedAtMs) {
21190
21851
  const elapsedMs = Date.now() - startedAtMs;
21191
21852
  const { active_ms, billed_mc } = await addActiveMs(sessionId, elapsedMs);
@@ -21241,6 +21902,8 @@ async function loadOpenSession(id, userId) {
21241
21902
  const row = await getSessionRow(id);
21242
21903
  if (!row) return null;
21243
21904
  if (row.user_id != null && row.user_id !== userId) return null;
21905
+ if (row.status !== "open") return null;
21906
+ await extendConcurrencyGate(row.concurrency_lock_id, DEFAULT_BROWSER_SESSION_LOCK_TTL_SECONDS);
21244
21907
  return row;
21245
21908
  }
21246
21909
  function buildBrowserAgentRoutes() {
@@ -21256,23 +21919,39 @@ function buildBrowserAgentRoutes() {
21256
21919
  return c.json(insufficientBalanceResponse(Number(user.balance_mc ?? 0), BROWSER_OPEN_MIN_BALANCE_MC), 402);
21257
21920
  }
21258
21921
  const body = await c.req.json().catch(() => ({}));
21922
+ const timeoutSeconds = typeof body.timeout_seconds === "number" ? body.timeout_seconds : void 0;
21923
+ const gate = await acquireConcurrencyGate(user, "browser_agent_session", {
21924
+ ttlSeconds: browserSessionLockTtlSeconds(timeoutSeconds),
21925
+ metadata: { label: typeof body.label === "string" ? body.label : null }
21926
+ });
21927
+ if (!gate.ok) return c.json(concurrencyLimitExceededResponse(gate), 429, { "Retry-After": String(gate.retryAfterSeconds) });
21928
+ let runtimeSessionId = null;
21259
21929
  try {
21260
21930
  const created = await createSession({
21261
- timeoutSeconds: typeof body.timeout_seconds === "number" ? body.timeout_seconds : void 0,
21931
+ timeoutSeconds,
21262
21932
  proxyId: typeof body.proxy_id === "string" ? body.proxy_id : void 0,
21263
21933
  profileName: typeof body.profile === "string" ? body.profile : void 0,
21264
21934
  disableDefaultProxy: body.disable_default_proxy === true,
21265
21935
  viewport: body.viewport && typeof body.viewport === "object" ? body.viewport : void 0
21266
21936
  });
21937
+ runtimeSessionId = created.runtimeSessionId;
21267
21938
  const row = await createSessionRow({
21268
21939
  runtimeSessionId: created.runtimeSessionId,
21269
21940
  liveViewUrl: created.liveViewUrl,
21270
21941
  cdpWsUrl: created.cdpWsUrl,
21271
21942
  label: typeof body.label === "string" ? body.label : null,
21272
- userId: user.id
21943
+ userId: user.id,
21944
+ concurrencyLockId: gate.lockId
21273
21945
  });
21274
21946
  return c.json({ ...publicSession(row), watch_url: `/console/${row.id}` });
21275
21947
  } catch (err) {
21948
+ if (runtimeSessionId) {
21949
+ try {
21950
+ await closeSession(runtimeSessionId);
21951
+ } catch {
21952
+ }
21953
+ }
21954
+ await releaseConcurrencyGate(gate.lockId);
21276
21955
  return c.json(failure(err), 502);
21277
21956
  }
21278
21957
  });
@@ -21303,6 +21982,7 @@ function buildBrowserAgentRoutes() {
21303
21982
  } catch {
21304
21983
  }
21305
21984
  await markSessionClosed(row.id);
21985
+ await releaseConcurrencyGate(row.concurrency_lock_id);
21306
21986
  return c.json({ ok: true });
21307
21987
  });
21308
21988
  app2.post("/sessions/:id/goto", async (c) => {
@@ -21560,7 +22240,7 @@ function buildBrowserAgentRoutes() {
21560
22240
  });
21561
22241
  return app2;
21562
22242
  }
21563
- var import_hono10, auth;
22243
+ var import_hono10, auth, DEFAULT_BROWSER_SESSION_LOCK_TTL_SECONDS;
21564
22244
  var init_browser_agent_routes = __esm({
21565
22245
  "src/api/browser-agent-routes.ts"() {
21566
22246
  "use strict";
@@ -21571,7 +22251,9 @@ var init_browser_agent_routes = __esm({
21571
22251
  init_rates();
21572
22252
  init_browser_agent_db();
21573
22253
  init_browser_agent_service();
22254
+ init_concurrency_gates();
21574
22255
  auth = createApiKeyAuth();
22256
+ DEFAULT_BROWSER_SESSION_LOCK_TTL_SECONDS = 2 * 60 * 60;
21575
22257
  }
21576
22258
  });
21577
22259
 
@@ -22073,14 +22755,14 @@ function getSessionSecret() {
22073
22755
  function safeEqualHex(a, b) {
22074
22756
  if (a.length !== b.length) return false;
22075
22757
  try {
22076
- return (0, import_node_crypto5.timingSafeEqual)(Buffer.from(a, "hex"), Buffer.from(b, "hex"));
22758
+ return (0, import_node_crypto6.timingSafeEqual)(Buffer.from(a, "hex"), Buffer.from(b, "hex"));
22077
22759
  } catch {
22078
22760
  return false;
22079
22761
  }
22080
22762
  }
22081
22763
  function signSession(userId) {
22082
22764
  const payload = String(userId);
22083
- const sig = (0, import_node_crypto5.createHmac)("sha256", secret()).update(payload).digest("hex");
22765
+ const sig = (0, import_node_crypto6.createHmac)("sha256", secret()).update(payload).digest("hex");
22084
22766
  return `${payload}.${sig}`;
22085
22767
  }
22086
22768
  function verifySession(token) {
@@ -22088,16 +22770,16 @@ function verifySession(token) {
22088
22770
  if (dot === -1) return null;
22089
22771
  const payload = token.slice(0, dot);
22090
22772
  const sig = token.slice(dot + 1);
22091
- const expected = (0, import_node_crypto5.createHmac)("sha256", secret()).update(payload).digest("hex");
22773
+ const expected = (0, import_node_crypto6.createHmac)("sha256", secret()).update(payload).digest("hex");
22092
22774
  if (!safeEqualHex(sig, expected)) return null;
22093
22775
  const id = parseInt(payload);
22094
22776
  return isNaN(id) ? null : id;
22095
22777
  }
22096
- var import_node_crypto5, isProduction, secret;
22778
+ var import_node_crypto6, isProduction, secret;
22097
22779
  var init_session = __esm({
22098
22780
  "src/api/session.ts"() {
22099
22781
  "use strict";
22100
- import_node_crypto5 = require("crypto");
22782
+ import_node_crypto6 = require("crypto");
22101
22783
  isProduction = () => process.env.NODE_ENV === "production" || process.env.VERCEL === "1";
22102
22784
  secret = () => getSessionSecret();
22103
22785
  }
@@ -22306,14 +22988,23 @@ function countPaaQuestions2(result) {
22306
22988
  function paaCostForQuestionCount2(questionCount) {
22307
22989
  return Math.max(1, questionCount) * MC_COSTS.paa;
22308
22990
  }
22309
- async function checkHarvestLimits(userId, email, extraSlots = 0) {
22310
- if (BYPASS_EMAILS.has(email)) return null;
22311
- const limit = 1 + extraSlots;
22312
- const active = await countActiveJobsForUser(userId);
22313
- if (active >= limit) return { error: `You have ${active} job${active !== 1 ? "s" : ""} running. Your account allows ${limit} concurrent job${limit !== 1 ? "s" : ""}. Wait for one to finish or add a concurrency slot at mcpscraper.dev/billing.` };
22991
+ async function checkHarvestLimits(user, reuseLockId) {
22992
+ if (reuseLockId && await reuseExistingConcurrencyGate(user.id, reuseLockId, 30 * 60)) return null;
22993
+ if (isConcurrencyLimitBypassed(user)) return null;
22994
+ const limit = concurrencyLimitForUser(user);
22995
+ const active = await countActiveUsageForUser(user.id);
22996
+ if (active >= limit) {
22997
+ return concurrencyLimitExceededResponse({
22998
+ ok: false,
22999
+ active,
23000
+ limit,
23001
+ operation: "harvest",
23002
+ retryAfterSeconds: 30
23003
+ });
23004
+ }
22314
23005
  return null;
22315
23006
  }
22316
- var import_resend, import_hono12, import_hono13, import_factory6, import_cookie, import_stripe2, secureCookies, isProduction2, sessionCookieOptions, requireAllowedOrigin, auth2, adminAuth, sessionAuth, app, STRIPE_API_VERSION, BYPASS_EMAILS, SYNC_HARVEST_TIMEOUT_OVERRIDE_MS;
23007
+ var import_resend, import_hono12, import_hono13, import_factory6, import_cookie, import_stripe2, secureCookies, isProduction2, sessionCookieOptions, requireAllowedOrigin, auth2, adminAuth, sessionAuth, app, STRIPE_API_VERSION, SYNC_HARVEST_TIMEOUT_OVERRIDE_MS;
22317
23008
  var init_server = __esm({
22318
23009
  "src/api/server.ts"() {
22319
23010
  "use strict";
@@ -22359,6 +23050,7 @@ var init_server = __esm({
22359
23050
  init_harvest_problems();
22360
23051
  init_harvest_attempt_events();
22361
23052
  init_session();
23053
+ init_concurrency_gates();
22362
23054
  secureCookies = process.env.NODE_ENV === "production" || process.env.VERCEL === "1";
22363
23055
  isProduction2 = secureCookies;
22364
23056
  sessionCookieOptions = {
@@ -22555,9 +23247,6 @@ var init_server = __esm({
22555
23247
  await revokeApiKey(user.id);
22556
23248
  return c.json({ ok: true });
22557
23249
  });
22558
- BYPASS_EMAILS = new Set(
22559
- (process.env.HARVEST_LIMIT_BYPASS_EMAILS ?? "").split(",").map((e) => e.trim()).filter(Boolean)
22560
- );
22561
23250
  SYNC_HARVEST_TIMEOUT_OVERRIDE_MS = (() => {
22562
23251
  const raw = process.env.SYNC_HARVEST_TIMEOUT_MS;
22563
23252
  const parsed = raw === void 0 ? NaN : Number(raw);
@@ -22575,8 +23264,8 @@ var init_server = __esm({
22575
23264
  if (checked.error) return c.json({ error: checked.error }, 400);
22576
23265
  body.callback_url = checked.parsed?.href;
22577
23266
  }
22578
- const limitErr = await checkHarvestLimits(user.id, user.email, user.extra_concurrency_slots);
22579
- if (limitErr) return c.json(limitErr, 429);
23267
+ const limitErr = await checkHarvestLimits(user, c.req.header("x-mcp-scraper-concurrency-lock"));
23268
+ if (limitErr) return c.json(limitErr, 429, { "Retry-After": "30" });
22580
23269
  const options = {
22581
23270
  query: body.query.trim(),
22582
23271
  location: body.location,
@@ -22611,8 +23300,8 @@ var init_server = __esm({
22611
23300
  const bodyResult = HarvestBodySchema.safeParse(raw);
22612
23301
  if (!bodyResult.success) return c.json({ error: bodyResult.error.issues[0]?.message ?? "Invalid request" }, 400);
22613
23302
  const body = bodyResult.data;
22614
- const limitErr = await checkHarvestLimits(user.id, user.email, user.extra_concurrency_slots);
22615
- if (limitErr) return c.json(limitErr, 429);
23303
+ const limitErr = await checkHarvestLimits(user, c.req.header("x-mcp-scraper-concurrency-lock"));
23304
+ if (limitErr) return c.json(limitErr, 429, { "Retry-After": "30" });
22616
23305
  const options = {
22617
23306
  query: body.query.trim(),
22618
23307
  location: body.location,
@@ -22771,9 +23460,16 @@ var init_server = __esm({
22771
23460
  }
22772
23461
  })();
22773
23462
  const user = c.get("user");
22774
- const { ok: euOk, balance_mc: euBal } = await debitMc(user.id, MC_COSTS.page_scrape, LedgerOperation.EXTRACT_URL, new URL(canonicalUrl).hostname);
22775
- if (!euOk) return c.json(insufficientBalanceResponse(euBal, MC_COSTS.page_scrape), 402);
23463
+ const gate = await acquireConcurrencyGate(user, "extract_url", {
23464
+ reuseLockId: c.req.header("x-mcp-scraper-concurrency-lock"),
23465
+ metadata: { url: canonicalUrl }
23466
+ });
23467
+ if (!gate.ok) return c.json(concurrencyLimitExceededResponse(gate), 429, { "Retry-After": String(gate.retryAfterSeconds) });
23468
+ let debited = false;
22776
23469
  try {
23470
+ const { ok: euOk, balance_mc: euBal } = await debitMc(user.id, MC_COSTS.page_scrape, LedgerOperation.EXTRACT_URL, new URL(canonicalUrl).hostname);
23471
+ if (!euOk) return c.json(insufficientBalanceResponse(euBal, MC_COSTS.page_scrape), 402);
23472
+ debited = true;
22777
23473
  const kernelApiKey = browserServiceApiKey();
22778
23474
  const device = screenshotDevice === "mobile" ? "mobile" : "desktop";
22779
23475
  const [result, pageData] = await Promise.all([
@@ -22791,9 +23487,11 @@ var init_server = __esm({
22791
23487
  return c.json({ ...result, screenshot: screenshotMeta, branding: brandingData, media: mediaMeta });
22792
23488
  } catch (err) {
22793
23489
  const msg = err instanceof Error ? err.message : String(err);
22794
- await creditMc(user.id, MC_COSTS.page_scrape, LedgerOperation.EXTRACT_URL, "failed call");
23490
+ if (debited) await creditMc(user.id, MC_COSTS.page_scrape, LedgerOperation.EXTRACT_URL, "failed call");
22795
23491
  await logRequestEvent({ userId: user.id, source: "extract_url", status: "failed", query: canonicalUrl, error: msg });
22796
23492
  return c.json({ error: msg }, 500);
23493
+ } finally {
23494
+ await releaseConcurrencyGate(gate.lockId);
22797
23495
  }
22798
23496
  });
22799
23497
  app.post("/map-urls", auth2, async (c) => {
@@ -22805,9 +23503,16 @@ var init_server = __esm({
22805
23503
  if (checked.error || !checked.parsed) return c.json({ error: checked.error ?? "Invalid URL" }, 400);
22806
23504
  const parsed = checked.parsed;
22807
23505
  const user = c.get("user");
22808
- const { ok: mapOk, balance_mc: mapBal } = await debitMc(user.id, MC_COSTS.url_map, LedgerOperation.URL_MAP, parsed.hostname);
22809
- if (!mapOk) return c.json(insufficientBalanceResponse(mapBal, MC_COSTS.url_map), 402);
23506
+ const gate = await acquireConcurrencyGate(user, "map_urls", {
23507
+ reuseLockId: c.req.header("x-mcp-scraper-concurrency-lock"),
23508
+ metadata: { url: parsed.href }
23509
+ });
23510
+ if (!gate.ok) return c.json(concurrencyLimitExceededResponse(gate), 429, { "Retry-After": String(gate.retryAfterSeconds) });
23511
+ let debited = false;
22810
23512
  try {
23513
+ const { ok: mapOk, balance_mc: mapBal } = await debitMc(user.id, MC_COSTS.url_map, LedgerOperation.URL_MAP, parsed.hostname);
23514
+ if (!mapOk) return c.json(insufficientBalanceResponse(mapBal, MC_COSTS.url_map), 402);
23515
+ debited = true;
22811
23516
  const result = await spiderSite({
22812
23517
  startUrl: parsed.href,
22813
23518
  maxUrls: Math.min(2e3, Math.max(1, body.maxUrls ?? 500)),
@@ -22825,7 +23530,7 @@ var init_server = __esm({
22825
23530
  return c.json(result);
22826
23531
  } catch (err) {
22827
23532
  const msg = err instanceof Error ? err.message : String(err);
22828
- await creditMc(user.id, MC_COSTS.url_map, LedgerOperation.URL_MAP_REFUND, "failed call");
23533
+ if (debited) await creditMc(user.id, MC_COSTS.url_map, LedgerOperation.URL_MAP_REFUND, "failed call");
22829
23534
  await logRequestEvent({
22830
23535
  userId: user.id,
22831
23536
  source: "map_urls",
@@ -22834,6 +23539,8 @@ var init_server = __esm({
22834
23539
  error: msg
22835
23540
  });
22836
23541
  return c.json({ error: msg }, 500);
23542
+ } finally {
23543
+ await releaseConcurrencyGate(gate.lockId);
22837
23544
  }
22838
23545
  });
22839
23546
  app.post("/extract-site", auth2, async (c) => {
@@ -22846,9 +23553,16 @@ var init_server = __esm({
22846
23553
  const parsed = checked.parsed;
22847
23554
  const user = c.get("user");
22848
23555
  const siteHoldMc = MC_COSTS.page_scrape * 10;
22849
- const { ok: siteOk, balance_mc: siteBal } = await debitMc(user.id, siteHoldMc, LedgerOperation.EXTRACT_SITE_HOLD, parsed.hostname);
22850
- if (!siteOk) return c.json(insufficientBalanceResponse(siteBal, siteHoldMc), 402);
23556
+ const gate = await acquireConcurrencyGate(user, "extract_site", {
23557
+ reuseLockId: c.req.header("x-mcp-scraper-concurrency-lock"),
23558
+ metadata: { url: parsed.href }
23559
+ });
23560
+ if (!gate.ok) return c.json(concurrencyLimitExceededResponse(gate), 429, { "Retry-After": String(gate.retryAfterSeconds) });
23561
+ let debited = false;
22851
23562
  try {
23563
+ const { ok: siteOk, balance_mc: siteBal } = await debitMc(user.id, siteHoldMc, LedgerOperation.EXTRACT_SITE_HOLD, parsed.hostname);
23564
+ if (!siteOk) return c.json(insufficientBalanceResponse(siteBal, siteHoldMc), 402);
23565
+ debited = true;
22852
23566
  const result = await extractSite({
22853
23567
  startUrl: parsed.href,
22854
23568
  maxPages: Math.min(200, Math.max(1, body.maxPages ?? 100)),
@@ -22869,7 +23583,7 @@ var init_server = __esm({
22869
23583
  });
22870
23584
  return c.json(result);
22871
23585
  } catch (err) {
22872
- await creditMc(user.id, siteHoldMc, LedgerOperation.EXTRACT_SITE_REFUND, "failed call");
23586
+ if (debited) await creditMc(user.id, siteHoldMc, LedgerOperation.EXTRACT_SITE_REFUND, "failed call");
22873
23587
  const msg = err instanceof Error ? err.message : String(err);
22874
23588
  await logRequestEvent({
22875
23589
  userId: user.id,
@@ -22879,6 +23593,8 @@ var init_server = __esm({
22879
23593
  error: msg
22880
23594
  });
22881
23595
  return c.json({ error: msg }, 500);
23596
+ } finally {
23597
+ await releaseConcurrencyGate(gate.lockId);
22882
23598
  }
22883
23599
  });
22884
23600
  app.post("/billing/checkout", requireAllowedOrigin, sessionAuth, async (c) => {
@@ -22947,6 +23663,60 @@ var init_server = __esm({
22947
23663
  return c.json({ error: message }, 500);
22948
23664
  }
22949
23665
  });
23666
+ app.post("/billing/concurrency/terminal-checkout", auth2, async (c) => {
23667
+ try {
23668
+ const user = c.get("user");
23669
+ const upgrade = concurrencySlotBillingInfo();
23670
+ if (user.concurrency_stripe_sub_id) {
23671
+ return c.json({
23672
+ error: "Already subscribed. Your account already has an active extra concurrency slot subscription.",
23673
+ error_code: "concurrency_subscription_exists",
23674
+ concurrency: {
23675
+ extra_slots: user.extra_concurrency_slots,
23676
+ limit: concurrencyLimitForUser(user),
23677
+ has_subscription: true,
23678
+ upgrade
23679
+ }
23680
+ }, 409);
23681
+ }
23682
+ const secret2 = process.env.STRIPE_SECRET_KEY?.trim();
23683
+ if (!secret2) return c.json({ error: "Stripe is not configured." }, 503);
23684
+ const stripeClient = new import_stripe2.default(secret2, { apiVersion: STRIPE_API_VERSION });
23685
+ let customerId = user.stripe_customer_id;
23686
+ if (!customerId) {
23687
+ const customer = await stripeClient.customers.create({ email: user.email });
23688
+ customerId = customer.id;
23689
+ await setStripeCustomerId(user.id, customerId);
23690
+ }
23691
+ const session = await stripeClient.checkout.sessions.create({
23692
+ customer: customerId,
23693
+ mode: "subscription",
23694
+ line_items: [{ price: CONCURRENCY_PRICE_ID, quantity: 1 }],
23695
+ automatic_tax: { enabled: true },
23696
+ billing_address_collection: "required",
23697
+ customer_update: { address: "auto", name: "auto" },
23698
+ tax_id_collection: { enabled: true },
23699
+ success_url: `${appOrigin()}/billing?slot_added=1&source=terminal`,
23700
+ cancel_url: `${appOrigin()}/billing?slot_cancelled=1&source=terminal`
23701
+ });
23702
+ if (!session.url) return c.json({ error: "Stripe did not return a checkout URL." }, 502);
23703
+ return c.json({
23704
+ checkout_url: session.url,
23705
+ price: upgrade,
23706
+ concurrency: {
23707
+ current_extra_slots: user.extra_concurrency_slots,
23708
+ current_limit: concurrencyLimitForUser(user),
23709
+ after_checkout_extra_slots: user.extra_concurrency_slots + 1,
23710
+ after_checkout_limit: concurrencyLimitForUser({ extra_concurrency_slots: user.extra_concurrency_slots + 1 })
23711
+ },
23712
+ next_step: "Open checkout_url in a browser, complete checkout, then restart or retry the MCP request."
23713
+ });
23714
+ } catch (err) {
23715
+ const message = err instanceof Error ? err.message : "Unable to start terminal checkout.";
23716
+ console.error("[billing/concurrency/terminal-checkout]", message);
23717
+ return c.json({ error: message }, 500);
23718
+ }
23719
+ });
22950
23720
  app.post("/billing/concurrency/cancel", requireAllowedOrigin, sessionAuth, async (c) => {
22951
23721
  const user = c.get("sessionUser");
22952
23722
  if (!user.concurrency_stripe_sub_id) return c.json({ error: "No active concurrency subscription." }, 404);
@@ -22954,9 +23724,10 @@ var init_server = __esm({
22954
23724
  if (!stripeSecret) return c.json({ error: "Stripe is not configured." }, 503);
22955
23725
  const stripeClient = new import_stripe2.default(stripeSecret, { apiVersion: STRIPE_API_VERSION });
22956
23726
  await stripeClient.subscriptions.cancel(user.concurrency_stripe_sub_id);
22957
- await setExtraConcurrencySlots(user.id, Math.max(0, user.extra_concurrency_slots - 1));
23727
+ const nextSlots = Math.max(0, user.extra_concurrency_slots - 1);
23728
+ await setExtraConcurrencySlots(user.id, nextSlots);
22958
23729
  await setConcurrencySubId(user.id, null);
22959
- return c.json({ ok: true, concurrency_limit: user.extra_concurrency_slots });
23730
+ return c.json({ ok: true, extra_concurrency_slots: nextSlots, concurrency_limit: 1 + nextSlots });
22960
23731
  });
22961
23732
  app.get("/billing/balance", auth2, async (c) => {
22962
23733
  const user = c.get("user");
@@ -22991,6 +23762,12 @@ var init_server = __esm({
22991
23762
  item: body.item ?? null,
22992
23763
  matched_cost: matchedCost ? (({ aliases, ...cost }) => cost)(matchedCost) : null,
22993
23764
  costs,
23765
+ concurrency: {
23766
+ current_extra_slots: user.extra_concurrency_slots,
23767
+ current_limit: concurrencyLimitForUser(user),
23768
+ has_subscription: !!user.concurrency_stripe_sub_id,
23769
+ upgrade: concurrencySlotBillingInfo()
23770
+ },
22994
23771
  ledger
22995
23772
  });
22996
23773
  });