mcp-scraper 0.4.12 → 0.4.13

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (35) hide show
  1. package/dist/bin/api-server.cjs +196 -79
  2. package/dist/bin/api-server.cjs.map +1 -1
  3. package/dist/bin/api-server.js +2 -2
  4. package/dist/bin/mcp-scraper-cli.cjs +1 -1
  5. package/dist/bin/mcp-scraper-cli.cjs.map +1 -1
  6. package/dist/bin/mcp-scraper-cli.js +1 -1
  7. package/dist/bin/mcp-scraper-install.cjs +1 -1
  8. package/dist/bin/mcp-scraper-install.cjs.map +1 -1
  9. package/dist/bin/mcp-scraper-install.js +1 -1
  10. package/dist/bin/mcp-stdio-server.cjs +1 -1
  11. package/dist/bin/mcp-stdio-server.cjs.map +1 -1
  12. package/dist/bin/mcp-stdio-server.js +2 -2
  13. package/dist/bin/paa-harvest.js +1 -1
  14. package/dist/{chunk-2YY46QYT.js → chunk-AUCXKRRH.js} +3 -1
  15. package/dist/{chunk-6EXP6DQG.js → chunk-EA2AANFI.js} +2 -2
  16. package/dist/chunk-O2D5HGKF.js +7 -0
  17. package/dist/chunk-O2D5HGKF.js.map +1 -0
  18. package/dist/{chunk-EMY7ELRU.js → chunk-SIPXEFTR.js} +14 -10
  19. package/dist/chunk-SIPXEFTR.js.map +1 -0
  20. package/dist/{chunk-ONIOF5XW.js → chunk-VQV7MB2S.js} +2 -2
  21. package/dist/index.js +1 -1
  22. package/dist/{server-UNID3SJU.js → server-HNZG36IP.js} +178 -72
  23. package/dist/server-HNZG36IP.js.map +1 -0
  24. package/dist/{site-extract-repository-HTIY52MW.js → site-extract-repository-6X2C7RLF.js} +3 -3
  25. package/dist/{worker-HTYZAYGB.js → worker-WCSKUAOT.js} +3 -3
  26. package/package.json +1 -1
  27. package/dist/chunk-EMY7ELRU.js.map +0 -1
  28. package/dist/chunk-SHXJQQOH.js +0 -7
  29. package/dist/chunk-SHXJQQOH.js.map +0 -1
  30. package/dist/server-UNID3SJU.js.map +0 -1
  31. /package/dist/{chunk-2YY46QYT.js.map → chunk-AUCXKRRH.js.map} +0 -0
  32. /package/dist/{chunk-6EXP6DQG.js.map → chunk-EA2AANFI.js.map} +0 -0
  33. /package/dist/{chunk-ONIOF5XW.js.map → chunk-VQV7MB2S.js.map} +0 -0
  34. /package/dist/{site-extract-repository-HTIY52MW.js.map → site-extract-repository-6X2C7RLF.js.map} +0 -0
  35. /package/dist/{worker-HTYZAYGB.js.map → worker-WCSKUAOT.js.map} +0 -0
@@ -7986,7 +7986,7 @@ function insufficientBalanceResponse(balanceMc, requiredMc) {
7986
7986
  topup_url: topupUrl
7987
7987
  };
7988
7988
  }
7989
- var MC_COSTS, MC_PER_BROWSER_MS, BROWSER_OPEN_MIN_BALANCE_MC, MC_PER_CREDIT, CREDIT_COST_CATALOG, CONCURRENCY_PRICE_ID, SUBSCRIPTION_TIERS, SUBSCRIPTION_TIER_BY_KEY, CONCURRENCY_SLOT_PRICE_USD, CONCURRENCY_SLOT_PRICE_CURRENCY, CONCURRENCY_SLOT_PRICE_INTERVAL, CONCURRENCY_SLOT_PRICE_LABEL, CONCURRENCY_SLOT_TERMINAL_COMMAND, FREE_SIGNUP_MC, MEMORY_FREE_QUOTA_BYTES, MEMORY_PLANS, MEMORY_PLAN_QUOTA, MEMORY_MARGIN_MULTIPLE, MEMORY_PRO_COST_BUDGET_USD, SCHEDULING_PLANS, LedgerOperation, MEMORY_AI_MARGIN_MULTIPLE, SITE_AUDIT_LLM_MARGIN_MULTIPLE, VIDEO_ANALYSIS_LLM_MARGIN_MULTIPLE, MC_PER_USD;
7989
+ var MC_COSTS, MC_PER_BROWSER_MS, BROWSER_OPEN_MIN_BALANCE_MC, MC_PER_CREDIT, CREDIT_COST_CATALOG, CONCURRENCY_PRICE_ID, SUBSCRIPTION_TIERS, SUBSCRIPTION_TIER_BY_KEY, CONCURRENCY_SLOT_PRICE_USD, CONCURRENCY_SLOT_PRICE_CURRENCY, CONCURRENCY_SLOT_PRICE_INTERVAL, CONCURRENCY_SLOT_PRICE_LABEL, CONCURRENCY_SLOT_TERMINAL_COMMAND, FREE_SIGNUP_MC, MEMORY_FREE_QUOTA_BYTES, MEMORY_PLANS, MEMORY_PLAN_QUOTA, MEMORY_MARGIN_MULTIPLE, MEMORY_PRO_COST_BUDGET_USD, SCHEDULING_PLANS, LedgerOperation, MEMORY_AI_MARGIN_MULTIPLE, SITE_AUDIT_LLM_MARGIN_MULTIPLE, VIDEO_ANALYSIS_LLM_MARGIN_MULTIPLE, MEDIA_TRANSCRIBE_MARGIN_MULTIPLE, MEDIA_TRANSCRIBE_HOLD_MC, MC_PER_USD;
7990
7990
  var init_rates = __esm({
7991
7991
  "src/api/rates.ts"() {
7992
7992
  "use strict";
@@ -8003,13 +8003,13 @@ var init_rates = __esm({
8003
8003
  maps_place: 6e3,
8004
8004
  maps_review: 100,
8005
8005
  fb_search: 600,
8006
- fb_transcribe: 200,
8006
+ fb_transcribe: 33300,
8007
8007
  google_ads_search: 600,
8008
8008
  google_ads_intel: 200,
8009
- google_ads_transcribe: 200,
8009
+ google_ads_transcribe: 33300,
8010
8010
  instagram_profile: 400,
8011
8011
  instagram_media: 400,
8012
- instagram_transcribe: 50,
8012
+ instagram_transcribe: 33300,
8013
8013
  browser_minute: 12e3,
8014
8014
  reddit_thread: 3e3,
8015
8015
  reddit_comment: 200,
@@ -8131,16 +8131,16 @@ var init_rates = __esm({
8131
8131
  label: "Google ad video transcription",
8132
8132
  aliases: ["google_ads_transcribe", "google ad transcript"],
8133
8133
  credits: mcToCredits(MC_COSTS.google_ads_transcribe),
8134
- unit: "per minute",
8135
- notes: "A hold is taken, then reconciled to actual video duration."
8134
+ unit: "per minute (typical)",
8135
+ notes: "A hold is taken, then reconciled to the real transcription cost for the actual audio duration."
8136
8136
  },
8137
8137
  {
8138
8138
  key: "fb_transcribe",
8139
8139
  label: "Facebook video / ad transcription",
8140
8140
  aliases: ["facebook_transcribe", "facebook_video_transcribe", "fb_transcribe", "fb ad transcript"],
8141
8141
  credits: mcToCredits(MC_COSTS.fb_transcribe),
8142
- unit: "per minute",
8143
- notes: "A hold is taken, then reconciled to actual video duration."
8142
+ unit: "per minute (typical)",
8143
+ notes: "A hold is taken, then reconciled to the real transcription cost for the actual audio duration."
8144
8144
  },
8145
8145
  {
8146
8146
  key: "instagram_profile",
@@ -8163,8 +8163,8 @@ var init_rates = __esm({
8163
8163
  label: "Instagram media transcription",
8164
8164
  aliases: ["instagram transcript", "instagram reel transcript", "ig transcribe"],
8165
8165
  credits: mcToCredits(MC_COSTS.instagram_transcribe),
8166
- unit: "per call",
8167
- notes: "Whisper transcription of selected Instagram audio/video media via fal.ai."
8166
+ unit: "per minute (typical)",
8167
+ notes: "Whisper transcription of selected Instagram audio/video media via fal.ai. A hold is taken, then reconciled to the real transcription cost for the actual audio duration."
8168
8168
  },
8169
8169
  {
8170
8170
  key: "browser_minute",
@@ -8333,6 +8333,8 @@ var init_rates = __esm({
8333
8333
  MEMORY_AI_MARGIN_MULTIPLE = 3;
8334
8334
  SITE_AUDIT_LLM_MARGIN_MULTIPLE = 1.5;
8335
8335
  VIDEO_ANALYSIS_LLM_MARGIN_MULTIPLE = 1.5;
8336
+ MEDIA_TRANSCRIBE_MARGIN_MULTIPLE = 3;
8337
+ MEDIA_TRANSCRIBE_HOLD_MC = MC_COSTS.fb_transcribe * 3;
8336
8338
  MC_PER_USD = SUBSCRIPTION_TIERS["price_1TmiHRS8aAcsk3TGwmSNfNIa"].credits_mc / SUBSCRIPTION_TIERS["price_1TmiHRS8aAcsk3TGwmSNfNIa"].monthly_usd;
8337
8339
  }
8338
8340
  });
@@ -13537,6 +13539,104 @@ var init_kernel_proxy_resolver = __esm({
13537
13539
  }
13538
13540
  });
13539
13541
 
13542
+ // src/youtube/audio-resolver.ts
13543
+ function mediaResolverUrl() {
13544
+ return process.env.FRAMES_SERVICE_URL?.trim() || void 0;
13545
+ }
13546
+ function mediaResolverSecret() {
13547
+ return process.env.FRAMES_SERVICE_SECRET?.trim() || void 0;
13548
+ }
13549
+ function mediaResolverConfigured() {
13550
+ return Boolean(mediaResolverUrl() && mediaResolverSecret());
13551
+ }
13552
+ async function resolveYouTubeAudio(videoId) {
13553
+ const base = mediaResolverUrl();
13554
+ const secret2 = mediaResolverSecret();
13555
+ if (!base || !secret2) return null;
13556
+ const controller = new AbortController();
13557
+ const timer = setTimeout(() => controller.abort(), RESOLVE_TIMEOUT_MS);
13558
+ try {
13559
+ const resp = await fetch(`${base.replace(/\/+$/, "")}/resolve-audio`, {
13560
+ method: "POST",
13561
+ headers: { "content-type": "application/json", "x-frames-secret": secret2 },
13562
+ body: JSON.stringify({ videoUrl: `https://www.youtube.com/watch?v=${videoId}` }),
13563
+ signal: controller.signal
13564
+ });
13565
+ if (!resp.ok) {
13566
+ const detail = await resp.text().catch(() => "");
13567
+ console.warn(`[audio-resolver] ${videoId}: http_${resp.status} ${detail.slice(0, 200)}`);
13568
+ return null;
13569
+ }
13570
+ const data = await resp.json();
13571
+ if (!data.mediaUrl) {
13572
+ console.warn(`[audio-resolver] ${videoId}: response missing mediaUrl`);
13573
+ return null;
13574
+ }
13575
+ return {
13576
+ mediaUrl: data.mediaUrl,
13577
+ durationS: Number(data.durationS) || 0,
13578
+ sizeBytes: Number(data.sizeBytes) || 0,
13579
+ mimeType: data.mimeType ?? "audio/mp4"
13580
+ };
13581
+ } catch (err) {
13582
+ const msg = err instanceof Error ? err.message : String(err);
13583
+ console.warn(`[audio-resolver] ${videoId}: resolve failed \u2014 ${msg}`);
13584
+ return null;
13585
+ } finally {
13586
+ clearTimeout(timer);
13587
+ }
13588
+ }
13589
+ var RESOLVE_TIMEOUT_MS;
13590
+ var init_audio_resolver = __esm({
13591
+ "src/youtube/audio-resolver.ts"() {
13592
+ "use strict";
13593
+ RESOLVE_TIMEOUT_MS = 24e4;
13594
+ }
13595
+ });
13596
+
13597
+ // src/services/media-transcription.ts
13598
+ function transcriptMarkdown(title, text, chunks, durationMs) {
13599
+ const fmtTs2 = (s) => `${Math.floor(s / 60)}:${String(Math.floor(s % 60)).padStart(2, "0")}`;
13600
+ const lines = [title, "", `*Transcribed in ${(durationMs / 1e3).toFixed(1)}s*`, "", "## Full Text", "", text, ""];
13601
+ if (chunks.length) {
13602
+ lines.push("## Timestamped Segments", "");
13603
+ for (const ch of chunks) {
13604
+ lines.push(`**[${fmtTs2(ch.timestamp[0])} -> ${fmtTs2(ch.timestamp[1])}]** ${ch.text.trim()}`, "");
13605
+ }
13606
+ }
13607
+ return lines.join("\n");
13608
+ }
13609
+ async function transcribeMediaUrl(mediaUrl, markdownTitle = "# Media Transcript") {
13610
+ const startMs = Date.now();
13611
+ const result = await import_client3.fal.subscribe("fal-ai/wizper", {
13612
+ input: { audio_url: mediaUrl, task: "transcribe", language: "en" },
13613
+ logs: false,
13614
+ pollInterval: 3e3
13615
+ });
13616
+ const data = result.data;
13617
+ const text = data.text ?? "";
13618
+ const chunks = data.chunks ?? [];
13619
+ const durationMs = Date.now() - startMs;
13620
+ void recordVendorUsage({ vendor: "fal_wizper", model: "fal-ai/wizper", units: durationMs / 1e3, unitType: "compute_sec" });
13621
+ const costUsd = vendorCostUsd("fal_wizper", durationMs / 1e3);
13622
+ return {
13623
+ text,
13624
+ chunks,
13625
+ durationMs,
13626
+ costUsd,
13627
+ markdown: transcriptMarkdown(markdownTitle, text, chunks, durationMs)
13628
+ };
13629
+ }
13630
+ var import_client3;
13631
+ var init_media_transcription = __esm({
13632
+ "src/services/media-transcription.ts"() {
13633
+ "use strict";
13634
+ import_client3 = require("@fal-ai/client");
13635
+ init_cost_telemetry();
13636
+ init_cost_rates();
13637
+ }
13638
+ });
13639
+
13540
13640
  // src/youtube/CaptionFetcher.ts
13541
13641
  async function fetchViaYoutubeTranscript(videoId) {
13542
13642
  try {
@@ -13693,12 +13793,12 @@ async function transcribeWithWizper(videoId, audio, startMs) {
13693
13793
  const falKey = process.env.FAL_KEY;
13694
13794
  if (!falKey) return null;
13695
13795
  try {
13696
- import_client3.fal.config({ credentials: falKey });
13796
+ import_client4.fal.config({ credentials: falKey });
13697
13797
  const ext = audio.mimeType.includes("webm") ? "webm" : audio.mimeType.includes("mp4") ? "m4a" : "audio";
13698
13798
  const audioFile = new File([new Uint8Array(audio.bytes)], `audio.${ext}`, { type: audio.mimeType });
13699
- const uploadedUrl = await import_client3.fal.storage.upload(audioFile);
13799
+ const uploadedUrl = await import_client4.fal.storage.upload(audioFile);
13700
13800
  const subscribeStartMs = Date.now();
13701
- const result = await import_client3.fal.subscribe("fal-ai/wizper", {
13801
+ const result = await import_client4.fal.subscribe("fal-ai/wizper", {
13702
13802
  input: { audio_url: uploadedUrl, task: "transcribe", language: "en", chunk_level: "segment" },
13703
13803
  logs: false,
13704
13804
  pollInterval: 3e3
@@ -13816,6 +13916,28 @@ async function fetchViaKernelSession(videoId) {
13816
13916
  }
13817
13917
  return null;
13818
13918
  }
13919
+ async function fetchViaResolvedAudio(videoId) {
13920
+ if (!mediaResolverConfigured()) return null;
13921
+ return runWithCostContext({ ...currentCostContext(), op: "yt_transcription", subOp: "audio-whisper" }, async () => {
13922
+ const resolved = await resolveYouTubeAudio(videoId);
13923
+ if (!resolved) return null;
13924
+ try {
13925
+ const transcript = await transcribeMediaUrl(resolved.mediaUrl, `# Transcript \u2014 ${videoId}`);
13926
+ if (!transcript.text) return null;
13927
+ const chunks = transcript.chunks.map((c) => ({ timestamp: c.timestamp, text: c.text.trim() }));
13928
+ return {
13929
+ videoId,
13930
+ text: transcript.text,
13931
+ chunks,
13932
+ durationMs: resolved.durationS > 0 ? Math.round(resolved.durationS * 1e3) : transcript.durationMs,
13933
+ method: "audio-whisper"
13934
+ };
13935
+ } catch (err) {
13936
+ console.warn(`[CaptionFetcher] transcription of resolved audio failed for ${videoId}: ${err instanceof Error ? err.message : String(err)}`);
13937
+ return null;
13938
+ }
13939
+ });
13940
+ }
13819
13941
  function finalizeCaptions(result) {
13820
13942
  const chunks = result.chunks.map((c) => {
13821
13943
  const startOk = Number.isFinite(c.timestamp[0]);
@@ -13836,18 +13958,22 @@ async function fetchCaptions(videoId) {
13836
13958
  if (primary) return finalizeCaptions(primary);
13837
13959
  const result = await fetchViaKernelSession(videoId);
13838
13960
  if (result) return finalizeCaptions(result);
13961
+ const viaResolvedAudio = await fetchViaResolvedAudio(videoId);
13962
+ if (viaResolvedAudio) return finalizeCaptions(viaResolvedAudio);
13839
13963
  throw new Error(
13840
- `Could not transcribe ${videoId}. It has no caption track, and its audio stream could not be downloaded \u2014 YouTube permits a single media request per IP, so streams that do not fit in one response are unreachable. Captioned videos of any length still work.`
13964
+ `Could not transcribe ${videoId}: it has no caption track and its audio could not be retrieved.`
13841
13965
  );
13842
13966
  }
13843
- var import_client3, INNERTUBE_CLIENTS, MAX_AUDIO_BYTES, YT_RESIDENTIAL_LOCATION;
13967
+ var import_client4, INNERTUBE_CLIENTS, MAX_AUDIO_BYTES, YT_RESIDENTIAL_LOCATION;
13844
13968
  var init_CaptionFetcher = __esm({
13845
13969
  "src/youtube/CaptionFetcher.ts"() {
13846
13970
  "use strict";
13847
13971
  init_BrowserDriver();
13848
13972
  init_browser_service_env();
13849
13973
  init_kernel_proxy_resolver();
13850
- import_client3 = require("@fal-ai/client");
13974
+ init_audio_resolver();
13975
+ init_media_transcription();
13976
+ import_client4 = require("@fal-ai/client");
13851
13977
  init_cost_telemetry();
13852
13978
  init_cost_context();
13853
13979
  INNERTUBE_CLIENTS = [
@@ -15392,46 +15518,6 @@ var init_schemas3 = __esm({
15392
15518
  }
15393
15519
  });
15394
15520
 
15395
- // src/services/media-transcription.ts
15396
- function transcriptMarkdown(title, text, chunks, durationMs) {
15397
- const fmtTs2 = (s) => `${Math.floor(s / 60)}:${String(Math.floor(s % 60)).padStart(2, "0")}`;
15398
- const lines = [title, "", `*Transcribed in ${(durationMs / 1e3).toFixed(1)}s*`, "", "## Full Text", "", text, ""];
15399
- if (chunks.length) {
15400
- lines.push("## Timestamped Segments", "");
15401
- for (const ch of chunks) {
15402
- lines.push(`**[${fmtTs2(ch.timestamp[0])} -> ${fmtTs2(ch.timestamp[1])}]** ${ch.text.trim()}`, "");
15403
- }
15404
- }
15405
- return lines.join("\n");
15406
- }
15407
- async function transcribeMediaUrl(mediaUrl, markdownTitle = "# Media Transcript") {
15408
- const startMs = Date.now();
15409
- const result = await import_client4.fal.subscribe("fal-ai/wizper", {
15410
- input: { audio_url: mediaUrl, task: "transcribe", language: "en" },
15411
- logs: false,
15412
- pollInterval: 3e3
15413
- });
15414
- const data = result.data;
15415
- const text = data.text ?? "";
15416
- const chunks = data.chunks ?? [];
15417
- const durationMs = Date.now() - startMs;
15418
- void recordVendorUsage({ vendor: "fal_wizper", model: "fal-ai/wizper", units: durationMs / 1e3, unitType: "compute_sec" });
15419
- return {
15420
- text,
15421
- chunks,
15422
- durationMs,
15423
- markdown: transcriptMarkdown(markdownTitle, text, chunks, durationMs)
15424
- };
15425
- }
15426
- var import_client4;
15427
- var init_media_transcription = __esm({
15428
- "src/services/media-transcription.ts"() {
15429
- "use strict";
15430
- import_client4 = require("@fal-ai/client");
15431
- init_cost_telemetry();
15432
- }
15433
- });
15434
-
15435
15521
  // src/api/facebook-ad-routes.ts
15436
15522
  function invalidRequest(message) {
15437
15523
  return { error_code: "invalid_request", message };
@@ -15483,6 +15569,7 @@ var init_facebook_ad_routes = __esm({
15483
15569
  import_zod16 = require("zod");
15484
15570
  init_db();
15485
15571
  init_rates();
15572
+ init_cost_rates();
15486
15573
  init_BrowserDriver();
15487
15574
  init_FacebookAdExtractor();
15488
15575
  init_FacebookAdGraphql();
@@ -15633,16 +15720,16 @@ var init_facebook_ad_routes = __esm({
15633
15720
  });
15634
15721
  if (!gate.ok) return c.json(concurrencyLimitExceededResponse(gate), 429, { "Retry-After": String(gate.retryAfterSeconds) });
15635
15722
  import_client5.fal.config({ credentials: process.env.FAL_KEY });
15636
- const holdMc = MC_COSTS.fb_transcribe * 5;
15723
+ const holdMc = MEDIA_TRANSCRIBE_HOLD_MC;
15637
15724
  let debited = false;
15638
15725
  try {
15639
15726
  const { ok, balance_mc } = await debitMc(fbUser.id, holdMc, LedgerOperation.FB_TRANSCRIBE, videoUrl);
15640
15727
  if (!ok) return c.json(insufficientBalanceResponse(balance_mc, holdMc), 402);
15641
15728
  debited = true;
15642
15729
  const transcript = await transcribeFacebookVideoUrl(videoUrl, "# Facebook Ad Transcript");
15643
- const actualMins = Math.max(1, Math.ceil((Number.isFinite(transcript.durationMs) ? transcript.durationMs / 1e3 : 60) / 60));
15644
- const diff = holdMc - MC_COSTS.fb_transcribe * actualMins;
15645
- if (diff > 0) await creditMc(fbUser.id, diff, LedgerOperation.FB_TRANSCRIBE_REFUND, "overestimate refund");
15730
+ const settleMc = Math.round(transcript.costUsd * MEDIA_TRANSCRIBE_MARGIN_MULTIPLE * MC_PER_USD);
15731
+ const diff = holdMc - settleMc;
15732
+ if (diff > 0) await creditMc(fbUser.id, diff, LedgerOperation.FB_TRANSCRIBE_REFUND, "real-cost reconcile");
15646
15733
  else if (diff < 0) await debitMc(fbUser.id, -diff, LedgerOperation.FB_TRANSCRIBE, videoUrl);
15647
15734
  await logRequestEvent({ userId: fbUser.id, source: "facebook_transcribe", status: "done", query: videoUrl, resultCount: transcript.chunks.length, result: { text: transcript.text, chunks: transcript.chunks, durationMs: transcript.durationMs } });
15648
15735
  return c.json(transcript);
@@ -15678,20 +15765,22 @@ var init_facebook_ad_routes = __esm({
15678
15765
  if (!gate.ok) return c.json(concurrencyLimitExceededResponse(gate), 429, { "Retry-After": String(gate.retryAfterSeconds) });
15679
15766
  import_client5.fal.config({ credentials: process.env.FAL_KEY });
15680
15767
  const driver = new BrowserDriver();
15681
- const holdMc = MC_COSTS.fb_transcribe * 5;
15768
+ const holdMc = MEDIA_TRANSCRIBE_HOLD_MC;
15682
15769
  let debited = false;
15683
15770
  try {
15684
15771
  const { ok, balance_mc } = await debitMc(fbUser.id, holdMc, LedgerOperation.FB_TRANSCRIBE, sourceUrl.href);
15685
15772
  if (!ok) return c.json(insufficientBalanceResponse(balance_mc, holdMc), 402);
15686
15773
  debited = true;
15774
+ const kernelStartMs = Date.now();
15687
15775
  await driver.launch(await kernelLaunchOptsResidential());
15688
15776
  await driver.navigateTo(sourceUrl.href);
15689
15777
  const page = driver.getPage();
15690
15778
  const video = await extractFacebookOrganicVideoFromPage(page, sourceUrl.href, body.quality);
15779
+ const kernelMs = Date.now() - kernelStartMs;
15691
15780
  const transcript = await transcribeFacebookVideoUrl(video.videoUrl, "# Facebook Organic Video Transcript");
15692
- const actualMins = Math.max(1, Math.ceil((Number.isFinite(transcript.durationMs) ? transcript.durationMs / 1e3 : 60) / 60));
15693
- const diff = holdMc - MC_COSTS.fb_transcribe * actualMins;
15694
- if (diff > 0) await creditMc(fbUser.id, diff, LedgerOperation.FB_TRANSCRIBE_REFUND, "overestimate refund");
15781
+ const settleMc = Math.round((transcript.costUsd + kernelCostUsd(kernelMs, true)) * MEDIA_TRANSCRIBE_MARGIN_MULTIPLE * MC_PER_USD);
15782
+ const diff = holdMc - settleMc;
15783
+ if (diff > 0) await creditMc(fbUser.id, diff, LedgerOperation.FB_TRANSCRIBE_REFUND, "real-cost reconcile");
15695
15784
  else if (diff < 0) await debitMc(fbUser.id, -diff, LedgerOperation.FB_TRANSCRIBE, sourceUrl.href);
15696
15785
  const result = {
15697
15786
  ...video,
@@ -16073,6 +16162,7 @@ var init_google_ads_routes = __esm({
16073
16162
  init_url_utils();
16074
16163
  init_concurrency_gates();
16075
16164
  init_media_transcription();
16165
+ init_cost_context();
16076
16166
  GoogleAdsSearchBodySchema = import_zod17.z.object({
16077
16167
  query: import_zod17.z.string().trim().min(1, "query is required"),
16078
16168
  region: import_zod17.z.string().trim().toUpperCase().length(2).optional(),
@@ -16104,9 +16194,11 @@ var init_google_ads_routes = __esm({
16104
16194
  const { ok, balance_mc } = await debitMc(user.id, MC_COSTS.google_ads_search, LedgerOperation.GOOGLE_ADS_SEARCH, body.query.trim());
16105
16195
  if (!ok) return c.json(insufficientBalanceResponse(balance_mc, MC_COSTS.google_ads_search), 402);
16106
16196
  debited = true;
16107
- await driver.launch(await kernelLaunchOptsResidential2());
16108
- const extractor = new GoogleAdsExtractor(driver);
16109
- const result = await extractor.search(body.query.trim(), region, maxResults);
16197
+ const result = await runWithCostContext({ ...currentCostContext(), op: "google_ads_search", userId: user.id }, async () => {
16198
+ await driver.launch(await kernelLaunchOptsResidential2());
16199
+ const extractor = new GoogleAdsExtractor(driver);
16200
+ return extractor.search(body.query.trim(), region, maxResults);
16201
+ });
16110
16202
  if (result.advertisers.length === 0) {
16111
16203
  await creditMc(user.id, MC_COSTS.google_ads_search, LedgerOperation.GOOGLE_ADS_SEARCH_REFUND, "empty result");
16112
16204
  refunded = true;
@@ -16142,9 +16234,11 @@ var init_google_ads_routes = __esm({
16142
16234
  const { ok, balance_mc } = await debitMc(user.id, MC_COSTS.google_ads_intel, LedgerOperation.GOOGLE_ADS_INTEL, queryLabel);
16143
16235
  if (!ok) return c.json(insufficientBalanceResponse(balance_mc, MC_COSTS.google_ads_intel), 402);
16144
16236
  debited = true;
16145
- await driver.launch(await kernelLaunchOptsResidential2());
16146
- const extractor = new GoogleAdsExtractor(driver);
16147
- const result = await extractor.pageIntel({ advertiserId: body.advertiserId, domain: body.domain, region, maxAds });
16237
+ const result = await runWithCostContext({ ...currentCostContext(), op: "google_ads_intel", userId: user.id }, async () => {
16238
+ await driver.launch(await kernelLaunchOptsResidential2());
16239
+ const extractor = new GoogleAdsExtractor(driver);
16240
+ return extractor.pageIntel({ advertiserId: body.advertiserId, domain: body.domain, region, maxAds });
16241
+ });
16148
16242
  if (result.ads.length === 0) {
16149
16243
  await creditMc(user.id, MC_COSTS.google_ads_intel, LedgerOperation.GOOGLE_ADS_INTEL_REFUND, "empty result");
16150
16244
  refunded = true;
@@ -16175,16 +16269,16 @@ var init_google_ads_routes = __esm({
16175
16269
  const gate = await acquireConcurrencyGate(user, "google_ads_transcribe", { reuseLockId: c.req.header("x-mcp-scraper-concurrency-lock"), metadata: { videoUrl } });
16176
16270
  if (!gate.ok) return c.json(concurrencyLimitExceededResponse(gate), 429, { "Retry-After": String(gate.retryAfterSeconds) });
16177
16271
  import_client6.fal.config({ credentials: process.env.FAL_KEY });
16178
- const holdMc = MC_COSTS.google_ads_transcribe * 5;
16272
+ const holdMc = MEDIA_TRANSCRIBE_HOLD_MC;
16179
16273
  let debited = false;
16180
16274
  try {
16181
16275
  const { ok, balance_mc } = await debitMc(user.id, holdMc, LedgerOperation.GOOGLE_ADS_TRANSCRIBE, videoUrl);
16182
16276
  if (!ok) return c.json(insufficientBalanceResponse(balance_mc, holdMc), 402);
16183
16277
  debited = true;
16184
- const transcript = await transcribeMediaUrl(videoUrl, "# Google Ad Transcript");
16185
- const actualMins = Math.max(1, Math.ceil((Number.isFinite(transcript.durationMs) ? transcript.durationMs / 1e3 : 60) / 60));
16186
- const diff = holdMc - MC_COSTS.google_ads_transcribe * actualMins;
16187
- if (diff > 0) await creditMc(user.id, diff, LedgerOperation.GOOGLE_ADS_TRANSCRIBE_REFUND, "overestimate refund");
16278
+ const transcript = await runWithCostContext({ ...currentCostContext(), op: "google_ads_transcribe", userId: user.id }, () => transcribeMediaUrl(videoUrl, "# Google Ad Transcript"));
16279
+ const settleMc = Math.round(transcript.costUsd * MEDIA_TRANSCRIBE_MARGIN_MULTIPLE * MC_PER_USD);
16280
+ const diff = holdMc - settleMc;
16281
+ if (diff > 0) await creditMc(user.id, diff, LedgerOperation.GOOGLE_ADS_TRANSCRIBE_REFUND, "real-cost reconcile");
16188
16282
  else if (diff < 0) await debitMc(user.id, -diff, LedgerOperation.GOOGLE_ADS_TRANSCRIBE, videoUrl);
16189
16283
  await logRequestEvent({ userId: user.id, source: "google_ads_transcribe", status: "done", query: videoUrl, resultCount: transcript.chunks.length, result: { text: transcript.text, chunks: transcript.chunks, durationMs: transcript.durationMs } });
16190
16284
  return c.json(transcript);
@@ -16852,10 +16946,15 @@ var init_instagram_routes = __esm({
16852
16946
  if (!audioUrl) {
16853
16947
  warnings.push("Transcript requested, but no audio/video track was available.");
16854
16948
  } else {
16855
- const tx = await debitMc(user.id, MC_COSTS.instagram_transcribe, LedgerOperation.INSTAGRAM_TRANSCRIBE, sourceUrl.href);
16856
- if (!tx.ok) return c.json(insufficientBalanceResponse(tx.balance_mc, MC_COSTS.instagram_transcribe), 402);
16949
+ const holdMc = MEDIA_TRANSCRIBE_HOLD_MC;
16950
+ const tx = await debitMc(user.id, holdMc, LedgerOperation.INSTAGRAM_TRANSCRIBE, sourceUrl.href);
16951
+ if (!tx.ok) return c.json(insufficientBalanceResponse(tx.balance_mc, holdMc), 402);
16857
16952
  transcriptDebited = true;
16858
16953
  transcript = await transcribeMediaUrl(audioUrl, "# Instagram Media Transcript");
16954
+ const settleMc = Math.round(transcript.costUsd * MEDIA_TRANSCRIBE_MARGIN_MULTIPLE * MC_PER_USD);
16955
+ const diff = holdMc - settleMc;
16956
+ if (diff > 0) await creditMc(user.id, diff, LedgerOperation.INSTAGRAM_TRANSCRIBE_REFUND, "real-cost reconcile");
16957
+ else if (diff < 0) await debitMc(user.id, -diff, LedgerOperation.INSTAGRAM_TRANSCRIBE, sourceUrl.href);
16859
16958
  }
16860
16959
  }
16861
16960
  const result = {
@@ -16870,7 +16969,7 @@ var init_instagram_routes = __esm({
16870
16969
  return c.json(result);
16871
16970
  } catch (err) {
16872
16971
  const msg = err instanceof Error ? err.message : String(err);
16873
- if (transcriptDebited) await creditMc(user.id, MC_COSTS.instagram_transcribe, LedgerOperation.INSTAGRAM_TRANSCRIBE_REFUND, "failed call");
16972
+ if (transcriptDebited) await creditMc(user.id, MEDIA_TRANSCRIBE_HOLD_MC, LedgerOperation.INSTAGRAM_TRANSCRIBE_REFUND, "failed call");
16874
16973
  if (mediaDebited) await creditMc(user.id, MC_COSTS.instagram_media, LedgerOperation.INSTAGRAM_MEDIA_REFUND, "failed call");
16875
16974
  await logRequestEvent({ userId: user.id, source: "instagram_media_download", status: "failed", query: sourceUrl.href, error: msg });
16876
16975
  if (msg.toLowerCase().includes("blocked") || msg.toLowerCase().includes("captcha")) return c.json({ error: msg }, 503);
@@ -17011,10 +17110,23 @@ var init_reddit_routes = __esm({
17011
17110
  result.comments = result.comments.slice(0, body.maxComments);
17012
17111
  result.numComments = result.comments.length;
17013
17112
  }
17113
+ const commentCount = Array.isArray(result.comments) ? result.comments.length : 0;
17114
+ commentDebitMc = commentCount * MC_COSTS.reddit_comment;
17115
+ if (commentDebitMc > 0) {
17116
+ const commentDebit = await debitMc(user.id, commentDebitMc, LedgerOperation.REDDIT_COMMENT, body.url);
17117
+ if (!commentDebit.ok) {
17118
+ const affordable = Math.floor((commentDebit.balance_mc ?? 0) / MC_COSTS.reddit_comment);
17119
+ result.comments = result.comments.slice(0, affordable);
17120
+ result.numComments = affordable;
17121
+ commentDebitMc = affordable * MC_COSTS.reddit_comment;
17122
+ if (commentDebitMc > 0) await debitMc(user.id, commentDebitMc, LedgerOperation.REDDIT_COMMENT, body.url);
17123
+ }
17124
+ }
17014
17125
  await logRequestEvent({ userId: user.id, source: "reddit_thread", status: "done", query: body.url, resultCount: Number(result.numComments ?? 0), result });
17015
17126
  return c.json(result);
17016
17127
  } catch (err) {
17017
17128
  if (debited && !refunded) await creditMc(user.id, MC_COSTS.reddit_thread, LedgerOperation.REDDIT_THREAD_REFUND, "failed call");
17129
+ if (commentDebitMc > 0) await creditMc(user.id, commentDebitMc, LedgerOperation.REDDIT_COMMENT_REFUND, "failed call");
17018
17130
  const msg = err instanceof Error ? err.message : String(err);
17019
17131
  await logRequestEvent({ userId: user.id, source: "reddit_thread", status: "failed", query: body.url, error: msg });
17020
17132
  return c.json({ error: msg }, 500);
@@ -27287,7 +27399,7 @@ var PACKAGE_VERSION;
27287
27399
  var init_version = __esm({
27288
27400
  "src/version.ts"() {
27289
27401
  "use strict";
27290
- PACKAGE_VERSION = "0.4.12";
27402
+ PACKAGE_VERSION = "0.4.13";
27291
27403
  }
27292
27404
  });
27293
27405
 
@@ -38373,6 +38485,11 @@ function opForCostPath(p) {
38373
38485
  return "fb_ad";
38374
38486
  }
38375
38487
  if (p.startsWith("/instagram/")) return "instagram";
38488
+ if (p.startsWith("/google-ads/")) {
38489
+ if (p.endsWith("/transcribe")) return "google_ads_transcribe";
38490
+ if (p.endsWith("/search")) return "google_ads_search";
38491
+ return "google_ads_intel";
38492
+ }
38376
38493
  if (p === "/reddit/thread") return "reddit_thread";
38377
38494
  if (p.startsWith("/trustpilot/")) return "trustpilot_reviews";
38378
38495
  if (p.startsWith("/g2/")) return "g2_reviews";