mcp-scraper 0.4.12 → 0.4.13

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (35) hide show
  1. package/dist/bin/api-server.cjs +196 -79
  2. package/dist/bin/api-server.cjs.map +1 -1
  3. package/dist/bin/api-server.js +2 -2
  4. package/dist/bin/mcp-scraper-cli.cjs +1 -1
  5. package/dist/bin/mcp-scraper-cli.cjs.map +1 -1
  6. package/dist/bin/mcp-scraper-cli.js +1 -1
  7. package/dist/bin/mcp-scraper-install.cjs +1 -1
  8. package/dist/bin/mcp-scraper-install.cjs.map +1 -1
  9. package/dist/bin/mcp-scraper-install.js +1 -1
  10. package/dist/bin/mcp-stdio-server.cjs +1 -1
  11. package/dist/bin/mcp-stdio-server.cjs.map +1 -1
  12. package/dist/bin/mcp-stdio-server.js +2 -2
  13. package/dist/bin/paa-harvest.js +1 -1
  14. package/dist/{chunk-2YY46QYT.js → chunk-AUCXKRRH.js} +3 -1
  15. package/dist/{chunk-6EXP6DQG.js → chunk-EA2AANFI.js} +2 -2
  16. package/dist/chunk-O2D5HGKF.js +7 -0
  17. package/dist/chunk-O2D5HGKF.js.map +1 -0
  18. package/dist/{chunk-EMY7ELRU.js → chunk-SIPXEFTR.js} +14 -10
  19. package/dist/chunk-SIPXEFTR.js.map +1 -0
  20. package/dist/{chunk-ONIOF5XW.js → chunk-VQV7MB2S.js} +2 -2
  21. package/dist/index.js +1 -1
  22. package/dist/{server-UNID3SJU.js → server-HNZG36IP.js} +178 -72
  23. package/dist/server-HNZG36IP.js.map +1 -0
  24. package/dist/{site-extract-repository-HTIY52MW.js → site-extract-repository-6X2C7RLF.js} +3 -3
  25. package/dist/{worker-HTYZAYGB.js → worker-WCSKUAOT.js} +3 -3
  26. package/package.json +1 -1
  27. package/dist/chunk-EMY7ELRU.js.map +0 -1
  28. package/dist/chunk-SHXJQQOH.js +0 -7
  29. package/dist/chunk-SHXJQQOH.js.map +0 -1
  30. package/dist/server-UNID3SJU.js.map +0 -1
  31. /package/dist/{chunk-2YY46QYT.js.map → chunk-AUCXKRRH.js.map} +0 -0
  32. /package/dist/{chunk-6EXP6DQG.js.map → chunk-EA2AANFI.js.map} +0 -0
  33. /package/dist/{chunk-ONIOF5XW.js.map → chunk-VQV7MB2S.js.map} +0 -0
  34. /package/dist/{site-extract-repository-HTIY52MW.js.map → site-extract-repository-6X2C7RLF.js.map} +0 -0
  35. /package/dist/{worker-HTYZAYGB.js.map → worker-WCSKUAOT.js.map} +0 -0
@@ -11,12 +11,14 @@ import {
11
11
  currentCostContext,
12
12
  deleteKernelProxyId,
13
13
  harvest,
14
+ kernelCostUsd,
14
15
  migrateCostTelemetry,
15
16
  recordKernelSession,
16
17
  recordVendorUsage,
17
18
  resolveKernelProxyId,
18
- runWithCostContext
19
- } from "./chunk-2YY46QYT.js";
19
+ runWithCostContext,
20
+ vendorCostUsd
21
+ } from "./chunk-AUCXKRRH.js";
20
22
  import {
21
23
  HttpMcpToolExecutor,
22
24
  MemoryMcpToolExecutor,
@@ -31,7 +33,7 @@ import {
31
33
  registerSerpIntelligenceCaptureTools,
32
34
  sanitizeAttempts,
33
35
  sanitizeHarvestResult
34
- } from "./chunk-6EXP6DQG.js";
36
+ } from "./chunk-EA2AANFI.js";
35
37
  import {
36
38
  auditImages,
37
39
  getBlobStore
@@ -65,7 +67,7 @@ import {
65
67
  RawMapsOverviewSchema,
66
68
  RawMapsReviewStatsSchema
67
69
  } from "./chunk-XGIPATLV.js";
68
- import "./chunk-SHXJQQOH.js";
70
+ import "./chunk-O2D5HGKF.js";
69
71
  import {
70
72
  completeExtractJob,
71
73
  countSuccessfulPages,
@@ -76,7 +78,7 @@ import {
76
78
  saveExtractPages,
77
79
  setExtractJobTotal,
78
80
  settleExtractJob
79
- } from "./chunk-ONIOF5XW.js";
81
+ } from "./chunk-VQV7MB2S.js";
80
82
  import {
81
83
  BROWSER_OPEN_MIN_BALANCE_MC,
82
84
  CONCURRENCY_PRICE_ID,
@@ -86,6 +88,8 @@ import {
86
88
  MC_COSTS,
87
89
  MC_PER_CREDIT,
88
90
  MC_PER_USD,
91
+ MEDIA_TRANSCRIBE_HOLD_MC,
92
+ MEDIA_TRANSCRIBE_MARGIN_MULTIPLE,
89
93
  MEMORY_AI_MARGIN_MULTIPLE,
90
94
  MEMORY_PLANS,
91
95
  MEMORY_PLAN_QUOTA,
@@ -97,7 +101,7 @@ import {
97
101
  browserActiveCostMc,
98
102
  concurrencySlotBillingInfo,
99
103
  insufficientBalanceResponse
100
- } from "./chunk-EMY7ELRU.js";
104
+ } from "./chunk-SIPXEFTR.js";
101
105
  import {
102
106
  CaptchaError,
103
107
  RECAPTCHA_INSTRUCTIONS,
@@ -8699,8 +8703,92 @@ async function ytHarvest(rawOptions) {
8699
8703
  );
8700
8704
  }
8701
8705
 
8702
- // src/youtube/CaptionFetcher.ts
8706
+ // src/youtube/audio-resolver.ts
8707
+ function mediaResolverUrl() {
8708
+ return process.env.FRAMES_SERVICE_URL?.trim() || void 0;
8709
+ }
8710
+ function mediaResolverSecret() {
8711
+ return process.env.FRAMES_SERVICE_SECRET?.trim() || void 0;
8712
+ }
8713
+ function mediaResolverConfigured() {
8714
+ return Boolean(mediaResolverUrl() && mediaResolverSecret());
8715
+ }
8716
+ var RESOLVE_TIMEOUT_MS = 24e4;
8717
+ async function resolveYouTubeAudio(videoId) {
8718
+ const base = mediaResolverUrl();
8719
+ const secret2 = mediaResolverSecret();
8720
+ if (!base || !secret2) return null;
8721
+ const controller = new AbortController();
8722
+ const timer = setTimeout(() => controller.abort(), RESOLVE_TIMEOUT_MS);
8723
+ try {
8724
+ const resp = await fetch(`${base.replace(/\/+$/, "")}/resolve-audio`, {
8725
+ method: "POST",
8726
+ headers: { "content-type": "application/json", "x-frames-secret": secret2 },
8727
+ body: JSON.stringify({ videoUrl: `https://www.youtube.com/watch?v=${videoId}` }),
8728
+ signal: controller.signal
8729
+ });
8730
+ if (!resp.ok) {
8731
+ const detail = await resp.text().catch(() => "");
8732
+ console.warn(`[audio-resolver] ${videoId}: http_${resp.status} ${detail.slice(0, 200)}`);
8733
+ return null;
8734
+ }
8735
+ const data = await resp.json();
8736
+ if (!data.mediaUrl) {
8737
+ console.warn(`[audio-resolver] ${videoId}: response missing mediaUrl`);
8738
+ return null;
8739
+ }
8740
+ return {
8741
+ mediaUrl: data.mediaUrl,
8742
+ durationS: Number(data.durationS) || 0,
8743
+ sizeBytes: Number(data.sizeBytes) || 0,
8744
+ mimeType: data.mimeType ?? "audio/mp4"
8745
+ };
8746
+ } catch (err) {
8747
+ const msg = err instanceof Error ? err.message : String(err);
8748
+ console.warn(`[audio-resolver] ${videoId}: resolve failed \u2014 ${msg}`);
8749
+ return null;
8750
+ } finally {
8751
+ clearTimeout(timer);
8752
+ }
8753
+ }
8754
+
8755
+ // src/services/media-transcription.ts
8703
8756
  import { fal } from "@fal-ai/client";
8757
+ function transcriptMarkdown(title, text, chunks, durationMs) {
8758
+ const fmtTs2 = (s) => `${Math.floor(s / 60)}:${String(Math.floor(s % 60)).padStart(2, "0")}`;
8759
+ const lines = [title, "", `*Transcribed in ${(durationMs / 1e3).toFixed(1)}s*`, "", "## Full Text", "", text, ""];
8760
+ if (chunks.length) {
8761
+ lines.push("## Timestamped Segments", "");
8762
+ for (const ch of chunks) {
8763
+ lines.push(`**[${fmtTs2(ch.timestamp[0])} -> ${fmtTs2(ch.timestamp[1])}]** ${ch.text.trim()}`, "");
8764
+ }
8765
+ }
8766
+ return lines.join("\n");
8767
+ }
8768
+ async function transcribeMediaUrl(mediaUrl, markdownTitle = "# Media Transcript") {
8769
+ const startMs = Date.now();
8770
+ const result = await fal.subscribe("fal-ai/wizper", {
8771
+ input: { audio_url: mediaUrl, task: "transcribe", language: "en" },
8772
+ logs: false,
8773
+ pollInterval: 3e3
8774
+ });
8775
+ const data = result.data;
8776
+ const text = data.text ?? "";
8777
+ const chunks = data.chunks ?? [];
8778
+ const durationMs = Date.now() - startMs;
8779
+ void recordVendorUsage({ vendor: "fal_wizper", model: "fal-ai/wizper", units: durationMs / 1e3, unitType: "compute_sec" });
8780
+ const costUsd = vendorCostUsd("fal_wizper", durationMs / 1e3);
8781
+ return {
8782
+ text,
8783
+ chunks,
8784
+ durationMs,
8785
+ costUsd,
8786
+ markdown: transcriptMarkdown(markdownTitle, text, chunks, durationMs)
8787
+ };
8788
+ }
8789
+
8790
+ // src/youtube/CaptionFetcher.ts
8791
+ import { fal as fal2 } from "@fal-ai/client";
8704
8792
  async function fetchViaYoutubeTranscript(videoId) {
8705
8793
  try {
8706
8794
  const { YoutubeTranscript } = await import("youtube-transcript");
@@ -8862,12 +8950,12 @@ async function transcribeWithWizper(videoId, audio, startMs) {
8862
8950
  const falKey = process.env.FAL_KEY;
8863
8951
  if (!falKey) return null;
8864
8952
  try {
8865
- fal.config({ credentials: falKey });
8953
+ fal2.config({ credentials: falKey });
8866
8954
  const ext = audio.mimeType.includes("webm") ? "webm" : audio.mimeType.includes("mp4") ? "m4a" : "audio";
8867
8955
  const audioFile = new File([new Uint8Array(audio.bytes)], `audio.${ext}`, { type: audio.mimeType });
8868
- const uploadedUrl = await fal.storage.upload(audioFile);
8956
+ const uploadedUrl = await fal2.storage.upload(audioFile);
8869
8957
  const subscribeStartMs = Date.now();
8870
- const result = await fal.subscribe("fal-ai/wizper", {
8958
+ const result = await fal2.subscribe("fal-ai/wizper", {
8871
8959
  input: { audio_url: uploadedUrl, task: "transcribe", language: "en", chunk_level: "segment" },
8872
8960
  logs: false,
8873
8961
  pollInterval: 3e3
@@ -8986,6 +9074,28 @@ async function fetchViaKernelSession(videoId) {
8986
9074
  }
8987
9075
  return null;
8988
9076
  }
9077
+ async function fetchViaResolvedAudio(videoId) {
9078
+ if (!mediaResolverConfigured()) return null;
9079
+ return runWithCostContext({ ...currentCostContext(), op: "yt_transcription", subOp: "audio-whisper" }, async () => {
9080
+ const resolved = await resolveYouTubeAudio(videoId);
9081
+ if (!resolved) return null;
9082
+ try {
9083
+ const transcript = await transcribeMediaUrl(resolved.mediaUrl, `# Transcript \u2014 ${videoId}`);
9084
+ if (!transcript.text) return null;
9085
+ const chunks = transcript.chunks.map((c) => ({ timestamp: c.timestamp, text: c.text.trim() }));
9086
+ return {
9087
+ videoId,
9088
+ text: transcript.text,
9089
+ chunks,
9090
+ durationMs: resolved.durationS > 0 ? Math.round(resolved.durationS * 1e3) : transcript.durationMs,
9091
+ method: "audio-whisper"
9092
+ };
9093
+ } catch (err) {
9094
+ console.warn(`[CaptionFetcher] transcription of resolved audio failed for ${videoId}: ${err instanceof Error ? err.message : String(err)}`);
9095
+ return null;
9096
+ }
9097
+ });
9098
+ }
8989
9099
  function finalizeCaptions(result) {
8990
9100
  const chunks = result.chunks.map((c) => {
8991
9101
  const startOk = Number.isFinite(c.timestamp[0]);
@@ -9006,8 +9116,10 @@ async function fetchCaptions(videoId) {
9006
9116
  if (primary) return finalizeCaptions(primary);
9007
9117
  const result = await fetchViaKernelSession(videoId);
9008
9118
  if (result) return finalizeCaptions(result);
9119
+ const viaResolvedAudio = await fetchViaResolvedAudio(videoId);
9120
+ if (viaResolvedAudio) return finalizeCaptions(viaResolvedAudio);
9009
9121
  throw new Error(
9010
- `Could not transcribe ${videoId}. It has no caption track, and its audio stream could not be downloaded \u2014 YouTube permits a single media request per IP, so streams that do not fit in one response are unreachable. Captioned videos of any length still work.`
9122
+ `Could not transcribe ${videoId}: it has no caption track and its audio could not be retrieved.`
9011
9123
  );
9012
9124
  }
9013
9125
 
@@ -10374,41 +10486,6 @@ async function extractFacebookOrganicVideoFromPage(page, sourceUrl, quality = "b
10374
10486
 
10375
10487
  // src/api/facebook-ad-routes.ts
10376
10488
  import { fal as fal3 } from "@fal-ai/client";
10377
-
10378
- // src/services/media-transcription.ts
10379
- import { fal as fal2 } from "@fal-ai/client";
10380
- function transcriptMarkdown(title, text, chunks, durationMs) {
10381
- const fmtTs2 = (s) => `${Math.floor(s / 60)}:${String(Math.floor(s % 60)).padStart(2, "0")}`;
10382
- const lines = [title, "", `*Transcribed in ${(durationMs / 1e3).toFixed(1)}s*`, "", "## Full Text", "", text, ""];
10383
- if (chunks.length) {
10384
- lines.push("## Timestamped Segments", "");
10385
- for (const ch of chunks) {
10386
- lines.push(`**[${fmtTs2(ch.timestamp[0])} -> ${fmtTs2(ch.timestamp[1])}]** ${ch.text.trim()}`, "");
10387
- }
10388
- }
10389
- return lines.join("\n");
10390
- }
10391
- async function transcribeMediaUrl(mediaUrl, markdownTitle = "# Media Transcript") {
10392
- const startMs = Date.now();
10393
- const result = await fal2.subscribe("fal-ai/wizper", {
10394
- input: { audio_url: mediaUrl, task: "transcribe", language: "en" },
10395
- logs: false,
10396
- pollInterval: 3e3
10397
- });
10398
- const data = result.data;
10399
- const text = data.text ?? "";
10400
- const chunks = data.chunks ?? [];
10401
- const durationMs = Date.now() - startMs;
10402
- void recordVendorUsage({ vendor: "fal_wizper", model: "fal-ai/wizper", units: durationMs / 1e3, unitType: "compute_sec" });
10403
- return {
10404
- text,
10405
- chunks,
10406
- durationMs,
10407
- markdown: transcriptMarkdown(markdownTitle, text, chunks, durationMs)
10408
- };
10409
- }
10410
-
10411
- // src/api/facebook-ad-routes.ts
10412
10489
  var FacebookAdBodySchema = z13.object({
10413
10490
  url: z13.string().trim().optional(),
10414
10491
  libraryId: z13.string().trim().optional(),
@@ -10589,16 +10666,16 @@ facebookAdApp.post("/transcribe", createApiKeyAuth(), async (c) => {
10589
10666
  });
10590
10667
  if (!gate.ok) return c.json(concurrencyLimitExceededResponse(gate), 429, { "Retry-After": String(gate.retryAfterSeconds) });
10591
10668
  fal3.config({ credentials: process.env.FAL_KEY });
10592
- const holdMc = MC_COSTS.fb_transcribe * 5;
10669
+ const holdMc = MEDIA_TRANSCRIBE_HOLD_MC;
10593
10670
  let debited = false;
10594
10671
  try {
10595
10672
  const { ok, balance_mc } = await debitMc(fbUser.id, holdMc, LedgerOperation.FB_TRANSCRIBE, videoUrl);
10596
10673
  if (!ok) return c.json(insufficientBalanceResponse(balance_mc, holdMc), 402);
10597
10674
  debited = true;
10598
10675
  const transcript = await transcribeFacebookVideoUrl(videoUrl, "# Facebook Ad Transcript");
10599
- const actualMins = Math.max(1, Math.ceil((Number.isFinite(transcript.durationMs) ? transcript.durationMs / 1e3 : 60) / 60));
10600
- const diff = holdMc - MC_COSTS.fb_transcribe * actualMins;
10601
- if (diff > 0) await creditMc(fbUser.id, diff, LedgerOperation.FB_TRANSCRIBE_REFUND, "overestimate refund");
10676
+ const settleMc = Math.round(transcript.costUsd * MEDIA_TRANSCRIBE_MARGIN_MULTIPLE * MC_PER_USD);
10677
+ const diff = holdMc - settleMc;
10678
+ if (diff > 0) await creditMc(fbUser.id, diff, LedgerOperation.FB_TRANSCRIBE_REFUND, "real-cost reconcile");
10602
10679
  else if (diff < 0) await debitMc(fbUser.id, -diff, LedgerOperation.FB_TRANSCRIBE, videoUrl);
10603
10680
  await logRequestEvent({ userId: fbUser.id, source: "facebook_transcribe", status: "done", query: videoUrl, resultCount: transcript.chunks.length, result: { text: transcript.text, chunks: transcript.chunks, durationMs: transcript.durationMs } });
10604
10681
  return c.json(transcript);
@@ -10634,20 +10711,22 @@ facebookAdApp.post("/video-transcribe", createApiKeyAuth(), async (c) => {
10634
10711
  if (!gate.ok) return c.json(concurrencyLimitExceededResponse(gate), 429, { "Retry-After": String(gate.retryAfterSeconds) });
10635
10712
  fal3.config({ credentials: process.env.FAL_KEY });
10636
10713
  const driver = new BrowserDriver();
10637
- const holdMc = MC_COSTS.fb_transcribe * 5;
10714
+ const holdMc = MEDIA_TRANSCRIBE_HOLD_MC;
10638
10715
  let debited = false;
10639
10716
  try {
10640
10717
  const { ok, balance_mc } = await debitMc(fbUser.id, holdMc, LedgerOperation.FB_TRANSCRIBE, sourceUrl.href);
10641
10718
  if (!ok) return c.json(insufficientBalanceResponse(balance_mc, holdMc), 402);
10642
10719
  debited = true;
10720
+ const kernelStartMs = Date.now();
10643
10721
  await driver.launch(await kernelLaunchOptsResidential());
10644
10722
  await driver.navigateTo(sourceUrl.href);
10645
10723
  const page = driver.getPage();
10646
10724
  const video = await extractFacebookOrganicVideoFromPage(page, sourceUrl.href, body.quality);
10725
+ const kernelMs = Date.now() - kernelStartMs;
10647
10726
  const transcript = await transcribeFacebookVideoUrl(video.videoUrl, "# Facebook Organic Video Transcript");
10648
- const actualMins = Math.max(1, Math.ceil((Number.isFinite(transcript.durationMs) ? transcript.durationMs / 1e3 : 60) / 60));
10649
- const diff = holdMc - MC_COSTS.fb_transcribe * actualMins;
10650
- if (diff > 0) await creditMc(fbUser.id, diff, LedgerOperation.FB_TRANSCRIBE_REFUND, "overestimate refund");
10727
+ const settleMc = Math.round((transcript.costUsd + kernelCostUsd(kernelMs, true)) * MEDIA_TRANSCRIBE_MARGIN_MULTIPLE * MC_PER_USD);
10728
+ const diff = holdMc - settleMc;
10729
+ if (diff > 0) await creditMc(fbUser.id, diff, LedgerOperation.FB_TRANSCRIBE_REFUND, "real-cost reconcile");
10651
10730
  else if (diff < 0) await debitMc(fbUser.id, -diff, LedgerOperation.FB_TRANSCRIBE, sourceUrl.href);
10652
10731
  const result = {
10653
10732
  ...video,
@@ -11039,9 +11118,11 @@ googleAdsApp.post("/search", createApiKeyAuth(), async (c) => {
11039
11118
  const { ok, balance_mc } = await debitMc(user.id, MC_COSTS.google_ads_search, LedgerOperation.GOOGLE_ADS_SEARCH, body.query.trim());
11040
11119
  if (!ok) return c.json(insufficientBalanceResponse(balance_mc, MC_COSTS.google_ads_search), 402);
11041
11120
  debited = true;
11042
- await driver.launch(await kernelLaunchOptsResidential2());
11043
- const extractor = new GoogleAdsExtractor(driver);
11044
- const result = await extractor.search(body.query.trim(), region, maxResults);
11121
+ const result = await runWithCostContext({ ...currentCostContext(), op: "google_ads_search", userId: user.id }, async () => {
11122
+ await driver.launch(await kernelLaunchOptsResidential2());
11123
+ const extractor = new GoogleAdsExtractor(driver);
11124
+ return extractor.search(body.query.trim(), region, maxResults);
11125
+ });
11045
11126
  if (result.advertisers.length === 0) {
11046
11127
  await creditMc(user.id, MC_COSTS.google_ads_search, LedgerOperation.GOOGLE_ADS_SEARCH_REFUND, "empty result");
11047
11128
  refunded = true;
@@ -11077,9 +11158,11 @@ googleAdsApp.post("/page-intel", createApiKeyAuth(), async (c) => {
11077
11158
  const { ok, balance_mc } = await debitMc(user.id, MC_COSTS.google_ads_intel, LedgerOperation.GOOGLE_ADS_INTEL, queryLabel);
11078
11159
  if (!ok) return c.json(insufficientBalanceResponse(balance_mc, MC_COSTS.google_ads_intel), 402);
11079
11160
  debited = true;
11080
- await driver.launch(await kernelLaunchOptsResidential2());
11081
- const extractor = new GoogleAdsExtractor(driver);
11082
- const result = await extractor.pageIntel({ advertiserId: body.advertiserId, domain: body.domain, region, maxAds });
11161
+ const result = await runWithCostContext({ ...currentCostContext(), op: "google_ads_intel", userId: user.id }, async () => {
11162
+ await driver.launch(await kernelLaunchOptsResidential2());
11163
+ const extractor = new GoogleAdsExtractor(driver);
11164
+ return extractor.pageIntel({ advertiserId: body.advertiserId, domain: body.domain, region, maxAds });
11165
+ });
11083
11166
  if (result.ads.length === 0) {
11084
11167
  await creditMc(user.id, MC_COSTS.google_ads_intel, LedgerOperation.GOOGLE_ADS_INTEL_REFUND, "empty result");
11085
11168
  refunded = true;
@@ -11110,16 +11193,16 @@ googleAdsApp.post("/transcribe", createApiKeyAuth(), async (c) => {
11110
11193
  const gate = await acquireConcurrencyGate(user, "google_ads_transcribe", { reuseLockId: c.req.header("x-mcp-scraper-concurrency-lock"), metadata: { videoUrl } });
11111
11194
  if (!gate.ok) return c.json(concurrencyLimitExceededResponse(gate), 429, { "Retry-After": String(gate.retryAfterSeconds) });
11112
11195
  fal4.config({ credentials: process.env.FAL_KEY });
11113
- const holdMc = MC_COSTS.google_ads_transcribe * 5;
11196
+ const holdMc = MEDIA_TRANSCRIBE_HOLD_MC;
11114
11197
  let debited = false;
11115
11198
  try {
11116
11199
  const { ok, balance_mc } = await debitMc(user.id, holdMc, LedgerOperation.GOOGLE_ADS_TRANSCRIBE, videoUrl);
11117
11200
  if (!ok) return c.json(insufficientBalanceResponse(balance_mc, holdMc), 402);
11118
11201
  debited = true;
11119
- const transcript = await transcribeMediaUrl(videoUrl, "# Google Ad Transcript");
11120
- const actualMins = Math.max(1, Math.ceil((Number.isFinite(transcript.durationMs) ? transcript.durationMs / 1e3 : 60) / 60));
11121
- const diff = holdMc - MC_COSTS.google_ads_transcribe * actualMins;
11122
- if (diff > 0) await creditMc(user.id, diff, LedgerOperation.GOOGLE_ADS_TRANSCRIBE_REFUND, "overestimate refund");
11202
+ const transcript = await runWithCostContext({ ...currentCostContext(), op: "google_ads_transcribe", userId: user.id }, () => transcribeMediaUrl(videoUrl, "# Google Ad Transcript"));
11203
+ const settleMc = Math.round(transcript.costUsd * MEDIA_TRANSCRIBE_MARGIN_MULTIPLE * MC_PER_USD);
11204
+ const diff = holdMc - settleMc;
11205
+ if (diff > 0) await creditMc(user.id, diff, LedgerOperation.GOOGLE_ADS_TRANSCRIBE_REFUND, "real-cost reconcile");
11123
11206
  else if (diff < 0) await debitMc(user.id, -diff, LedgerOperation.GOOGLE_ADS_TRANSCRIBE, videoUrl);
11124
11207
  await logRequestEvent({ userId: user.id, source: "google_ads_transcribe", status: "done", query: videoUrl, resultCount: transcript.chunks.length, result: { text: transcript.text, chunks: transcript.chunks, durationMs: transcript.durationMs } });
11125
11208
  return c.json(transcript);
@@ -11768,10 +11851,15 @@ instagramApp.post("/media-download", createApiKeyAuth(), async (c) => {
11768
11851
  if (!audioUrl) {
11769
11852
  warnings.push("Transcript requested, but no audio/video track was available.");
11770
11853
  } else {
11771
- const tx = await debitMc(user.id, MC_COSTS.instagram_transcribe, LedgerOperation.INSTAGRAM_TRANSCRIBE, sourceUrl.href);
11772
- if (!tx.ok) return c.json(insufficientBalanceResponse(tx.balance_mc, MC_COSTS.instagram_transcribe), 402);
11854
+ const holdMc = MEDIA_TRANSCRIBE_HOLD_MC;
11855
+ const tx = await debitMc(user.id, holdMc, LedgerOperation.INSTAGRAM_TRANSCRIBE, sourceUrl.href);
11856
+ if (!tx.ok) return c.json(insufficientBalanceResponse(tx.balance_mc, holdMc), 402);
11773
11857
  transcriptDebited = true;
11774
11858
  transcript = await transcribeMediaUrl(audioUrl, "# Instagram Media Transcript");
11859
+ const settleMc = Math.round(transcript.costUsd * MEDIA_TRANSCRIBE_MARGIN_MULTIPLE * MC_PER_USD);
11860
+ const diff = holdMc - settleMc;
11861
+ if (diff > 0) await creditMc(user.id, diff, LedgerOperation.INSTAGRAM_TRANSCRIBE_REFUND, "real-cost reconcile");
11862
+ else if (diff < 0) await debitMc(user.id, -diff, LedgerOperation.INSTAGRAM_TRANSCRIBE, sourceUrl.href);
11775
11863
  }
11776
11864
  }
11777
11865
  const result = {
@@ -11786,7 +11874,7 @@ instagramApp.post("/media-download", createApiKeyAuth(), async (c) => {
11786
11874
  return c.json(result);
11787
11875
  } catch (err) {
11788
11876
  const msg = err instanceof Error ? err.message : String(err);
11789
- if (transcriptDebited) await creditMc(user.id, MC_COSTS.instagram_transcribe, LedgerOperation.INSTAGRAM_TRANSCRIBE_REFUND, "failed call");
11877
+ if (transcriptDebited) await creditMc(user.id, MEDIA_TRANSCRIBE_HOLD_MC, LedgerOperation.INSTAGRAM_TRANSCRIBE_REFUND, "failed call");
11790
11878
  if (mediaDebited) await creditMc(user.id, MC_COSTS.instagram_media, LedgerOperation.INSTAGRAM_MEDIA_REFUND, "failed call");
11791
11879
  await logRequestEvent({ userId: user.id, source: "instagram_media_download", status: "failed", query: sourceUrl.href, error: msg });
11792
11880
  if (msg.toLowerCase().includes("blocked") || msg.toLowerCase().includes("captcha")) return c.json({ error: msg }, 503);
@@ -11914,10 +12002,23 @@ redditApp.post("/thread", createApiKeyAuth(), async (c) => {
11914
12002
  result.comments = result.comments.slice(0, body.maxComments);
11915
12003
  result.numComments = result.comments.length;
11916
12004
  }
12005
+ const commentCount = Array.isArray(result.comments) ? result.comments.length : 0;
12006
+ commentDebitMc = commentCount * MC_COSTS.reddit_comment;
12007
+ if (commentDebitMc > 0) {
12008
+ const commentDebit = await debitMc(user.id, commentDebitMc, LedgerOperation.REDDIT_COMMENT, body.url);
12009
+ if (!commentDebit.ok) {
12010
+ const affordable = Math.floor((commentDebit.balance_mc ?? 0) / MC_COSTS.reddit_comment);
12011
+ result.comments = result.comments.slice(0, affordable);
12012
+ result.numComments = affordable;
12013
+ commentDebitMc = affordable * MC_COSTS.reddit_comment;
12014
+ if (commentDebitMc > 0) await debitMc(user.id, commentDebitMc, LedgerOperation.REDDIT_COMMENT, body.url);
12015
+ }
12016
+ }
11917
12017
  await logRequestEvent({ userId: user.id, source: "reddit_thread", status: "done", query: body.url, resultCount: Number(result.numComments ?? 0), result });
11918
12018
  return c.json(result);
11919
12019
  } catch (err) {
11920
12020
  if (debited && !refunded) await creditMc(user.id, MC_COSTS.reddit_thread, LedgerOperation.REDDIT_THREAD_REFUND, "failed call");
12021
+ if (commentDebitMc > 0) await creditMc(user.id, commentDebitMc, LedgerOperation.REDDIT_COMMENT_REFUND, "failed call");
11921
12022
  const msg = err instanceof Error ? err.message : String(err);
11922
12023
  await logRequestEvent({ userId: user.id, source: "reddit_thread", status: "failed", query: body.url, error: msg });
11923
12024
  return c.json({ error: msg }, 500);
@@ -20408,6 +20509,11 @@ function opForCostPath(p) {
20408
20509
  return "fb_ad";
20409
20510
  }
20410
20511
  if (p.startsWith("/instagram/")) return "instagram";
20512
+ if (p.startsWith("/google-ads/")) {
20513
+ if (p.endsWith("/transcribe")) return "google_ads_transcribe";
20514
+ if (p.endsWith("/search")) return "google_ads_search";
20515
+ return "google_ads_intel";
20516
+ }
20411
20517
  if (p === "/reddit/thread") return "reddit_thread";
20412
20518
  if (p.startsWith("/trustpilot/")) return "trustpilot_reviews";
20413
20519
  if (p.startsWith("/g2/")) return "g2_reviews";
@@ -21818,7 +21924,7 @@ app.get("/cron/tick", async (c) => {
21818
21924
  if (!process.env.CRON_SECRET || secret2 !== `Bearer ${process.env.CRON_SECRET}`) {
21819
21925
  return c.json({ error: "Unauthorized" }, 401);
21820
21926
  }
21821
- const { drainQueue } = await import("./worker-HTYZAYGB.js");
21927
+ const { drainQueue } = await import("./worker-WCSKUAOT.js");
21822
21928
  const budget = { maxJobs: 10, deadlineMs: Date.now() + 28e4 };
21823
21929
  const workflowDispatchResult = await dispatchDueWorkflowSchedules(`${new URL(c.req.url).protocol}//${new URL(c.req.url).host}`);
21824
21930
  const [results, sweepResult, reapResult, expiredResult, blobCleanup] = await Promise.all([
@@ -21836,7 +21942,7 @@ app.post("/api/internal/extract-refinalize/:id", async (c) => {
21836
21942
  return c.json({ error: "Unauthorized" }, 401);
21837
21943
  }
21838
21944
  const jobId = c.req.param("id");
21839
- const { getExtractJob: getExtractJob2, completeExtractJob: completeExtractJob2 } = await import("./site-extract-repository-HTIY52MW.js");
21945
+ const { getExtractJob: getExtractJob2, completeExtractJob: completeExtractJob2 } = await import("./site-extract-repository-6X2C7RLF.js");
21840
21946
  const { assembleExtractArtifacts } = await import("./extract-bundle-COS56ZDO.js");
21841
21947
  const job = await getExtractJob2(jobId);
21842
21948
  if (!job) return c.json({ error: "job not found" }, 404);
@@ -21844,7 +21950,7 @@ app.post("/api/internal/extract-refinalize/:id", async (c) => {
21844
21950
  await completeExtractJob2(jobId, stored);
21845
21951
  let settlement = "already_settled_or_refunded";
21846
21952
  if (job.billedMc == null && job.userId != null) {
21847
- const { settleExtractJob: settleExtractJob2, countSuccessfulPages: countSuccessfulPages2 } = await import("./site-extract-repository-HTIY52MW.js");
21953
+ const { settleExtractJob: settleExtractJob2, countSuccessfulPages: countSuccessfulPages2 } = await import("./site-extract-repository-6X2C7RLF.js");
21848
21954
  const heldMc = Number(job.options.heldMc ?? 0);
21849
21955
  const successful = await countSuccessfulPages2(jobId);
21850
21956
  const usedMc = Math.min(successful * MC_COSTS.page_scrape, heldMc);
@@ -22008,4 +22114,4 @@ app.get("/blog/:slug/", (c) => {
22008
22114
  export {
22009
22115
  app
22010
22116
  };
22011
- //# sourceMappingURL=server-UNID3SJU.js.map
22117
+ //# sourceMappingURL=server-HNZG36IP.js.map