mcp-scraper 0.4.12 → 0.4.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bin/api-server.cjs +196 -79
- package/dist/bin/api-server.cjs.map +1 -1
- package/dist/bin/api-server.js +2 -2
- package/dist/bin/mcp-scraper-cli.cjs +1 -1
- package/dist/bin/mcp-scraper-cli.cjs.map +1 -1
- package/dist/bin/mcp-scraper-cli.js +1 -1
- package/dist/bin/mcp-scraper-install.cjs +1 -1
- package/dist/bin/mcp-scraper-install.cjs.map +1 -1
- package/dist/bin/mcp-scraper-install.js +1 -1
- package/dist/bin/mcp-stdio-server.cjs +1 -1
- package/dist/bin/mcp-stdio-server.cjs.map +1 -1
- package/dist/bin/mcp-stdio-server.js +2 -2
- package/dist/bin/paa-harvest.js +1 -1
- package/dist/{chunk-2YY46QYT.js → chunk-AUCXKRRH.js} +3 -1
- package/dist/{chunk-6EXP6DQG.js → chunk-EA2AANFI.js} +2 -2
- package/dist/chunk-O2D5HGKF.js +7 -0
- package/dist/chunk-O2D5HGKF.js.map +1 -0
- package/dist/{chunk-EMY7ELRU.js → chunk-SIPXEFTR.js} +14 -10
- package/dist/chunk-SIPXEFTR.js.map +1 -0
- package/dist/{chunk-ONIOF5XW.js → chunk-VQV7MB2S.js} +2 -2
- package/dist/index.js +1 -1
- package/dist/{server-UNID3SJU.js → server-HNZG36IP.js} +178 -72
- package/dist/server-HNZG36IP.js.map +1 -0
- package/dist/{site-extract-repository-HTIY52MW.js → site-extract-repository-6X2C7RLF.js} +3 -3
- package/dist/{worker-HTYZAYGB.js → worker-WCSKUAOT.js} +3 -3
- package/package.json +1 -1
- package/dist/chunk-EMY7ELRU.js.map +0 -1
- package/dist/chunk-SHXJQQOH.js +0 -7
- package/dist/chunk-SHXJQQOH.js.map +0 -1
- package/dist/server-UNID3SJU.js.map +0 -1
- /package/dist/{chunk-2YY46QYT.js.map → chunk-AUCXKRRH.js.map} +0 -0
- /package/dist/{chunk-6EXP6DQG.js.map → chunk-EA2AANFI.js.map} +0 -0
- /package/dist/{chunk-ONIOF5XW.js.map → chunk-VQV7MB2S.js.map} +0 -0
- /package/dist/{site-extract-repository-HTIY52MW.js.map → site-extract-repository-6X2C7RLF.js.map} +0 -0
- /package/dist/{worker-HTYZAYGB.js.map → worker-WCSKUAOT.js.map} +0 -0
package/dist/bin/api-server.cjs
CHANGED
|
@@ -7986,7 +7986,7 @@ function insufficientBalanceResponse(balanceMc, requiredMc) {
|
|
|
7986
7986
|
topup_url: topupUrl
|
|
7987
7987
|
};
|
|
7988
7988
|
}
|
|
7989
|
-
var MC_COSTS, MC_PER_BROWSER_MS, BROWSER_OPEN_MIN_BALANCE_MC, MC_PER_CREDIT, CREDIT_COST_CATALOG, CONCURRENCY_PRICE_ID, SUBSCRIPTION_TIERS, SUBSCRIPTION_TIER_BY_KEY, CONCURRENCY_SLOT_PRICE_USD, CONCURRENCY_SLOT_PRICE_CURRENCY, CONCURRENCY_SLOT_PRICE_INTERVAL, CONCURRENCY_SLOT_PRICE_LABEL, CONCURRENCY_SLOT_TERMINAL_COMMAND, FREE_SIGNUP_MC, MEMORY_FREE_QUOTA_BYTES, MEMORY_PLANS, MEMORY_PLAN_QUOTA, MEMORY_MARGIN_MULTIPLE, MEMORY_PRO_COST_BUDGET_USD, SCHEDULING_PLANS, LedgerOperation, MEMORY_AI_MARGIN_MULTIPLE, SITE_AUDIT_LLM_MARGIN_MULTIPLE, VIDEO_ANALYSIS_LLM_MARGIN_MULTIPLE, MC_PER_USD;
|
|
7989
|
+
var MC_COSTS, MC_PER_BROWSER_MS, BROWSER_OPEN_MIN_BALANCE_MC, MC_PER_CREDIT, CREDIT_COST_CATALOG, CONCURRENCY_PRICE_ID, SUBSCRIPTION_TIERS, SUBSCRIPTION_TIER_BY_KEY, CONCURRENCY_SLOT_PRICE_USD, CONCURRENCY_SLOT_PRICE_CURRENCY, CONCURRENCY_SLOT_PRICE_INTERVAL, CONCURRENCY_SLOT_PRICE_LABEL, CONCURRENCY_SLOT_TERMINAL_COMMAND, FREE_SIGNUP_MC, MEMORY_FREE_QUOTA_BYTES, MEMORY_PLANS, MEMORY_PLAN_QUOTA, MEMORY_MARGIN_MULTIPLE, MEMORY_PRO_COST_BUDGET_USD, SCHEDULING_PLANS, LedgerOperation, MEMORY_AI_MARGIN_MULTIPLE, SITE_AUDIT_LLM_MARGIN_MULTIPLE, VIDEO_ANALYSIS_LLM_MARGIN_MULTIPLE, MEDIA_TRANSCRIBE_MARGIN_MULTIPLE, MEDIA_TRANSCRIBE_HOLD_MC, MC_PER_USD;
|
|
7990
7990
|
var init_rates = __esm({
|
|
7991
7991
|
"src/api/rates.ts"() {
|
|
7992
7992
|
"use strict";
|
|
@@ -8003,13 +8003,13 @@ var init_rates = __esm({
|
|
|
8003
8003
|
maps_place: 6e3,
|
|
8004
8004
|
maps_review: 100,
|
|
8005
8005
|
fb_search: 600,
|
|
8006
|
-
fb_transcribe:
|
|
8006
|
+
fb_transcribe: 33300,
|
|
8007
8007
|
google_ads_search: 600,
|
|
8008
8008
|
google_ads_intel: 200,
|
|
8009
|
-
google_ads_transcribe:
|
|
8009
|
+
google_ads_transcribe: 33300,
|
|
8010
8010
|
instagram_profile: 400,
|
|
8011
8011
|
instagram_media: 400,
|
|
8012
|
-
instagram_transcribe:
|
|
8012
|
+
instagram_transcribe: 33300,
|
|
8013
8013
|
browser_minute: 12e3,
|
|
8014
8014
|
reddit_thread: 3e3,
|
|
8015
8015
|
reddit_comment: 200,
|
|
@@ -8131,16 +8131,16 @@ var init_rates = __esm({
|
|
|
8131
8131
|
label: "Google ad video transcription",
|
|
8132
8132
|
aliases: ["google_ads_transcribe", "google ad transcript"],
|
|
8133
8133
|
credits: mcToCredits(MC_COSTS.google_ads_transcribe),
|
|
8134
|
-
unit: "per minute",
|
|
8135
|
-
notes: "A hold is taken, then reconciled to actual
|
|
8134
|
+
unit: "per minute (typical)",
|
|
8135
|
+
notes: "A hold is taken, then reconciled to the real transcription cost for the actual audio duration."
|
|
8136
8136
|
},
|
|
8137
8137
|
{
|
|
8138
8138
|
key: "fb_transcribe",
|
|
8139
8139
|
label: "Facebook video / ad transcription",
|
|
8140
8140
|
aliases: ["facebook_transcribe", "facebook_video_transcribe", "fb_transcribe", "fb ad transcript"],
|
|
8141
8141
|
credits: mcToCredits(MC_COSTS.fb_transcribe),
|
|
8142
|
-
unit: "per minute",
|
|
8143
|
-
notes: "A hold is taken, then reconciled to actual
|
|
8142
|
+
unit: "per minute (typical)",
|
|
8143
|
+
notes: "A hold is taken, then reconciled to the real transcription cost for the actual audio duration."
|
|
8144
8144
|
},
|
|
8145
8145
|
{
|
|
8146
8146
|
key: "instagram_profile",
|
|
@@ -8163,8 +8163,8 @@ var init_rates = __esm({
|
|
|
8163
8163
|
label: "Instagram media transcription",
|
|
8164
8164
|
aliases: ["instagram transcript", "instagram reel transcript", "ig transcribe"],
|
|
8165
8165
|
credits: mcToCredits(MC_COSTS.instagram_transcribe),
|
|
8166
|
-
unit: "per
|
|
8167
|
-
notes: "Whisper transcription of selected Instagram audio/video media via fal.ai."
|
|
8166
|
+
unit: "per minute (typical)",
|
|
8167
|
+
notes: "Whisper transcription of selected Instagram audio/video media via fal.ai. A hold is taken, then reconciled to the real transcription cost for the actual audio duration."
|
|
8168
8168
|
},
|
|
8169
8169
|
{
|
|
8170
8170
|
key: "browser_minute",
|
|
@@ -8333,6 +8333,8 @@ var init_rates = __esm({
|
|
|
8333
8333
|
MEMORY_AI_MARGIN_MULTIPLE = 3;
|
|
8334
8334
|
SITE_AUDIT_LLM_MARGIN_MULTIPLE = 1.5;
|
|
8335
8335
|
VIDEO_ANALYSIS_LLM_MARGIN_MULTIPLE = 1.5;
|
|
8336
|
+
MEDIA_TRANSCRIBE_MARGIN_MULTIPLE = 3;
|
|
8337
|
+
MEDIA_TRANSCRIBE_HOLD_MC = MC_COSTS.fb_transcribe * 3;
|
|
8336
8338
|
MC_PER_USD = SUBSCRIPTION_TIERS["price_1TmiHRS8aAcsk3TGwmSNfNIa"].credits_mc / SUBSCRIPTION_TIERS["price_1TmiHRS8aAcsk3TGwmSNfNIa"].monthly_usd;
|
|
8337
8339
|
}
|
|
8338
8340
|
});
|
|
@@ -13537,6 +13539,104 @@ var init_kernel_proxy_resolver = __esm({
|
|
|
13537
13539
|
}
|
|
13538
13540
|
});
|
|
13539
13541
|
|
|
13542
|
+
// src/youtube/audio-resolver.ts
|
|
13543
|
+
function mediaResolverUrl() {
|
|
13544
|
+
return process.env.FRAMES_SERVICE_URL?.trim() || void 0;
|
|
13545
|
+
}
|
|
13546
|
+
function mediaResolverSecret() {
|
|
13547
|
+
return process.env.FRAMES_SERVICE_SECRET?.trim() || void 0;
|
|
13548
|
+
}
|
|
13549
|
+
function mediaResolverConfigured() {
|
|
13550
|
+
return Boolean(mediaResolverUrl() && mediaResolverSecret());
|
|
13551
|
+
}
|
|
13552
|
+
async function resolveYouTubeAudio(videoId) {
|
|
13553
|
+
const base = mediaResolverUrl();
|
|
13554
|
+
const secret2 = mediaResolverSecret();
|
|
13555
|
+
if (!base || !secret2) return null;
|
|
13556
|
+
const controller = new AbortController();
|
|
13557
|
+
const timer = setTimeout(() => controller.abort(), RESOLVE_TIMEOUT_MS);
|
|
13558
|
+
try {
|
|
13559
|
+
const resp = await fetch(`${base.replace(/\/+$/, "")}/resolve-audio`, {
|
|
13560
|
+
method: "POST",
|
|
13561
|
+
headers: { "content-type": "application/json", "x-frames-secret": secret2 },
|
|
13562
|
+
body: JSON.stringify({ videoUrl: `https://www.youtube.com/watch?v=${videoId}` }),
|
|
13563
|
+
signal: controller.signal
|
|
13564
|
+
});
|
|
13565
|
+
if (!resp.ok) {
|
|
13566
|
+
const detail = await resp.text().catch(() => "");
|
|
13567
|
+
console.warn(`[audio-resolver] ${videoId}: http_${resp.status} ${detail.slice(0, 200)}`);
|
|
13568
|
+
return null;
|
|
13569
|
+
}
|
|
13570
|
+
const data = await resp.json();
|
|
13571
|
+
if (!data.mediaUrl) {
|
|
13572
|
+
console.warn(`[audio-resolver] ${videoId}: response missing mediaUrl`);
|
|
13573
|
+
return null;
|
|
13574
|
+
}
|
|
13575
|
+
return {
|
|
13576
|
+
mediaUrl: data.mediaUrl,
|
|
13577
|
+
durationS: Number(data.durationS) || 0,
|
|
13578
|
+
sizeBytes: Number(data.sizeBytes) || 0,
|
|
13579
|
+
mimeType: data.mimeType ?? "audio/mp4"
|
|
13580
|
+
};
|
|
13581
|
+
} catch (err) {
|
|
13582
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
13583
|
+
console.warn(`[audio-resolver] ${videoId}: resolve failed \u2014 ${msg}`);
|
|
13584
|
+
return null;
|
|
13585
|
+
} finally {
|
|
13586
|
+
clearTimeout(timer);
|
|
13587
|
+
}
|
|
13588
|
+
}
|
|
13589
|
+
var RESOLVE_TIMEOUT_MS;
|
|
13590
|
+
var init_audio_resolver = __esm({
|
|
13591
|
+
"src/youtube/audio-resolver.ts"() {
|
|
13592
|
+
"use strict";
|
|
13593
|
+
RESOLVE_TIMEOUT_MS = 24e4;
|
|
13594
|
+
}
|
|
13595
|
+
});
|
|
13596
|
+
|
|
13597
|
+
// src/services/media-transcription.ts
|
|
13598
|
+
function transcriptMarkdown(title, text, chunks, durationMs) {
|
|
13599
|
+
const fmtTs2 = (s) => `${Math.floor(s / 60)}:${String(Math.floor(s % 60)).padStart(2, "0")}`;
|
|
13600
|
+
const lines = [title, "", `*Transcribed in ${(durationMs / 1e3).toFixed(1)}s*`, "", "## Full Text", "", text, ""];
|
|
13601
|
+
if (chunks.length) {
|
|
13602
|
+
lines.push("## Timestamped Segments", "");
|
|
13603
|
+
for (const ch of chunks) {
|
|
13604
|
+
lines.push(`**[${fmtTs2(ch.timestamp[0])} -> ${fmtTs2(ch.timestamp[1])}]** ${ch.text.trim()}`, "");
|
|
13605
|
+
}
|
|
13606
|
+
}
|
|
13607
|
+
return lines.join("\n");
|
|
13608
|
+
}
|
|
13609
|
+
async function transcribeMediaUrl(mediaUrl, markdownTitle = "# Media Transcript") {
|
|
13610
|
+
const startMs = Date.now();
|
|
13611
|
+
const result = await import_client3.fal.subscribe("fal-ai/wizper", {
|
|
13612
|
+
input: { audio_url: mediaUrl, task: "transcribe", language: "en" },
|
|
13613
|
+
logs: false,
|
|
13614
|
+
pollInterval: 3e3
|
|
13615
|
+
});
|
|
13616
|
+
const data = result.data;
|
|
13617
|
+
const text = data.text ?? "";
|
|
13618
|
+
const chunks = data.chunks ?? [];
|
|
13619
|
+
const durationMs = Date.now() - startMs;
|
|
13620
|
+
void recordVendorUsage({ vendor: "fal_wizper", model: "fal-ai/wizper", units: durationMs / 1e3, unitType: "compute_sec" });
|
|
13621
|
+
const costUsd = vendorCostUsd("fal_wizper", durationMs / 1e3);
|
|
13622
|
+
return {
|
|
13623
|
+
text,
|
|
13624
|
+
chunks,
|
|
13625
|
+
durationMs,
|
|
13626
|
+
costUsd,
|
|
13627
|
+
markdown: transcriptMarkdown(markdownTitle, text, chunks, durationMs)
|
|
13628
|
+
};
|
|
13629
|
+
}
|
|
13630
|
+
var import_client3;
|
|
13631
|
+
var init_media_transcription = __esm({
|
|
13632
|
+
"src/services/media-transcription.ts"() {
|
|
13633
|
+
"use strict";
|
|
13634
|
+
import_client3 = require("@fal-ai/client");
|
|
13635
|
+
init_cost_telemetry();
|
|
13636
|
+
init_cost_rates();
|
|
13637
|
+
}
|
|
13638
|
+
});
|
|
13639
|
+
|
|
13540
13640
|
// src/youtube/CaptionFetcher.ts
|
|
13541
13641
|
async function fetchViaYoutubeTranscript(videoId) {
|
|
13542
13642
|
try {
|
|
@@ -13693,12 +13793,12 @@ async function transcribeWithWizper(videoId, audio, startMs) {
|
|
|
13693
13793
|
const falKey = process.env.FAL_KEY;
|
|
13694
13794
|
if (!falKey) return null;
|
|
13695
13795
|
try {
|
|
13696
|
-
|
|
13796
|
+
import_client4.fal.config({ credentials: falKey });
|
|
13697
13797
|
const ext = audio.mimeType.includes("webm") ? "webm" : audio.mimeType.includes("mp4") ? "m4a" : "audio";
|
|
13698
13798
|
const audioFile = new File([new Uint8Array(audio.bytes)], `audio.${ext}`, { type: audio.mimeType });
|
|
13699
|
-
const uploadedUrl = await
|
|
13799
|
+
const uploadedUrl = await import_client4.fal.storage.upload(audioFile);
|
|
13700
13800
|
const subscribeStartMs = Date.now();
|
|
13701
|
-
const result = await
|
|
13801
|
+
const result = await import_client4.fal.subscribe("fal-ai/wizper", {
|
|
13702
13802
|
input: { audio_url: uploadedUrl, task: "transcribe", language: "en", chunk_level: "segment" },
|
|
13703
13803
|
logs: false,
|
|
13704
13804
|
pollInterval: 3e3
|
|
@@ -13816,6 +13916,28 @@ async function fetchViaKernelSession(videoId) {
|
|
|
13816
13916
|
}
|
|
13817
13917
|
return null;
|
|
13818
13918
|
}
|
|
13919
|
+
async function fetchViaResolvedAudio(videoId) {
|
|
13920
|
+
if (!mediaResolverConfigured()) return null;
|
|
13921
|
+
return runWithCostContext({ ...currentCostContext(), op: "yt_transcription", subOp: "audio-whisper" }, async () => {
|
|
13922
|
+
const resolved = await resolveYouTubeAudio(videoId);
|
|
13923
|
+
if (!resolved) return null;
|
|
13924
|
+
try {
|
|
13925
|
+
const transcript = await transcribeMediaUrl(resolved.mediaUrl, `# Transcript \u2014 ${videoId}`);
|
|
13926
|
+
if (!transcript.text) return null;
|
|
13927
|
+
const chunks = transcript.chunks.map((c) => ({ timestamp: c.timestamp, text: c.text.trim() }));
|
|
13928
|
+
return {
|
|
13929
|
+
videoId,
|
|
13930
|
+
text: transcript.text,
|
|
13931
|
+
chunks,
|
|
13932
|
+
durationMs: resolved.durationS > 0 ? Math.round(resolved.durationS * 1e3) : transcript.durationMs,
|
|
13933
|
+
method: "audio-whisper"
|
|
13934
|
+
};
|
|
13935
|
+
} catch (err) {
|
|
13936
|
+
console.warn(`[CaptionFetcher] transcription of resolved audio failed for ${videoId}: ${err instanceof Error ? err.message : String(err)}`);
|
|
13937
|
+
return null;
|
|
13938
|
+
}
|
|
13939
|
+
});
|
|
13940
|
+
}
|
|
13819
13941
|
function finalizeCaptions(result) {
|
|
13820
13942
|
const chunks = result.chunks.map((c) => {
|
|
13821
13943
|
const startOk = Number.isFinite(c.timestamp[0]);
|
|
@@ -13836,18 +13958,22 @@ async function fetchCaptions(videoId) {
|
|
|
13836
13958
|
if (primary) return finalizeCaptions(primary);
|
|
13837
13959
|
const result = await fetchViaKernelSession(videoId);
|
|
13838
13960
|
if (result) return finalizeCaptions(result);
|
|
13961
|
+
const viaResolvedAudio = await fetchViaResolvedAudio(videoId);
|
|
13962
|
+
if (viaResolvedAudio) return finalizeCaptions(viaResolvedAudio);
|
|
13839
13963
|
throw new Error(
|
|
13840
|
-
`Could not transcribe ${videoId}
|
|
13964
|
+
`Could not transcribe ${videoId}: it has no caption track and its audio could not be retrieved.`
|
|
13841
13965
|
);
|
|
13842
13966
|
}
|
|
13843
|
-
var
|
|
13967
|
+
var import_client4, INNERTUBE_CLIENTS, MAX_AUDIO_BYTES, YT_RESIDENTIAL_LOCATION;
|
|
13844
13968
|
var init_CaptionFetcher = __esm({
|
|
13845
13969
|
"src/youtube/CaptionFetcher.ts"() {
|
|
13846
13970
|
"use strict";
|
|
13847
13971
|
init_BrowserDriver();
|
|
13848
13972
|
init_browser_service_env();
|
|
13849
13973
|
init_kernel_proxy_resolver();
|
|
13850
|
-
|
|
13974
|
+
init_audio_resolver();
|
|
13975
|
+
init_media_transcription();
|
|
13976
|
+
import_client4 = require("@fal-ai/client");
|
|
13851
13977
|
init_cost_telemetry();
|
|
13852
13978
|
init_cost_context();
|
|
13853
13979
|
INNERTUBE_CLIENTS = [
|
|
@@ -15392,46 +15518,6 @@ var init_schemas3 = __esm({
|
|
|
15392
15518
|
}
|
|
15393
15519
|
});
|
|
15394
15520
|
|
|
15395
|
-
// src/services/media-transcription.ts
|
|
15396
|
-
function transcriptMarkdown(title, text, chunks, durationMs) {
|
|
15397
|
-
const fmtTs2 = (s) => `${Math.floor(s / 60)}:${String(Math.floor(s % 60)).padStart(2, "0")}`;
|
|
15398
|
-
const lines = [title, "", `*Transcribed in ${(durationMs / 1e3).toFixed(1)}s*`, "", "## Full Text", "", text, ""];
|
|
15399
|
-
if (chunks.length) {
|
|
15400
|
-
lines.push("## Timestamped Segments", "");
|
|
15401
|
-
for (const ch of chunks) {
|
|
15402
|
-
lines.push(`**[${fmtTs2(ch.timestamp[0])} -> ${fmtTs2(ch.timestamp[1])}]** ${ch.text.trim()}`, "");
|
|
15403
|
-
}
|
|
15404
|
-
}
|
|
15405
|
-
return lines.join("\n");
|
|
15406
|
-
}
|
|
15407
|
-
async function transcribeMediaUrl(mediaUrl, markdownTitle = "# Media Transcript") {
|
|
15408
|
-
const startMs = Date.now();
|
|
15409
|
-
const result = await import_client4.fal.subscribe("fal-ai/wizper", {
|
|
15410
|
-
input: { audio_url: mediaUrl, task: "transcribe", language: "en" },
|
|
15411
|
-
logs: false,
|
|
15412
|
-
pollInterval: 3e3
|
|
15413
|
-
});
|
|
15414
|
-
const data = result.data;
|
|
15415
|
-
const text = data.text ?? "";
|
|
15416
|
-
const chunks = data.chunks ?? [];
|
|
15417
|
-
const durationMs = Date.now() - startMs;
|
|
15418
|
-
void recordVendorUsage({ vendor: "fal_wizper", model: "fal-ai/wizper", units: durationMs / 1e3, unitType: "compute_sec" });
|
|
15419
|
-
return {
|
|
15420
|
-
text,
|
|
15421
|
-
chunks,
|
|
15422
|
-
durationMs,
|
|
15423
|
-
markdown: transcriptMarkdown(markdownTitle, text, chunks, durationMs)
|
|
15424
|
-
};
|
|
15425
|
-
}
|
|
15426
|
-
var import_client4;
|
|
15427
|
-
var init_media_transcription = __esm({
|
|
15428
|
-
"src/services/media-transcription.ts"() {
|
|
15429
|
-
"use strict";
|
|
15430
|
-
import_client4 = require("@fal-ai/client");
|
|
15431
|
-
init_cost_telemetry();
|
|
15432
|
-
}
|
|
15433
|
-
});
|
|
15434
|
-
|
|
15435
15521
|
// src/api/facebook-ad-routes.ts
|
|
15436
15522
|
function invalidRequest(message) {
|
|
15437
15523
|
return { error_code: "invalid_request", message };
|
|
@@ -15483,6 +15569,7 @@ var init_facebook_ad_routes = __esm({
|
|
|
15483
15569
|
import_zod16 = require("zod");
|
|
15484
15570
|
init_db();
|
|
15485
15571
|
init_rates();
|
|
15572
|
+
init_cost_rates();
|
|
15486
15573
|
init_BrowserDriver();
|
|
15487
15574
|
init_FacebookAdExtractor();
|
|
15488
15575
|
init_FacebookAdGraphql();
|
|
@@ -15633,16 +15720,16 @@ var init_facebook_ad_routes = __esm({
|
|
|
15633
15720
|
});
|
|
15634
15721
|
if (!gate.ok) return c.json(concurrencyLimitExceededResponse(gate), 429, { "Retry-After": String(gate.retryAfterSeconds) });
|
|
15635
15722
|
import_client5.fal.config({ credentials: process.env.FAL_KEY });
|
|
15636
|
-
const holdMc =
|
|
15723
|
+
const holdMc = MEDIA_TRANSCRIBE_HOLD_MC;
|
|
15637
15724
|
let debited = false;
|
|
15638
15725
|
try {
|
|
15639
15726
|
const { ok, balance_mc } = await debitMc(fbUser.id, holdMc, LedgerOperation.FB_TRANSCRIBE, videoUrl);
|
|
15640
15727
|
if (!ok) return c.json(insufficientBalanceResponse(balance_mc, holdMc), 402);
|
|
15641
15728
|
debited = true;
|
|
15642
15729
|
const transcript = await transcribeFacebookVideoUrl(videoUrl, "# Facebook Ad Transcript");
|
|
15643
|
-
const
|
|
15644
|
-
const diff = holdMc -
|
|
15645
|
-
if (diff > 0) await creditMc(fbUser.id, diff, LedgerOperation.FB_TRANSCRIBE_REFUND, "
|
|
15730
|
+
const settleMc = Math.round(transcript.costUsd * MEDIA_TRANSCRIBE_MARGIN_MULTIPLE * MC_PER_USD);
|
|
15731
|
+
const diff = holdMc - settleMc;
|
|
15732
|
+
if (diff > 0) await creditMc(fbUser.id, diff, LedgerOperation.FB_TRANSCRIBE_REFUND, "real-cost reconcile");
|
|
15646
15733
|
else if (diff < 0) await debitMc(fbUser.id, -diff, LedgerOperation.FB_TRANSCRIBE, videoUrl);
|
|
15647
15734
|
await logRequestEvent({ userId: fbUser.id, source: "facebook_transcribe", status: "done", query: videoUrl, resultCount: transcript.chunks.length, result: { text: transcript.text, chunks: transcript.chunks, durationMs: transcript.durationMs } });
|
|
15648
15735
|
return c.json(transcript);
|
|
@@ -15678,20 +15765,22 @@ var init_facebook_ad_routes = __esm({
|
|
|
15678
15765
|
if (!gate.ok) return c.json(concurrencyLimitExceededResponse(gate), 429, { "Retry-After": String(gate.retryAfterSeconds) });
|
|
15679
15766
|
import_client5.fal.config({ credentials: process.env.FAL_KEY });
|
|
15680
15767
|
const driver = new BrowserDriver();
|
|
15681
|
-
const holdMc =
|
|
15768
|
+
const holdMc = MEDIA_TRANSCRIBE_HOLD_MC;
|
|
15682
15769
|
let debited = false;
|
|
15683
15770
|
try {
|
|
15684
15771
|
const { ok, balance_mc } = await debitMc(fbUser.id, holdMc, LedgerOperation.FB_TRANSCRIBE, sourceUrl.href);
|
|
15685
15772
|
if (!ok) return c.json(insufficientBalanceResponse(balance_mc, holdMc), 402);
|
|
15686
15773
|
debited = true;
|
|
15774
|
+
const kernelStartMs = Date.now();
|
|
15687
15775
|
await driver.launch(await kernelLaunchOptsResidential());
|
|
15688
15776
|
await driver.navigateTo(sourceUrl.href);
|
|
15689
15777
|
const page = driver.getPage();
|
|
15690
15778
|
const video = await extractFacebookOrganicVideoFromPage(page, sourceUrl.href, body.quality);
|
|
15779
|
+
const kernelMs = Date.now() - kernelStartMs;
|
|
15691
15780
|
const transcript = await transcribeFacebookVideoUrl(video.videoUrl, "# Facebook Organic Video Transcript");
|
|
15692
|
-
const
|
|
15693
|
-
const diff = holdMc -
|
|
15694
|
-
if (diff > 0) await creditMc(fbUser.id, diff, LedgerOperation.FB_TRANSCRIBE_REFUND, "
|
|
15781
|
+
const settleMc = Math.round((transcript.costUsd + kernelCostUsd(kernelMs, true)) * MEDIA_TRANSCRIBE_MARGIN_MULTIPLE * MC_PER_USD);
|
|
15782
|
+
const diff = holdMc - settleMc;
|
|
15783
|
+
if (diff > 0) await creditMc(fbUser.id, diff, LedgerOperation.FB_TRANSCRIBE_REFUND, "real-cost reconcile");
|
|
15695
15784
|
else if (diff < 0) await debitMc(fbUser.id, -diff, LedgerOperation.FB_TRANSCRIBE, sourceUrl.href);
|
|
15696
15785
|
const result = {
|
|
15697
15786
|
...video,
|
|
@@ -16073,6 +16162,7 @@ var init_google_ads_routes = __esm({
|
|
|
16073
16162
|
init_url_utils();
|
|
16074
16163
|
init_concurrency_gates();
|
|
16075
16164
|
init_media_transcription();
|
|
16165
|
+
init_cost_context();
|
|
16076
16166
|
GoogleAdsSearchBodySchema = import_zod17.z.object({
|
|
16077
16167
|
query: import_zod17.z.string().trim().min(1, "query is required"),
|
|
16078
16168
|
region: import_zod17.z.string().trim().toUpperCase().length(2).optional(),
|
|
@@ -16104,9 +16194,11 @@ var init_google_ads_routes = __esm({
|
|
|
16104
16194
|
const { ok, balance_mc } = await debitMc(user.id, MC_COSTS.google_ads_search, LedgerOperation.GOOGLE_ADS_SEARCH, body.query.trim());
|
|
16105
16195
|
if (!ok) return c.json(insufficientBalanceResponse(balance_mc, MC_COSTS.google_ads_search), 402);
|
|
16106
16196
|
debited = true;
|
|
16107
|
-
await
|
|
16108
|
-
|
|
16109
|
-
|
|
16197
|
+
const result = await runWithCostContext({ ...currentCostContext(), op: "google_ads_search", userId: user.id }, async () => {
|
|
16198
|
+
await driver.launch(await kernelLaunchOptsResidential2());
|
|
16199
|
+
const extractor = new GoogleAdsExtractor(driver);
|
|
16200
|
+
return extractor.search(body.query.trim(), region, maxResults);
|
|
16201
|
+
});
|
|
16110
16202
|
if (result.advertisers.length === 0) {
|
|
16111
16203
|
await creditMc(user.id, MC_COSTS.google_ads_search, LedgerOperation.GOOGLE_ADS_SEARCH_REFUND, "empty result");
|
|
16112
16204
|
refunded = true;
|
|
@@ -16142,9 +16234,11 @@ var init_google_ads_routes = __esm({
|
|
|
16142
16234
|
const { ok, balance_mc } = await debitMc(user.id, MC_COSTS.google_ads_intel, LedgerOperation.GOOGLE_ADS_INTEL, queryLabel);
|
|
16143
16235
|
if (!ok) return c.json(insufficientBalanceResponse(balance_mc, MC_COSTS.google_ads_intel), 402);
|
|
16144
16236
|
debited = true;
|
|
16145
|
-
await
|
|
16146
|
-
|
|
16147
|
-
|
|
16237
|
+
const result = await runWithCostContext({ ...currentCostContext(), op: "google_ads_intel", userId: user.id }, async () => {
|
|
16238
|
+
await driver.launch(await kernelLaunchOptsResidential2());
|
|
16239
|
+
const extractor = new GoogleAdsExtractor(driver);
|
|
16240
|
+
return extractor.pageIntel({ advertiserId: body.advertiserId, domain: body.domain, region, maxAds });
|
|
16241
|
+
});
|
|
16148
16242
|
if (result.ads.length === 0) {
|
|
16149
16243
|
await creditMc(user.id, MC_COSTS.google_ads_intel, LedgerOperation.GOOGLE_ADS_INTEL_REFUND, "empty result");
|
|
16150
16244
|
refunded = true;
|
|
@@ -16175,16 +16269,16 @@ var init_google_ads_routes = __esm({
|
|
|
16175
16269
|
const gate = await acquireConcurrencyGate(user, "google_ads_transcribe", { reuseLockId: c.req.header("x-mcp-scraper-concurrency-lock"), metadata: { videoUrl } });
|
|
16176
16270
|
if (!gate.ok) return c.json(concurrencyLimitExceededResponse(gate), 429, { "Retry-After": String(gate.retryAfterSeconds) });
|
|
16177
16271
|
import_client6.fal.config({ credentials: process.env.FAL_KEY });
|
|
16178
|
-
const holdMc =
|
|
16272
|
+
const holdMc = MEDIA_TRANSCRIBE_HOLD_MC;
|
|
16179
16273
|
let debited = false;
|
|
16180
16274
|
try {
|
|
16181
16275
|
const { ok, balance_mc } = await debitMc(user.id, holdMc, LedgerOperation.GOOGLE_ADS_TRANSCRIBE, videoUrl);
|
|
16182
16276
|
if (!ok) return c.json(insufficientBalanceResponse(balance_mc, holdMc), 402);
|
|
16183
16277
|
debited = true;
|
|
16184
|
-
const transcript = await transcribeMediaUrl(videoUrl, "# Google Ad Transcript");
|
|
16185
|
-
const
|
|
16186
|
-
const diff = holdMc -
|
|
16187
|
-
if (diff > 0) await creditMc(user.id, diff, LedgerOperation.GOOGLE_ADS_TRANSCRIBE_REFUND, "
|
|
16278
|
+
const transcript = await runWithCostContext({ ...currentCostContext(), op: "google_ads_transcribe", userId: user.id }, () => transcribeMediaUrl(videoUrl, "# Google Ad Transcript"));
|
|
16279
|
+
const settleMc = Math.round(transcript.costUsd * MEDIA_TRANSCRIBE_MARGIN_MULTIPLE * MC_PER_USD);
|
|
16280
|
+
const diff = holdMc - settleMc;
|
|
16281
|
+
if (diff > 0) await creditMc(user.id, diff, LedgerOperation.GOOGLE_ADS_TRANSCRIBE_REFUND, "real-cost reconcile");
|
|
16188
16282
|
else if (diff < 0) await debitMc(user.id, -diff, LedgerOperation.GOOGLE_ADS_TRANSCRIBE, videoUrl);
|
|
16189
16283
|
await logRequestEvent({ userId: user.id, source: "google_ads_transcribe", status: "done", query: videoUrl, resultCount: transcript.chunks.length, result: { text: transcript.text, chunks: transcript.chunks, durationMs: transcript.durationMs } });
|
|
16190
16284
|
return c.json(transcript);
|
|
@@ -16852,10 +16946,15 @@ var init_instagram_routes = __esm({
|
|
|
16852
16946
|
if (!audioUrl) {
|
|
16853
16947
|
warnings.push("Transcript requested, but no audio/video track was available.");
|
|
16854
16948
|
} else {
|
|
16855
|
-
const
|
|
16856
|
-
|
|
16949
|
+
const holdMc = MEDIA_TRANSCRIBE_HOLD_MC;
|
|
16950
|
+
const tx = await debitMc(user.id, holdMc, LedgerOperation.INSTAGRAM_TRANSCRIBE, sourceUrl.href);
|
|
16951
|
+
if (!tx.ok) return c.json(insufficientBalanceResponse(tx.balance_mc, holdMc), 402);
|
|
16857
16952
|
transcriptDebited = true;
|
|
16858
16953
|
transcript = await transcribeMediaUrl(audioUrl, "# Instagram Media Transcript");
|
|
16954
|
+
const settleMc = Math.round(transcript.costUsd * MEDIA_TRANSCRIBE_MARGIN_MULTIPLE * MC_PER_USD);
|
|
16955
|
+
const diff = holdMc - settleMc;
|
|
16956
|
+
if (diff > 0) await creditMc(user.id, diff, LedgerOperation.INSTAGRAM_TRANSCRIBE_REFUND, "real-cost reconcile");
|
|
16957
|
+
else if (diff < 0) await debitMc(user.id, -diff, LedgerOperation.INSTAGRAM_TRANSCRIBE, sourceUrl.href);
|
|
16859
16958
|
}
|
|
16860
16959
|
}
|
|
16861
16960
|
const result = {
|
|
@@ -16870,7 +16969,7 @@ var init_instagram_routes = __esm({
|
|
|
16870
16969
|
return c.json(result);
|
|
16871
16970
|
} catch (err) {
|
|
16872
16971
|
const msg = err instanceof Error ? err.message : String(err);
|
|
16873
|
-
if (transcriptDebited) await creditMc(user.id,
|
|
16972
|
+
if (transcriptDebited) await creditMc(user.id, MEDIA_TRANSCRIBE_HOLD_MC, LedgerOperation.INSTAGRAM_TRANSCRIBE_REFUND, "failed call");
|
|
16874
16973
|
if (mediaDebited) await creditMc(user.id, MC_COSTS.instagram_media, LedgerOperation.INSTAGRAM_MEDIA_REFUND, "failed call");
|
|
16875
16974
|
await logRequestEvent({ userId: user.id, source: "instagram_media_download", status: "failed", query: sourceUrl.href, error: msg });
|
|
16876
16975
|
if (msg.toLowerCase().includes("blocked") || msg.toLowerCase().includes("captcha")) return c.json({ error: msg }, 503);
|
|
@@ -17011,10 +17110,23 @@ var init_reddit_routes = __esm({
|
|
|
17011
17110
|
result.comments = result.comments.slice(0, body.maxComments);
|
|
17012
17111
|
result.numComments = result.comments.length;
|
|
17013
17112
|
}
|
|
17113
|
+
const commentCount = Array.isArray(result.comments) ? result.comments.length : 0;
|
|
17114
|
+
commentDebitMc = commentCount * MC_COSTS.reddit_comment;
|
|
17115
|
+
if (commentDebitMc > 0) {
|
|
17116
|
+
const commentDebit = await debitMc(user.id, commentDebitMc, LedgerOperation.REDDIT_COMMENT, body.url);
|
|
17117
|
+
if (!commentDebit.ok) {
|
|
17118
|
+
const affordable = Math.floor((commentDebit.balance_mc ?? 0) / MC_COSTS.reddit_comment);
|
|
17119
|
+
result.comments = result.comments.slice(0, affordable);
|
|
17120
|
+
result.numComments = affordable;
|
|
17121
|
+
commentDebitMc = affordable * MC_COSTS.reddit_comment;
|
|
17122
|
+
if (commentDebitMc > 0) await debitMc(user.id, commentDebitMc, LedgerOperation.REDDIT_COMMENT, body.url);
|
|
17123
|
+
}
|
|
17124
|
+
}
|
|
17014
17125
|
await logRequestEvent({ userId: user.id, source: "reddit_thread", status: "done", query: body.url, resultCount: Number(result.numComments ?? 0), result });
|
|
17015
17126
|
return c.json(result);
|
|
17016
17127
|
} catch (err) {
|
|
17017
17128
|
if (debited && !refunded) await creditMc(user.id, MC_COSTS.reddit_thread, LedgerOperation.REDDIT_THREAD_REFUND, "failed call");
|
|
17129
|
+
if (commentDebitMc > 0) await creditMc(user.id, commentDebitMc, LedgerOperation.REDDIT_COMMENT_REFUND, "failed call");
|
|
17018
17130
|
const msg = err instanceof Error ? err.message : String(err);
|
|
17019
17131
|
await logRequestEvent({ userId: user.id, source: "reddit_thread", status: "failed", query: body.url, error: msg });
|
|
17020
17132
|
return c.json({ error: msg }, 500);
|
|
@@ -27287,7 +27399,7 @@ var PACKAGE_VERSION;
|
|
|
27287
27399
|
var init_version = __esm({
|
|
27288
27400
|
"src/version.ts"() {
|
|
27289
27401
|
"use strict";
|
|
27290
|
-
PACKAGE_VERSION = "0.4.
|
|
27402
|
+
PACKAGE_VERSION = "0.4.13";
|
|
27291
27403
|
}
|
|
27292
27404
|
});
|
|
27293
27405
|
|
|
@@ -38373,6 +38485,11 @@ function opForCostPath(p) {
|
|
|
38373
38485
|
return "fb_ad";
|
|
38374
38486
|
}
|
|
38375
38487
|
if (p.startsWith("/instagram/")) return "instagram";
|
|
38488
|
+
if (p.startsWith("/google-ads/")) {
|
|
38489
|
+
if (p.endsWith("/transcribe")) return "google_ads_transcribe";
|
|
38490
|
+
if (p.endsWith("/search")) return "google_ads_search";
|
|
38491
|
+
return "google_ads_intel";
|
|
38492
|
+
}
|
|
38376
38493
|
if (p === "/reddit/thread") return "reddit_thread";
|
|
38377
38494
|
if (p.startsWith("/trustpilot/")) return "trustpilot_reviews";
|
|
38378
38495
|
if (p.startsWith("/g2/")) return "g2_reviews";
|