mcp-scraper 0.4.12 → 0.4.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bin/api-server.cjs +196 -79
- package/dist/bin/api-server.cjs.map +1 -1
- package/dist/bin/api-server.js +2 -2
- package/dist/bin/mcp-scraper-cli.cjs +1 -1
- package/dist/bin/mcp-scraper-cli.cjs.map +1 -1
- package/dist/bin/mcp-scraper-cli.js +1 -1
- package/dist/bin/mcp-scraper-install.cjs +1 -1
- package/dist/bin/mcp-scraper-install.cjs.map +1 -1
- package/dist/bin/mcp-scraper-install.js +1 -1
- package/dist/bin/mcp-stdio-server.cjs +1 -1
- package/dist/bin/mcp-stdio-server.cjs.map +1 -1
- package/dist/bin/mcp-stdio-server.js +2 -2
- package/dist/bin/paa-harvest.js +1 -1
- package/dist/{chunk-2YY46QYT.js → chunk-AUCXKRRH.js} +3 -1
- package/dist/{chunk-6EXP6DQG.js → chunk-EA2AANFI.js} +2 -2
- package/dist/chunk-O2D5HGKF.js +7 -0
- package/dist/chunk-O2D5HGKF.js.map +1 -0
- package/dist/{chunk-EMY7ELRU.js → chunk-SIPXEFTR.js} +14 -10
- package/dist/chunk-SIPXEFTR.js.map +1 -0
- package/dist/{chunk-ONIOF5XW.js → chunk-VQV7MB2S.js} +2 -2
- package/dist/index.js +1 -1
- package/dist/{server-UNID3SJU.js → server-HNZG36IP.js} +178 -72
- package/dist/server-HNZG36IP.js.map +1 -0
- package/dist/{site-extract-repository-HTIY52MW.js → site-extract-repository-6X2C7RLF.js} +3 -3
- package/dist/{worker-HTYZAYGB.js → worker-WCSKUAOT.js} +3 -3
- package/package.json +1 -1
- package/dist/chunk-EMY7ELRU.js.map +0 -1
- package/dist/chunk-SHXJQQOH.js +0 -7
- package/dist/chunk-SHXJQQOH.js.map +0 -1
- package/dist/server-UNID3SJU.js.map +0 -1
- /package/dist/{chunk-2YY46QYT.js.map → chunk-AUCXKRRH.js.map} +0 -0
- /package/dist/{chunk-6EXP6DQG.js.map → chunk-EA2AANFI.js.map} +0 -0
- /package/dist/{chunk-ONIOF5XW.js.map → chunk-VQV7MB2S.js.map} +0 -0
- /package/dist/{site-extract-repository-HTIY52MW.js.map → site-extract-repository-6X2C7RLF.js.map} +0 -0
- /package/dist/{worker-HTYZAYGB.js.map → worker-WCSKUAOT.js.map} +0 -0
|
@@ -11,12 +11,14 @@ import {
|
|
|
11
11
|
currentCostContext,
|
|
12
12
|
deleteKernelProxyId,
|
|
13
13
|
harvest,
|
|
14
|
+
kernelCostUsd,
|
|
14
15
|
migrateCostTelemetry,
|
|
15
16
|
recordKernelSession,
|
|
16
17
|
recordVendorUsage,
|
|
17
18
|
resolveKernelProxyId,
|
|
18
|
-
runWithCostContext
|
|
19
|
-
|
|
19
|
+
runWithCostContext,
|
|
20
|
+
vendorCostUsd
|
|
21
|
+
} from "./chunk-AUCXKRRH.js";
|
|
20
22
|
import {
|
|
21
23
|
HttpMcpToolExecutor,
|
|
22
24
|
MemoryMcpToolExecutor,
|
|
@@ -31,7 +33,7 @@ import {
|
|
|
31
33
|
registerSerpIntelligenceCaptureTools,
|
|
32
34
|
sanitizeAttempts,
|
|
33
35
|
sanitizeHarvestResult
|
|
34
|
-
} from "./chunk-
|
|
36
|
+
} from "./chunk-EA2AANFI.js";
|
|
35
37
|
import {
|
|
36
38
|
auditImages,
|
|
37
39
|
getBlobStore
|
|
@@ -65,7 +67,7 @@ import {
|
|
|
65
67
|
RawMapsOverviewSchema,
|
|
66
68
|
RawMapsReviewStatsSchema
|
|
67
69
|
} from "./chunk-XGIPATLV.js";
|
|
68
|
-
import "./chunk-
|
|
70
|
+
import "./chunk-O2D5HGKF.js";
|
|
69
71
|
import {
|
|
70
72
|
completeExtractJob,
|
|
71
73
|
countSuccessfulPages,
|
|
@@ -76,7 +78,7 @@ import {
|
|
|
76
78
|
saveExtractPages,
|
|
77
79
|
setExtractJobTotal,
|
|
78
80
|
settleExtractJob
|
|
79
|
-
} from "./chunk-
|
|
81
|
+
} from "./chunk-VQV7MB2S.js";
|
|
80
82
|
import {
|
|
81
83
|
BROWSER_OPEN_MIN_BALANCE_MC,
|
|
82
84
|
CONCURRENCY_PRICE_ID,
|
|
@@ -86,6 +88,8 @@ import {
|
|
|
86
88
|
MC_COSTS,
|
|
87
89
|
MC_PER_CREDIT,
|
|
88
90
|
MC_PER_USD,
|
|
91
|
+
MEDIA_TRANSCRIBE_HOLD_MC,
|
|
92
|
+
MEDIA_TRANSCRIBE_MARGIN_MULTIPLE,
|
|
89
93
|
MEMORY_AI_MARGIN_MULTIPLE,
|
|
90
94
|
MEMORY_PLANS,
|
|
91
95
|
MEMORY_PLAN_QUOTA,
|
|
@@ -97,7 +101,7 @@ import {
|
|
|
97
101
|
browserActiveCostMc,
|
|
98
102
|
concurrencySlotBillingInfo,
|
|
99
103
|
insufficientBalanceResponse
|
|
100
|
-
} from "./chunk-
|
|
104
|
+
} from "./chunk-SIPXEFTR.js";
|
|
101
105
|
import {
|
|
102
106
|
CaptchaError,
|
|
103
107
|
RECAPTCHA_INSTRUCTIONS,
|
|
@@ -8699,8 +8703,92 @@ async function ytHarvest(rawOptions) {
|
|
|
8699
8703
|
);
|
|
8700
8704
|
}
|
|
8701
8705
|
|
|
8702
|
-
// src/youtube/
|
|
8706
|
+
// src/youtube/audio-resolver.ts
|
|
8707
|
+
function mediaResolverUrl() {
|
|
8708
|
+
return process.env.FRAMES_SERVICE_URL?.trim() || void 0;
|
|
8709
|
+
}
|
|
8710
|
+
function mediaResolverSecret() {
|
|
8711
|
+
return process.env.FRAMES_SERVICE_SECRET?.trim() || void 0;
|
|
8712
|
+
}
|
|
8713
|
+
function mediaResolverConfigured() {
|
|
8714
|
+
return Boolean(mediaResolverUrl() && mediaResolverSecret());
|
|
8715
|
+
}
|
|
8716
|
+
var RESOLVE_TIMEOUT_MS = 24e4;
|
|
8717
|
+
async function resolveYouTubeAudio(videoId) {
|
|
8718
|
+
const base = mediaResolverUrl();
|
|
8719
|
+
const secret2 = mediaResolverSecret();
|
|
8720
|
+
if (!base || !secret2) return null;
|
|
8721
|
+
const controller = new AbortController();
|
|
8722
|
+
const timer = setTimeout(() => controller.abort(), RESOLVE_TIMEOUT_MS);
|
|
8723
|
+
try {
|
|
8724
|
+
const resp = await fetch(`${base.replace(/\/+$/, "")}/resolve-audio`, {
|
|
8725
|
+
method: "POST",
|
|
8726
|
+
headers: { "content-type": "application/json", "x-frames-secret": secret2 },
|
|
8727
|
+
body: JSON.stringify({ videoUrl: `https://www.youtube.com/watch?v=${videoId}` }),
|
|
8728
|
+
signal: controller.signal
|
|
8729
|
+
});
|
|
8730
|
+
if (!resp.ok) {
|
|
8731
|
+
const detail = await resp.text().catch(() => "");
|
|
8732
|
+
console.warn(`[audio-resolver] ${videoId}: http_${resp.status} ${detail.slice(0, 200)}`);
|
|
8733
|
+
return null;
|
|
8734
|
+
}
|
|
8735
|
+
const data = await resp.json();
|
|
8736
|
+
if (!data.mediaUrl) {
|
|
8737
|
+
console.warn(`[audio-resolver] ${videoId}: response missing mediaUrl`);
|
|
8738
|
+
return null;
|
|
8739
|
+
}
|
|
8740
|
+
return {
|
|
8741
|
+
mediaUrl: data.mediaUrl,
|
|
8742
|
+
durationS: Number(data.durationS) || 0,
|
|
8743
|
+
sizeBytes: Number(data.sizeBytes) || 0,
|
|
8744
|
+
mimeType: data.mimeType ?? "audio/mp4"
|
|
8745
|
+
};
|
|
8746
|
+
} catch (err) {
|
|
8747
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
8748
|
+
console.warn(`[audio-resolver] ${videoId}: resolve failed \u2014 ${msg}`);
|
|
8749
|
+
return null;
|
|
8750
|
+
} finally {
|
|
8751
|
+
clearTimeout(timer);
|
|
8752
|
+
}
|
|
8753
|
+
}
|
|
8754
|
+
|
|
8755
|
+
// src/services/media-transcription.ts
|
|
8703
8756
|
import { fal } from "@fal-ai/client";
|
|
8757
|
+
function transcriptMarkdown(title, text, chunks, durationMs) {
|
|
8758
|
+
const fmtTs2 = (s) => `${Math.floor(s / 60)}:${String(Math.floor(s % 60)).padStart(2, "0")}`;
|
|
8759
|
+
const lines = [title, "", `*Transcribed in ${(durationMs / 1e3).toFixed(1)}s*`, "", "## Full Text", "", text, ""];
|
|
8760
|
+
if (chunks.length) {
|
|
8761
|
+
lines.push("## Timestamped Segments", "");
|
|
8762
|
+
for (const ch of chunks) {
|
|
8763
|
+
lines.push(`**[${fmtTs2(ch.timestamp[0])} -> ${fmtTs2(ch.timestamp[1])}]** ${ch.text.trim()}`, "");
|
|
8764
|
+
}
|
|
8765
|
+
}
|
|
8766
|
+
return lines.join("\n");
|
|
8767
|
+
}
|
|
8768
|
+
async function transcribeMediaUrl(mediaUrl, markdownTitle = "# Media Transcript") {
|
|
8769
|
+
const startMs = Date.now();
|
|
8770
|
+
const result = await fal.subscribe("fal-ai/wizper", {
|
|
8771
|
+
input: { audio_url: mediaUrl, task: "transcribe", language: "en" },
|
|
8772
|
+
logs: false,
|
|
8773
|
+
pollInterval: 3e3
|
|
8774
|
+
});
|
|
8775
|
+
const data = result.data;
|
|
8776
|
+
const text = data.text ?? "";
|
|
8777
|
+
const chunks = data.chunks ?? [];
|
|
8778
|
+
const durationMs = Date.now() - startMs;
|
|
8779
|
+
void recordVendorUsage({ vendor: "fal_wizper", model: "fal-ai/wizper", units: durationMs / 1e3, unitType: "compute_sec" });
|
|
8780
|
+
const costUsd = vendorCostUsd("fal_wizper", durationMs / 1e3);
|
|
8781
|
+
return {
|
|
8782
|
+
text,
|
|
8783
|
+
chunks,
|
|
8784
|
+
durationMs,
|
|
8785
|
+
costUsd,
|
|
8786
|
+
markdown: transcriptMarkdown(markdownTitle, text, chunks, durationMs)
|
|
8787
|
+
};
|
|
8788
|
+
}
|
|
8789
|
+
|
|
8790
|
+
// src/youtube/CaptionFetcher.ts
|
|
8791
|
+
import { fal as fal2 } from "@fal-ai/client";
|
|
8704
8792
|
async function fetchViaYoutubeTranscript(videoId) {
|
|
8705
8793
|
try {
|
|
8706
8794
|
const { YoutubeTranscript } = await import("youtube-transcript");
|
|
@@ -8862,12 +8950,12 @@ async function transcribeWithWizper(videoId, audio, startMs) {
|
|
|
8862
8950
|
const falKey = process.env.FAL_KEY;
|
|
8863
8951
|
if (!falKey) return null;
|
|
8864
8952
|
try {
|
|
8865
|
-
|
|
8953
|
+
fal2.config({ credentials: falKey });
|
|
8866
8954
|
const ext = audio.mimeType.includes("webm") ? "webm" : audio.mimeType.includes("mp4") ? "m4a" : "audio";
|
|
8867
8955
|
const audioFile = new File([new Uint8Array(audio.bytes)], `audio.${ext}`, { type: audio.mimeType });
|
|
8868
|
-
const uploadedUrl = await
|
|
8956
|
+
const uploadedUrl = await fal2.storage.upload(audioFile);
|
|
8869
8957
|
const subscribeStartMs = Date.now();
|
|
8870
|
-
const result = await
|
|
8958
|
+
const result = await fal2.subscribe("fal-ai/wizper", {
|
|
8871
8959
|
input: { audio_url: uploadedUrl, task: "transcribe", language: "en", chunk_level: "segment" },
|
|
8872
8960
|
logs: false,
|
|
8873
8961
|
pollInterval: 3e3
|
|
@@ -8986,6 +9074,28 @@ async function fetchViaKernelSession(videoId) {
|
|
|
8986
9074
|
}
|
|
8987
9075
|
return null;
|
|
8988
9076
|
}
|
|
9077
|
+
async function fetchViaResolvedAudio(videoId) {
|
|
9078
|
+
if (!mediaResolverConfigured()) return null;
|
|
9079
|
+
return runWithCostContext({ ...currentCostContext(), op: "yt_transcription", subOp: "audio-whisper" }, async () => {
|
|
9080
|
+
const resolved = await resolveYouTubeAudio(videoId);
|
|
9081
|
+
if (!resolved) return null;
|
|
9082
|
+
try {
|
|
9083
|
+
const transcript = await transcribeMediaUrl(resolved.mediaUrl, `# Transcript \u2014 ${videoId}`);
|
|
9084
|
+
if (!transcript.text) return null;
|
|
9085
|
+
const chunks = transcript.chunks.map((c) => ({ timestamp: c.timestamp, text: c.text.trim() }));
|
|
9086
|
+
return {
|
|
9087
|
+
videoId,
|
|
9088
|
+
text: transcript.text,
|
|
9089
|
+
chunks,
|
|
9090
|
+
durationMs: resolved.durationS > 0 ? Math.round(resolved.durationS * 1e3) : transcript.durationMs,
|
|
9091
|
+
method: "audio-whisper"
|
|
9092
|
+
};
|
|
9093
|
+
} catch (err) {
|
|
9094
|
+
console.warn(`[CaptionFetcher] transcription of resolved audio failed for ${videoId}: ${err instanceof Error ? err.message : String(err)}`);
|
|
9095
|
+
return null;
|
|
9096
|
+
}
|
|
9097
|
+
});
|
|
9098
|
+
}
|
|
8989
9099
|
function finalizeCaptions(result) {
|
|
8990
9100
|
const chunks = result.chunks.map((c) => {
|
|
8991
9101
|
const startOk = Number.isFinite(c.timestamp[0]);
|
|
@@ -9006,8 +9116,10 @@ async function fetchCaptions(videoId) {
|
|
|
9006
9116
|
if (primary) return finalizeCaptions(primary);
|
|
9007
9117
|
const result = await fetchViaKernelSession(videoId);
|
|
9008
9118
|
if (result) return finalizeCaptions(result);
|
|
9119
|
+
const viaResolvedAudio = await fetchViaResolvedAudio(videoId);
|
|
9120
|
+
if (viaResolvedAudio) return finalizeCaptions(viaResolvedAudio);
|
|
9009
9121
|
throw new Error(
|
|
9010
|
-
`Could not transcribe ${videoId}
|
|
9122
|
+
`Could not transcribe ${videoId}: it has no caption track and its audio could not be retrieved.`
|
|
9011
9123
|
);
|
|
9012
9124
|
}
|
|
9013
9125
|
|
|
@@ -10374,41 +10486,6 @@ async function extractFacebookOrganicVideoFromPage(page, sourceUrl, quality = "b
|
|
|
10374
10486
|
|
|
10375
10487
|
// src/api/facebook-ad-routes.ts
|
|
10376
10488
|
import { fal as fal3 } from "@fal-ai/client";
|
|
10377
|
-
|
|
10378
|
-
// src/services/media-transcription.ts
|
|
10379
|
-
import { fal as fal2 } from "@fal-ai/client";
|
|
10380
|
-
function transcriptMarkdown(title, text, chunks, durationMs) {
|
|
10381
|
-
const fmtTs2 = (s) => `${Math.floor(s / 60)}:${String(Math.floor(s % 60)).padStart(2, "0")}`;
|
|
10382
|
-
const lines = [title, "", `*Transcribed in ${(durationMs / 1e3).toFixed(1)}s*`, "", "## Full Text", "", text, ""];
|
|
10383
|
-
if (chunks.length) {
|
|
10384
|
-
lines.push("## Timestamped Segments", "");
|
|
10385
|
-
for (const ch of chunks) {
|
|
10386
|
-
lines.push(`**[${fmtTs2(ch.timestamp[0])} -> ${fmtTs2(ch.timestamp[1])}]** ${ch.text.trim()}`, "");
|
|
10387
|
-
}
|
|
10388
|
-
}
|
|
10389
|
-
return lines.join("\n");
|
|
10390
|
-
}
|
|
10391
|
-
async function transcribeMediaUrl(mediaUrl, markdownTitle = "# Media Transcript") {
|
|
10392
|
-
const startMs = Date.now();
|
|
10393
|
-
const result = await fal2.subscribe("fal-ai/wizper", {
|
|
10394
|
-
input: { audio_url: mediaUrl, task: "transcribe", language: "en" },
|
|
10395
|
-
logs: false,
|
|
10396
|
-
pollInterval: 3e3
|
|
10397
|
-
});
|
|
10398
|
-
const data = result.data;
|
|
10399
|
-
const text = data.text ?? "";
|
|
10400
|
-
const chunks = data.chunks ?? [];
|
|
10401
|
-
const durationMs = Date.now() - startMs;
|
|
10402
|
-
void recordVendorUsage({ vendor: "fal_wizper", model: "fal-ai/wizper", units: durationMs / 1e3, unitType: "compute_sec" });
|
|
10403
|
-
return {
|
|
10404
|
-
text,
|
|
10405
|
-
chunks,
|
|
10406
|
-
durationMs,
|
|
10407
|
-
markdown: transcriptMarkdown(markdownTitle, text, chunks, durationMs)
|
|
10408
|
-
};
|
|
10409
|
-
}
|
|
10410
|
-
|
|
10411
|
-
// src/api/facebook-ad-routes.ts
|
|
10412
10489
|
var FacebookAdBodySchema = z13.object({
|
|
10413
10490
|
url: z13.string().trim().optional(),
|
|
10414
10491
|
libraryId: z13.string().trim().optional(),
|
|
@@ -10589,16 +10666,16 @@ facebookAdApp.post("/transcribe", createApiKeyAuth(), async (c) => {
|
|
|
10589
10666
|
});
|
|
10590
10667
|
if (!gate.ok) return c.json(concurrencyLimitExceededResponse(gate), 429, { "Retry-After": String(gate.retryAfterSeconds) });
|
|
10591
10668
|
fal3.config({ credentials: process.env.FAL_KEY });
|
|
10592
|
-
const holdMc =
|
|
10669
|
+
const holdMc = MEDIA_TRANSCRIBE_HOLD_MC;
|
|
10593
10670
|
let debited = false;
|
|
10594
10671
|
try {
|
|
10595
10672
|
const { ok, balance_mc } = await debitMc(fbUser.id, holdMc, LedgerOperation.FB_TRANSCRIBE, videoUrl);
|
|
10596
10673
|
if (!ok) return c.json(insufficientBalanceResponse(balance_mc, holdMc), 402);
|
|
10597
10674
|
debited = true;
|
|
10598
10675
|
const transcript = await transcribeFacebookVideoUrl(videoUrl, "# Facebook Ad Transcript");
|
|
10599
|
-
const
|
|
10600
|
-
const diff = holdMc -
|
|
10601
|
-
if (diff > 0) await creditMc(fbUser.id, diff, LedgerOperation.FB_TRANSCRIBE_REFUND, "
|
|
10676
|
+
const settleMc = Math.round(transcript.costUsd * MEDIA_TRANSCRIBE_MARGIN_MULTIPLE * MC_PER_USD);
|
|
10677
|
+
const diff = holdMc - settleMc;
|
|
10678
|
+
if (diff > 0) await creditMc(fbUser.id, diff, LedgerOperation.FB_TRANSCRIBE_REFUND, "real-cost reconcile");
|
|
10602
10679
|
else if (diff < 0) await debitMc(fbUser.id, -diff, LedgerOperation.FB_TRANSCRIBE, videoUrl);
|
|
10603
10680
|
await logRequestEvent({ userId: fbUser.id, source: "facebook_transcribe", status: "done", query: videoUrl, resultCount: transcript.chunks.length, result: { text: transcript.text, chunks: transcript.chunks, durationMs: transcript.durationMs } });
|
|
10604
10681
|
return c.json(transcript);
|
|
@@ -10634,20 +10711,22 @@ facebookAdApp.post("/video-transcribe", createApiKeyAuth(), async (c) => {
|
|
|
10634
10711
|
if (!gate.ok) return c.json(concurrencyLimitExceededResponse(gate), 429, { "Retry-After": String(gate.retryAfterSeconds) });
|
|
10635
10712
|
fal3.config({ credentials: process.env.FAL_KEY });
|
|
10636
10713
|
const driver = new BrowserDriver();
|
|
10637
|
-
const holdMc =
|
|
10714
|
+
const holdMc = MEDIA_TRANSCRIBE_HOLD_MC;
|
|
10638
10715
|
let debited = false;
|
|
10639
10716
|
try {
|
|
10640
10717
|
const { ok, balance_mc } = await debitMc(fbUser.id, holdMc, LedgerOperation.FB_TRANSCRIBE, sourceUrl.href);
|
|
10641
10718
|
if (!ok) return c.json(insufficientBalanceResponse(balance_mc, holdMc), 402);
|
|
10642
10719
|
debited = true;
|
|
10720
|
+
const kernelStartMs = Date.now();
|
|
10643
10721
|
await driver.launch(await kernelLaunchOptsResidential());
|
|
10644
10722
|
await driver.navigateTo(sourceUrl.href);
|
|
10645
10723
|
const page = driver.getPage();
|
|
10646
10724
|
const video = await extractFacebookOrganicVideoFromPage(page, sourceUrl.href, body.quality);
|
|
10725
|
+
const kernelMs = Date.now() - kernelStartMs;
|
|
10647
10726
|
const transcript = await transcribeFacebookVideoUrl(video.videoUrl, "# Facebook Organic Video Transcript");
|
|
10648
|
-
const
|
|
10649
|
-
const diff = holdMc -
|
|
10650
|
-
if (diff > 0) await creditMc(fbUser.id, diff, LedgerOperation.FB_TRANSCRIBE_REFUND, "
|
|
10727
|
+
const settleMc = Math.round((transcript.costUsd + kernelCostUsd(kernelMs, true)) * MEDIA_TRANSCRIBE_MARGIN_MULTIPLE * MC_PER_USD);
|
|
10728
|
+
const diff = holdMc - settleMc;
|
|
10729
|
+
if (diff > 0) await creditMc(fbUser.id, diff, LedgerOperation.FB_TRANSCRIBE_REFUND, "real-cost reconcile");
|
|
10651
10730
|
else if (diff < 0) await debitMc(fbUser.id, -diff, LedgerOperation.FB_TRANSCRIBE, sourceUrl.href);
|
|
10652
10731
|
const result = {
|
|
10653
10732
|
...video,
|
|
@@ -11039,9 +11118,11 @@ googleAdsApp.post("/search", createApiKeyAuth(), async (c) => {
|
|
|
11039
11118
|
const { ok, balance_mc } = await debitMc(user.id, MC_COSTS.google_ads_search, LedgerOperation.GOOGLE_ADS_SEARCH, body.query.trim());
|
|
11040
11119
|
if (!ok) return c.json(insufficientBalanceResponse(balance_mc, MC_COSTS.google_ads_search), 402);
|
|
11041
11120
|
debited = true;
|
|
11042
|
-
await
|
|
11043
|
-
|
|
11044
|
-
|
|
11121
|
+
const result = await runWithCostContext({ ...currentCostContext(), op: "google_ads_search", userId: user.id }, async () => {
|
|
11122
|
+
await driver.launch(await kernelLaunchOptsResidential2());
|
|
11123
|
+
const extractor = new GoogleAdsExtractor(driver);
|
|
11124
|
+
return extractor.search(body.query.trim(), region, maxResults);
|
|
11125
|
+
});
|
|
11045
11126
|
if (result.advertisers.length === 0) {
|
|
11046
11127
|
await creditMc(user.id, MC_COSTS.google_ads_search, LedgerOperation.GOOGLE_ADS_SEARCH_REFUND, "empty result");
|
|
11047
11128
|
refunded = true;
|
|
@@ -11077,9 +11158,11 @@ googleAdsApp.post("/page-intel", createApiKeyAuth(), async (c) => {
|
|
|
11077
11158
|
const { ok, balance_mc } = await debitMc(user.id, MC_COSTS.google_ads_intel, LedgerOperation.GOOGLE_ADS_INTEL, queryLabel);
|
|
11078
11159
|
if (!ok) return c.json(insufficientBalanceResponse(balance_mc, MC_COSTS.google_ads_intel), 402);
|
|
11079
11160
|
debited = true;
|
|
11080
|
-
await
|
|
11081
|
-
|
|
11082
|
-
|
|
11161
|
+
const result = await runWithCostContext({ ...currentCostContext(), op: "google_ads_intel", userId: user.id }, async () => {
|
|
11162
|
+
await driver.launch(await kernelLaunchOptsResidential2());
|
|
11163
|
+
const extractor = new GoogleAdsExtractor(driver);
|
|
11164
|
+
return extractor.pageIntel({ advertiserId: body.advertiserId, domain: body.domain, region, maxAds });
|
|
11165
|
+
});
|
|
11083
11166
|
if (result.ads.length === 0) {
|
|
11084
11167
|
await creditMc(user.id, MC_COSTS.google_ads_intel, LedgerOperation.GOOGLE_ADS_INTEL_REFUND, "empty result");
|
|
11085
11168
|
refunded = true;
|
|
@@ -11110,16 +11193,16 @@ googleAdsApp.post("/transcribe", createApiKeyAuth(), async (c) => {
|
|
|
11110
11193
|
const gate = await acquireConcurrencyGate(user, "google_ads_transcribe", { reuseLockId: c.req.header("x-mcp-scraper-concurrency-lock"), metadata: { videoUrl } });
|
|
11111
11194
|
if (!gate.ok) return c.json(concurrencyLimitExceededResponse(gate), 429, { "Retry-After": String(gate.retryAfterSeconds) });
|
|
11112
11195
|
fal4.config({ credentials: process.env.FAL_KEY });
|
|
11113
|
-
const holdMc =
|
|
11196
|
+
const holdMc = MEDIA_TRANSCRIBE_HOLD_MC;
|
|
11114
11197
|
let debited = false;
|
|
11115
11198
|
try {
|
|
11116
11199
|
const { ok, balance_mc } = await debitMc(user.id, holdMc, LedgerOperation.GOOGLE_ADS_TRANSCRIBE, videoUrl);
|
|
11117
11200
|
if (!ok) return c.json(insufficientBalanceResponse(balance_mc, holdMc), 402);
|
|
11118
11201
|
debited = true;
|
|
11119
|
-
const transcript = await transcribeMediaUrl(videoUrl, "# Google Ad Transcript");
|
|
11120
|
-
const
|
|
11121
|
-
const diff = holdMc -
|
|
11122
|
-
if (diff > 0) await creditMc(user.id, diff, LedgerOperation.GOOGLE_ADS_TRANSCRIBE_REFUND, "
|
|
11202
|
+
const transcript = await runWithCostContext({ ...currentCostContext(), op: "google_ads_transcribe", userId: user.id }, () => transcribeMediaUrl(videoUrl, "# Google Ad Transcript"));
|
|
11203
|
+
const settleMc = Math.round(transcript.costUsd * MEDIA_TRANSCRIBE_MARGIN_MULTIPLE * MC_PER_USD);
|
|
11204
|
+
const diff = holdMc - settleMc;
|
|
11205
|
+
if (diff > 0) await creditMc(user.id, diff, LedgerOperation.GOOGLE_ADS_TRANSCRIBE_REFUND, "real-cost reconcile");
|
|
11123
11206
|
else if (diff < 0) await debitMc(user.id, -diff, LedgerOperation.GOOGLE_ADS_TRANSCRIBE, videoUrl);
|
|
11124
11207
|
await logRequestEvent({ userId: user.id, source: "google_ads_transcribe", status: "done", query: videoUrl, resultCount: transcript.chunks.length, result: { text: transcript.text, chunks: transcript.chunks, durationMs: transcript.durationMs } });
|
|
11125
11208
|
return c.json(transcript);
|
|
@@ -11768,10 +11851,15 @@ instagramApp.post("/media-download", createApiKeyAuth(), async (c) => {
|
|
|
11768
11851
|
if (!audioUrl) {
|
|
11769
11852
|
warnings.push("Transcript requested, but no audio/video track was available.");
|
|
11770
11853
|
} else {
|
|
11771
|
-
const
|
|
11772
|
-
|
|
11854
|
+
const holdMc = MEDIA_TRANSCRIBE_HOLD_MC;
|
|
11855
|
+
const tx = await debitMc(user.id, holdMc, LedgerOperation.INSTAGRAM_TRANSCRIBE, sourceUrl.href);
|
|
11856
|
+
if (!tx.ok) return c.json(insufficientBalanceResponse(tx.balance_mc, holdMc), 402);
|
|
11773
11857
|
transcriptDebited = true;
|
|
11774
11858
|
transcript = await transcribeMediaUrl(audioUrl, "# Instagram Media Transcript");
|
|
11859
|
+
const settleMc = Math.round(transcript.costUsd * MEDIA_TRANSCRIBE_MARGIN_MULTIPLE * MC_PER_USD);
|
|
11860
|
+
const diff = holdMc - settleMc;
|
|
11861
|
+
if (diff > 0) await creditMc(user.id, diff, LedgerOperation.INSTAGRAM_TRANSCRIBE_REFUND, "real-cost reconcile");
|
|
11862
|
+
else if (diff < 0) await debitMc(user.id, -diff, LedgerOperation.INSTAGRAM_TRANSCRIBE, sourceUrl.href);
|
|
11775
11863
|
}
|
|
11776
11864
|
}
|
|
11777
11865
|
const result = {
|
|
@@ -11786,7 +11874,7 @@ instagramApp.post("/media-download", createApiKeyAuth(), async (c) => {
|
|
|
11786
11874
|
return c.json(result);
|
|
11787
11875
|
} catch (err) {
|
|
11788
11876
|
const msg = err instanceof Error ? err.message : String(err);
|
|
11789
|
-
if (transcriptDebited) await creditMc(user.id,
|
|
11877
|
+
if (transcriptDebited) await creditMc(user.id, MEDIA_TRANSCRIBE_HOLD_MC, LedgerOperation.INSTAGRAM_TRANSCRIBE_REFUND, "failed call");
|
|
11790
11878
|
if (mediaDebited) await creditMc(user.id, MC_COSTS.instagram_media, LedgerOperation.INSTAGRAM_MEDIA_REFUND, "failed call");
|
|
11791
11879
|
await logRequestEvent({ userId: user.id, source: "instagram_media_download", status: "failed", query: sourceUrl.href, error: msg });
|
|
11792
11880
|
if (msg.toLowerCase().includes("blocked") || msg.toLowerCase().includes("captcha")) return c.json({ error: msg }, 503);
|
|
@@ -11914,10 +12002,23 @@ redditApp.post("/thread", createApiKeyAuth(), async (c) => {
|
|
|
11914
12002
|
result.comments = result.comments.slice(0, body.maxComments);
|
|
11915
12003
|
result.numComments = result.comments.length;
|
|
11916
12004
|
}
|
|
12005
|
+
const commentCount = Array.isArray(result.comments) ? result.comments.length : 0;
|
|
12006
|
+
commentDebitMc = commentCount * MC_COSTS.reddit_comment;
|
|
12007
|
+
if (commentDebitMc > 0) {
|
|
12008
|
+
const commentDebit = await debitMc(user.id, commentDebitMc, LedgerOperation.REDDIT_COMMENT, body.url);
|
|
12009
|
+
if (!commentDebit.ok) {
|
|
12010
|
+
const affordable = Math.floor((commentDebit.balance_mc ?? 0) / MC_COSTS.reddit_comment);
|
|
12011
|
+
result.comments = result.comments.slice(0, affordable);
|
|
12012
|
+
result.numComments = affordable;
|
|
12013
|
+
commentDebitMc = affordable * MC_COSTS.reddit_comment;
|
|
12014
|
+
if (commentDebitMc > 0) await debitMc(user.id, commentDebitMc, LedgerOperation.REDDIT_COMMENT, body.url);
|
|
12015
|
+
}
|
|
12016
|
+
}
|
|
11917
12017
|
await logRequestEvent({ userId: user.id, source: "reddit_thread", status: "done", query: body.url, resultCount: Number(result.numComments ?? 0), result });
|
|
11918
12018
|
return c.json(result);
|
|
11919
12019
|
} catch (err) {
|
|
11920
12020
|
if (debited && !refunded) await creditMc(user.id, MC_COSTS.reddit_thread, LedgerOperation.REDDIT_THREAD_REFUND, "failed call");
|
|
12021
|
+
if (commentDebitMc > 0) await creditMc(user.id, commentDebitMc, LedgerOperation.REDDIT_COMMENT_REFUND, "failed call");
|
|
11921
12022
|
const msg = err instanceof Error ? err.message : String(err);
|
|
11922
12023
|
await logRequestEvent({ userId: user.id, source: "reddit_thread", status: "failed", query: body.url, error: msg });
|
|
11923
12024
|
return c.json({ error: msg }, 500);
|
|
@@ -20408,6 +20509,11 @@ function opForCostPath(p) {
|
|
|
20408
20509
|
return "fb_ad";
|
|
20409
20510
|
}
|
|
20410
20511
|
if (p.startsWith("/instagram/")) return "instagram";
|
|
20512
|
+
if (p.startsWith("/google-ads/")) {
|
|
20513
|
+
if (p.endsWith("/transcribe")) return "google_ads_transcribe";
|
|
20514
|
+
if (p.endsWith("/search")) return "google_ads_search";
|
|
20515
|
+
return "google_ads_intel";
|
|
20516
|
+
}
|
|
20411
20517
|
if (p === "/reddit/thread") return "reddit_thread";
|
|
20412
20518
|
if (p.startsWith("/trustpilot/")) return "trustpilot_reviews";
|
|
20413
20519
|
if (p.startsWith("/g2/")) return "g2_reviews";
|
|
@@ -21818,7 +21924,7 @@ app.get("/cron/tick", async (c) => {
|
|
|
21818
21924
|
if (!process.env.CRON_SECRET || secret2 !== `Bearer ${process.env.CRON_SECRET}`) {
|
|
21819
21925
|
return c.json({ error: "Unauthorized" }, 401);
|
|
21820
21926
|
}
|
|
21821
|
-
const { drainQueue } = await import("./worker-
|
|
21927
|
+
const { drainQueue } = await import("./worker-WCSKUAOT.js");
|
|
21822
21928
|
const budget = { maxJobs: 10, deadlineMs: Date.now() + 28e4 };
|
|
21823
21929
|
const workflowDispatchResult = await dispatchDueWorkflowSchedules(`${new URL(c.req.url).protocol}//${new URL(c.req.url).host}`);
|
|
21824
21930
|
const [results, sweepResult, reapResult, expiredResult, blobCleanup] = await Promise.all([
|
|
@@ -21836,7 +21942,7 @@ app.post("/api/internal/extract-refinalize/:id", async (c) => {
|
|
|
21836
21942
|
return c.json({ error: "Unauthorized" }, 401);
|
|
21837
21943
|
}
|
|
21838
21944
|
const jobId = c.req.param("id");
|
|
21839
|
-
const { getExtractJob: getExtractJob2, completeExtractJob: completeExtractJob2 } = await import("./site-extract-repository-
|
|
21945
|
+
const { getExtractJob: getExtractJob2, completeExtractJob: completeExtractJob2 } = await import("./site-extract-repository-6X2C7RLF.js");
|
|
21840
21946
|
const { assembleExtractArtifacts } = await import("./extract-bundle-COS56ZDO.js");
|
|
21841
21947
|
const job = await getExtractJob2(jobId);
|
|
21842
21948
|
if (!job) return c.json({ error: "job not found" }, 404);
|
|
@@ -21844,7 +21950,7 @@ app.post("/api/internal/extract-refinalize/:id", async (c) => {
|
|
|
21844
21950
|
await completeExtractJob2(jobId, stored);
|
|
21845
21951
|
let settlement = "already_settled_or_refunded";
|
|
21846
21952
|
if (job.billedMc == null && job.userId != null) {
|
|
21847
|
-
const { settleExtractJob: settleExtractJob2, countSuccessfulPages: countSuccessfulPages2 } = await import("./site-extract-repository-
|
|
21953
|
+
const { settleExtractJob: settleExtractJob2, countSuccessfulPages: countSuccessfulPages2 } = await import("./site-extract-repository-6X2C7RLF.js");
|
|
21848
21954
|
const heldMc = Number(job.options.heldMc ?? 0);
|
|
21849
21955
|
const successful = await countSuccessfulPages2(jobId);
|
|
21850
21956
|
const usedMc = Math.min(successful * MC_COSTS.page_scrape, heldMc);
|
|
@@ -22008,4 +22114,4 @@ app.get("/blog/:slug/", (c) => {
|
|
|
22008
22114
|
export {
|
|
22009
22115
|
app
|
|
22010
22116
|
};
|
|
22011
|
-
//# sourceMappingURL=server-
|
|
22117
|
+
//# sourceMappingURL=server-HNZG36IP.js.map
|