mcp-scraper 0.4.6 → 0.4.12

package/dist/{server-R3KVQNOT.js → server-UNID3SJU.js}

@@ -16,7 +16,7 @@ import {
   recordVendorUsage,
   resolveKernelProxyId,
   runWithCostContext
-} from "./chunk-5GAD2AN2.js";
+} from "./chunk-2YY46QYT.js";
 import {
   HttpMcpToolExecutor,
   MemoryMcpToolExecutor,
@@ -31,7 +31,7 @@ import {
   registerSerpIntelligenceCaptureTools,
   sanitizeAttempts,
   sanitizeHarvestResult
-} from "./chunk-IFXACD4K.js";
+} from "./chunk-6EXP6DQG.js";
 import {
   auditImages,
   getBlobStore
@@ -65,7 +65,7 @@ import {
   RawMapsOverviewSchema,
   RawMapsReviewStatsSchema
 } from "./chunk-XGIPATLV.js";
-import "./chunk-5BB6Y3BB.js";
+import "./chunk-SHXJQQOH.js";
 import {
   completeExtractJob,
   countSuccessfulPages,
@@ -76,7 +76,7 @@ import {
   saveExtractPages,
   setExtractJobTotal,
   settleExtractJob
-} from "./chunk-NYAWN7CZ.js";
+} from "./chunk-ONIOF5XW.js";
 import {
   BROWSER_OPEN_MIN_BALANCE_MC,
   CONCURRENCY_PRICE_ID,
@@ -90,12 +90,14 @@ import {
   MEMORY_PLANS,
   MEMORY_PLAN_QUOTA,
   SCHEDULING_PLANS,
+  SITE_AUDIT_LLM_MARGIN_MULTIPLE,
   SUBSCRIPTION_TIERS,
   SUBSCRIPTION_TIER_BY_KEY,
+  VIDEO_ANALYSIS_LLM_MARGIN_MULTIPLE,
   browserActiveCostMc,
   concurrencySlotBillingInfo,
   insufficientBalanceResponse
-} from "./chunk-IVQYNY45.js";
+} from "./chunk-EMY7ELRU.js";
 import {
   CaptchaError,
   RECAPTCHA_INSTRUCTIONS,
@@ -5339,6 +5341,7 @@ function extractJson(raw) {
 var DeepInfraLlmClient = class {
   apiKey;
   model;
+  totalCostUsd = 0;
   constructor(apiKey, model = "Qwen/Qwen3.6-35B-A3B") {
     this.apiKey = apiKey;
     this.model = model;
@@ -5371,6 +5374,7 @@ ${raw}`
 var OpenRouterLlmClient = class {
   apiKey;
   model;
+  totalCostUsd = 0;
   constructor(apiKey, model = "qwen/qwen3-235b-a22b") {
     this.apiKey = apiKey;
     this.model = model;
@@ -5383,6 +5387,9 @@ var OpenRouterLlmClient = class {
     });
     if (!resp.ok) throw new Error(`OpenRouter error ${resp.status}: ${await resp.text()}`);
     const data = await resp.json();
+    const cost = data.usage?.cost ?? 0;
+    this.totalCostUsd += cost;
+    void recordVendorUsage({ vendor: "openrouter", model: this.model, units: cost, unitType: "usd" });
     return data.choices[0].message.content;
   }
   async completeJson(prompt) {
@@ -5408,6 +5415,9 @@ var FallbackLlmClient = class {
   secondary;
   lastServedBy = null;
   lastPrimaryError = null;
+  get totalCostUsd() {
+    return this.primary.totalCostUsd + this.secondary.totalCostUsd;
+  }
   async complete(prompt) {
     try {
       const result = await this.primary.complete(prompt);
@@ -6894,8 +6904,23 @@ var SiteAuditService = class {
   async getJobsForUser(userId) {
     return this.deps.repo.listSiteAuditJobs(userId);
   }
+  async runLlmBilled(userId, source, fn) {
+    const before = this.deps.llm.totalCostUsd;
+    const result = await runWithCostContext({ ...currentCostContext(), op: "audit_site", userId }, fn);
+    const deltaCostUsd = this.deps.llm.totalCostUsd - before;
+    if (deltaCostUsd <= 0) return result;
+    const mc = Math.round(deltaCostUsd * SITE_AUDIT_LLM_MARGIN_MULTIPLE * MC_PER_USD);
+    if (mc <= 0) return result;
+    const { ok } = await debitMc(userId, mc, LedgerOperation.SITE_AUDIT_LLM, source);
+    if (!ok) {
+      console.warn(`[site-audit-service] SITE_AUDIT_LLM surcharge debit failed for user ${userId} (${source}): insufficient balance for ${mc}mc`);
+    }
+    return result;
+  }
   async runIngestPhase(jobId, payload) {
     const { request } = payload;
+    const job = await this.deps.repo.getSiteAuditJob(jobId);
+    if (!job) throw new Error(`[site-audit-service] runIngestPhase: job ${jobId} not found`);
     const input = {
       clientDomain: request.clientDomain,
       sfInternalPath: request.sfInternalPath,
@@ -6910,14 +6935,18 @@ var SiteAuditService = class {
       businessContext: request.businessContext,
       priorAuditMemoryEnabled: request.priorAuditMemoryEnabled
     };
-    const result = await llmParseWithRetry(
-      ValidatedFileManifestSchema,
-      (attempt, error) => {
-        void attempt;
-        void error;
-        return buildIngestValidatePrompt(input);
-      },
-      this.deps.llm
+    const result = await this.runLlmBilled(
+      job.user_id,
+      `${jobId}:phase1-ingest`,
+      () => llmParseWithRetry(
+        ValidatedFileManifestSchema,
+        (attempt, error) => {
+          void attempt;
+          void error;
+          return buildIngestValidatePrompt(input);
+        },
+        this.deps.llm
+      )
     );
     await this.deps.repo.logSiteAuditPhaseComplete(jobId, "phase1-ingest", result);
   }
@@ -6932,23 +6961,27 @@ var SiteAuditService = class {
     const competitorDomainsCsvPath = request.competitorDomainsCsvPath;
     void competitorDomainsCsvPath;
     const competitors = await this._readCompetitorDomains(request);
-    const results = await Promise.all(
-      competitors.map(
-        (domain, idx) => limit(async () => {
-          const competitorIndex = idx + 1;
-          const rawPrompt = buildEnrichCompetitorPrompt({ domain, sessionPath });
-          const prompt = rawPrompt.replace("<competitor-index>", String(competitorIndex));
-          const result = await llmParseWithRetry(
-            EnrichCompetitorOutputSchema,
-            (attempt, error) => {
-              void attempt;
-              void error;
-              return prompt;
-            },
-            this.deps.llm
-          );
-          return result;
-        })
+    const results = await this.runLlmBilled(
+      job.user_id,
+      `${jobId}:phase1-enrich`,
+      () => Promise.all(
+        competitors.map(
+          (domain, idx) => limit(async () => {
+            const competitorIndex = idx + 1;
+            const rawPrompt = buildEnrichCompetitorPrompt({ domain, sessionPath });
+            const prompt = rawPrompt.replace("<competitor-index>", String(competitorIndex));
+            const result = await llmParseWithRetry(
+              EnrichCompetitorOutputSchema,
+              (attempt, error) => {
+                void attempt;
+                void error;
+                return prompt;
+              },
+              this.deps.llm
+            );
+            return result;
+          })
+        )
       )
     );
     await this.deps.repo.logSiteAuditPhaseComplete(jobId, "phase1-ingest", { competitors: results });
@@ -6962,19 +6995,23 @@ var SiteAuditService = class {
     const clientDomain = request.clientDomain;
     const urlInventory = [];
     const directedEdgeCount = 0;
-    const orphanAnnotation = await llmParseWithRetry(
-      OrphanAnnotationOutputSchema,
-      (attempt, error) => {
-        void attempt;
-        void error;
-        return buildGraphPerSitePrompt({
-          siteId: "client",
-          domain: clientDomain,
-          urlInventory,
-          directedEdgeCount
-        });
-      },
-      this.deps.llm
+    const orphanAnnotation = await this.runLlmBilled(
+      job.user_id,
+      `${jobId}:phase2-build-graph`,
+      () => llmParseWithRetry(
+        OrphanAnnotationOutputSchema,
+        (attempt, error) => {
+          void attempt;
+          void error;
+          return buildGraphPerSitePrompt({
+            siteId: "client",
+            domain: clientDomain,
+            urlInventory,
+            directedEdgeCount
+          });
+        },
+        this.deps.llm
+      )
     );
     const graphMetrics = await this.deps.python.runGraphMetrics({
       directedLinkGraphPath: `${sessionPath}/phase1/directed_link_graph.json`,
@@ -7019,50 +7056,52 @@ var SiteAuditService = class {
     }
     if (currentBatch.length > 0) batches.push(currentBatch);
     const usZipsCsvPath = process.env["US_ZIPS_CSV_PATH"] ?? "/Users/vilovieta/Downloads/sales-magician-api-leads-magician-01c6cff78e31/tools/analytics/data/uszips.csv";
-    for (const batch of batches) {
-      let classified = [];
-      let unclassifiedUrls = batch.map((u) => u.url);
-      try {
-        const pythonResult = await this.deps.python.runLocationClassifier({
-          urls: batch.map((u) => u.url),
-          usZipsCsvPath
-        });
-        classified = pythonResult.classified;
-        unclassifiedUrls = pythonResult.unclassified;
-      } catch (err) {
-        console.warn(
-          "[site-audit-service] runClassifyPhase: python.runLocationClassifier failed (falling back to LLM):",
-          err instanceof Error ? err.message : String(err)
-        );
-        unclassifiedUrls = batch.map((u) => u.url);
-      }
-      for (const c of classified) {
-        allClassifications.push({
-          url: c.url,
-          page_type: c.pageType,
-          confidence: c.confidence > 0.6 ? "high" : c.confidence > 0.4 ? "medium" : "low",
-          _classification_label: "[COMPUTED:python/location-classifier]",
-          topic_cluster: null,
-          location_signals: c.locationSignals
-        });
-      }
-      if (unclassifiedUrls.length > 0) {
-        const unclassifiedBatch = batch.filter((u) => unclassifiedUrls.includes(u.url));
-        const llmResults = await llmParseWithRetry(
-          z9.array(PageTypeClassificationRowSchema),
-          (attempt, error) => {
-            void attempt;
-            void error;
-            return buildMetricsClassifyPrompt({
-              urls: unclassifiedBatch,
-              siteDomain: clientDomain
-            });
-          },
-          this.deps.llm
-        );
-        allClassifications.push(...llmResults);
+    await this.runLlmBilled(job.user_id, `${jobId}:phase3-classify`, async () => {
+      for (const batch of batches) {
+        let classified = [];
+        let unclassifiedUrls = batch.map((u) => u.url);
+        try {
+          const pythonResult = await this.deps.python.runLocationClassifier({
+            urls: batch.map((u) => u.url),
+            usZipsCsvPath
+          });
+          classified = pythonResult.classified;
+          unclassifiedUrls = pythonResult.unclassified;
+        } catch (err) {
+          console.warn(
+            "[site-audit-service] runClassifyPhase: python.runLocationClassifier failed (falling back to LLM):",
+            err instanceof Error ? err.message : String(err)
+          );
+          unclassifiedUrls = batch.map((u) => u.url);
+        }
+        for (const c of classified) {
+          allClassifications.push({
+            url: c.url,
+            page_type: c.pageType,
+            confidence: c.confidence > 0.6 ? "high" : c.confidence > 0.4 ? "medium" : "low",
+            _classification_label: "[COMPUTED:python/location-classifier]",
+            topic_cluster: null,
+            location_signals: c.locationSignals
+          });
+        }
+        if (unclassifiedUrls.length > 0) {
+          const unclassifiedBatch = batch.filter((u) => unclassifiedUrls.includes(u.url));
+          const llmResults = await llmParseWithRetry(
+            z9.array(PageTypeClassificationRowSchema),
+            (attempt, error) => {
+              void attempt;
+              void error;
+              return buildMetricsClassifyPrompt({
+                urls: unclassifiedBatch,
+                siteDomain: clientDomain
+              });
+            },
+            this.deps.llm
+          );
+          allClassifications.push(...llmResults);
+        }
       }
-    }
+    });
     void sessionPath;
     await this.deps.repo.logSiteAuditPhaseComplete(jobId, "phase3-classify", {
       classifications: allClassifications,
@@ -7075,20 +7114,24 @@ var SiteAuditService = class {
     if (!job) throw new Error(`[site-audit-service] runComparePhase: job ${jobId} not found`);
     const request = JSON.parse(job.request);
     const clientDomain = request.clientDomain;
-    const result = await llmParseWithRetry(
-      CompareRecommendOutputSchema,
-      (attempt, error) => {
-        void attempt;
-        void error;
-        return buildCompareRecommendPrompt({
-          clientDomain,
-          clientMoneyPages: [],
-          competitorSites: [],
-          clientUrlMetrics: [],
-          clientPageTypes: []
-        });
-      },
-      this.deps.llm
+    const result = await this.runLlmBilled(
+      job.user_id,
+      `${jobId}:phase4-compare`,
+      () => llmParseWithRetry(
+        CompareRecommendOutputSchema,
+        (attempt, error) => {
+          void attempt;
+          void error;
+          return buildCompareRecommendPrompt({
+            clientDomain,
+            clientMoneyPages: [],
+            competitorSites: [],
+            clientUrlMetrics: [],
+            clientPageTypes: []
+          });
+        },
+        this.deps.llm
+      )
     );
     await this.deps.repo.logSiteAuditPhaseComplete(jobId, "phase4-compare", result);
     return result;
@@ -7118,20 +7161,24 @@ var SiteAuditService = class {
       total: ilesTotal,
       components: ilesComponents
     };
-    const reportMarkdown = await llmParseWithRetry(
-      z9.string(),
-      (attempt, error) => {
-        void attempt;
-        void error;
-        return buildScoreSynthesizePrompt({
-          preComputedArchitectureScore,
-          preComputedIlesScore,
-          rankedBacklog: [],
-          clientDomain,
-          sessionPath
-        });
-      },
-      this.deps.llm
+    const reportMarkdown = await this.runLlmBilled(
+      job.user_id,
+      `${jobId}:phase5-synthesize`,
+      () => llmParseWithRetry(
+        z9.string(),
+        (attempt, error) => {
+          void attempt;
+          void error;
+          return buildScoreSynthesizePrompt({
+            preComputedArchitectureScore,
+            preComputedIlesScore,
+            rankedBacklog: [],
+            clientDomain,
+            sessionPath
+          });
+        },
+        this.deps.llm
+      )
     );
     const auditResult = SiteAuditResultSchema.parse({
       jobId,
@@ -8140,6 +8187,11 @@ internalMemoryApp.post("/ai-debit", async (c) => {
   }
   const user = await getUserByEmail(body.identity.trim().toLowerCase());
   if (!user) return c.json({ ok: false, reason: "no_user" }, 200);
+  if (body.costUsd > 0) {
+    await runWithCostContext({ ...currentCostContext(), op: "memory_ai", userId: user.id }, async () => {
+      void recordVendorUsage({ vendor: "mcp_memory_ai", model: null, units: body.costUsd, unitType: "usd", error: null });
+    });
+  }
   const mc = Math.round(body.costUsd * MEMORY_AI_MARGIN_MULTIPLE * MC_PER_USD);
   if (mc <= 0) return c.json({ ok: true, mc: 0, balance_mc: null }, 200);
   const { ok, balance_mc } = await debitMc(user.id, mc, LedgerOperation.MEMORY_AI, `${body.source ?? "ai"}:${body.op ?? ""}`);
@@ -8660,6 +8712,11 @@ async function fetchViaYoutubeTranscript(videoId) {
     }));
     const text = chunks.map((c) => c.text).join(" ");
     const last = chunks[chunks.length - 1];
+    const now = Date.now();
+    await runWithCostContext(
+      { ...currentCostContext(), op: "yt_transcription", subOp: "youtube-transcript" },
+      () => recordKernelSession({ kernelSessionId: null, source: "yt_caption_tier", headlessSent: null, openedAtMs: now, closedAtMs: now })
+    );
     return { videoId, text, chunks, durationMs: last ? last.timestamp[1] * 1e3 : 0, method: "youtube-transcript" };
   } catch {
     return null;
@@ -8711,163 +8768,221 @@ function parseTimedtextXml(xml) {
   }
   return results;
 }
-async function fetchViaKernelInnertube(videoId) {
-  const kernelApiKey = browserServiceApiKey();
-  if (!kernelApiKey) return null;
-  const kernelProxyId = browserServiceProxyId();
-  const driver = new BrowserDriver();
-  const start = Date.now();
-  try {
-    await driver.launch({ kernelApiKey, kernelProxyId, headless: true, viewport: { width: 1280, height: 800 }, locale: "en-US" });
-    const page = driver.getPage();
-    await driver.navigateTo(`https://www.youtube.com/watch?v=${videoId}`);
-    await page.waitForFunction(
-      () => !!window.ytcfg,
-      { timeout: 2e4 }
-    );
-    const xml = await page.evaluate(async (vid) => {
-      const ANDROID_VERSION = "20.10.38";
-      const resp = await fetch("/youtubei/v1/player?prettyPrint=false", {
-        method: "POST",
-        headers: {
-          "Content-Type": "application/json",
-          "User-Agent": `com.google.android.youtube/${ANDROID_VERSION} (Linux; U; Android 14)`
-        },
-        body: JSON.stringify({
-          context: { client: { clientName: "ANDROID", clientVersion: ANDROID_VERSION } },
-          videoId: vid
-        })
+var INNERTUBE_CLIENTS = [
+  { name: "ANDROID", version: "20.10.38", userAgent: "com.google.android.youtube/20.10.38 (Linux; U; Android 14)" },
+  { name: "IOS", version: "20.10.4", userAgent: "com.google.ios.youtube/20.10.4 (iPhone16,2; U; CPU iOS 18_3_2 like Mac OS X;)" },
+  { name: "ANDROID_VR", version: "1.62.27", userAgent: "com.google.android.apps.youtube.vr.oculus/1.62.27 (Linux; U; Android 12L) gzip" }
+];
+function isBotBlockStatus(status) {
+  return status === "LOGIN_REQUIRED" || status === "AGE_VERIFICATION_REQUIRED";
+}
+async function probePlayer(page, videoId) {
+  return page.evaluate(
+    async ({ vid, clients }) => {
+      let lastStatus = null;
+      for (const client2 of clients) {
+        const resp = await fetch("/youtubei/v1/player?prettyPrint=false", {
+          method: "POST",
+          headers: { "Content-Type": "application/json", "User-Agent": client2.userAgent },
+          body: JSON.stringify({
+            context: { client: { clientName: client2.name, clientVersion: client2.version } },
+            videoId: vid
+          })
+        }).catch(() => null);
+        if (!resp?.ok) {
+          lastStatus = resp ? `player_http_${resp.status}` : "player_fetch_failed";
+          continue;
+        }
+        const data = await resp.json();
+        lastStatus = data?.playabilityStatus?.status ?? null;
+        const audioFormats = (data?.streamingData?.adaptiveFormats ?? []).filter((f) => typeof f.url === "string" && f.url && typeof f.mimeType === "string" && f.mimeType.startsWith("audio/")).sort((a, b) => (Number(a.contentLength) || Infinity) - (Number(b.contentLength) || Infinity));
+        const pick = audioFormats[0] ?? null;
+        let captions = null;
+        const tracks = data?.captions?.playerCaptionsTracklistRenderer?.captionTracks ?? [];
+        if (tracks.length > 0) {
+          const track = tracks.find((t) => t.languageCode === "en") ?? tracks[0];
+          const sep = track.baseUrl.includes("?") ? "&" : "?";
+          const xmlResp = await fetch(`${track.baseUrl}${sep}fmt=json3`).catch(() => null);
+          if (xmlResp?.ok) captions = await xmlResp.text();
+        }
+        if (!captions && !pick) continue;
+        return {
+          captions,
+          status: lastStatus,
+          audio: pick ? {
+            url: pick.url,
+            mimeType: pick.mimeType,
+            bitrate: pick.bitrate ?? null,
+            contentLength: pick.contentLength ? Number(pick.contentLength) || null : null,
+            approxDurationMs: pick.approxDurationMs ? Number(pick.approxDurationMs) || null : null
+          } : null
+        };
+      }
+      return { captions: null, audio: null, status: lastStatus };
+    },
+    { vid: videoId, clients: INNERTUBE_CLIENTS }
+  ).catch(() => null);
+}
+var MAX_AUDIO_BYTES = 96 * 1024 * 1024;
+async function downloadWholeAudioStreamInOneRequestBecauseYouTubeAllowsOnlyOnePerIp(page, audio) {
+  if (audio.contentLength && audio.contentLength > MAX_AUDIO_BYTES) {
+    console.warn(`[CaptionFetcher] audio stream ${audio.contentLength} bytes exceeds ${MAX_AUDIO_BYTES} byte cap`);
+    return null;
+  }
+  const sep = audio.url.includes("?") ? "&" : "?";
+  const url = audio.contentLength ? `${audio.url}${sep}range=0-${audio.contentLength - 1}` : audio.url;
+  const result = await page.evaluate(async (u) => {
+    try {
+      const resp = await fetch(u);
+      if (!resp.ok) return { ok: false, error: `http_${resp.status}` };
+      const blob = await resp.blob();
+      const dataUrl = await new Promise((resolve, reject) => {
+        const reader = new FileReader();
+        reader.onload = () => resolve(reader.result);
+        reader.onerror = () => reject(reader.error);
+        reader.readAsDataURL(blob);
       });
-      if (!resp.ok) return null;
-      const data = await resp.json();
-      const tracks = data?.captions?.playerCaptionsTracklistRenderer?.captionTracks ?? [];
-      if (tracks.length === 0) return null;
-      const track = tracks.find((t) => t.languageCode === "en") ?? tracks[0];
-      const sep = track.baseUrl.includes("?") ? "&" : "?";
-      const xmlResp = await fetch(`${track.baseUrl}${sep}fmt=json3`);
-      return xmlResp.ok ? xmlResp.text() : null;
-    }, videoId);
-    if (!xml) return null;
-    let entries = parseTimedtextJson3(xml);
-    if (entries.length === 0) entries = parseTimedtextXml(xml);
-    if (entries.length === 0) return null;
-    const chunks = entries.map((e) => ({
-      timestamp: [e.start / 1e3, (e.start + e.dur) / 1e3],
-      text: e.text
-    }));
-    const text = chunks.map((c) => c.text).join(" ");
-    const last = chunks[chunks.length - 1];
-    return { videoId, text, chunks, durationMs: Date.now() - start, method: "browser-innertube" };
-  } catch {
+      return { ok: true, base64: dataUrl.slice(dataUrl.indexOf(",") + 1) };
+    } catch (e) {
+      return { ok: false, error: String(e) };
+    }
+  }, url).catch((e) => ({ ok: false, error: String(e) }));
+  if (!result.ok || !result.base64) {
+    console.warn(`[CaptionFetcher] audio fetch failed (${result.error ?? "unknown"}) for a ${audio.contentLength ?? 0} byte stream`);
     return null;
-  } finally {
-    await driver.close();
   }
+  const bytes = Buffer.from(result.base64, "base64");
+  if (audio.contentLength && bytes.length < audio.contentLength) {
+    console.warn(`[CaptionFetcher] partial audio: ${bytes.length} of ${audio.contentLength} bytes`);
+    return null;
+  }
+  return { bytes, mimeType: audio.mimeType.split(";")[0].trim() };
 }
-var WHISPER_RECORD_SECONDS = 90;
-async function captureAudioViaMediaRecorder(page) {
-  const result = await page.evaluate((recordSecs) => {
-    return new Promise((resolve) => {
-      const video = document.querySelector("video");
-      if (!video) {
-        resolve({ ok: false, error: "no video element" });
-        return;
-      }
-      video.muted = false;
-      video.volume = 1;
-      try {
-        const ctx = new AudioContext();
-        const src = ctx.createMediaElementSource(video);
-        const dest = ctx.createMediaStreamDestination();
-        src.connect(dest);
-        src.connect(ctx.destination);
-        const mimeType = MediaRecorder.isTypeSupported("audio/webm;codecs=opus") ? "audio/webm;codecs=opus" : MediaRecorder.isTypeSupported("audio/webm") ? "audio/webm" : "audio/ogg";
-        const recorder = new MediaRecorder(dest.stream, { mimeType, audioBitsPerSecond: 48e3 });
-        const chunks = [];
-        recorder.ondataavailable = (e) => {
-          if (e.data.size > 0) chunks.push(e.data);
-        };
-        recorder.onstop = async () => {
-          const blob = new Blob(chunks, { type: mimeType });
-          const ab = await blob.arrayBuffer();
-          const arr = new Uint8Array(ab);
-          let binary = "";
-          const chunkSize = 8192;
-          for (let i = 0; i < arr.length; i += chunkSize) {
-            binary += String.fromCharCode(...arr.slice(i, Math.min(i + chunkSize, arr.length)));
-          }
-          resolve({ ok: true, base64: btoa(binary), size: ab.byteLength, mimeType });
-        };
-        recorder.start(1e3);
-        video.playbackRate = 2;
-        video.play().catch(() => void 0);
-        setTimeout(() => {
-          recorder.stop();
-          ctx.close();
-        }, recordSecs * 1e3);
-      } catch (e) {
-        resolve({ ok: false, error: String(e) });
-      }
-    });
-  }, WHISPER_RECORD_SECONDS);
-  if (!result.ok || !result.base64 || !result.mimeType) return null;
-  return { bytes: Buffer.from(result.base64, "base64"), mimeType: result.mimeType };
-}
-async function attemptKernelWhisper(videoId, kernelApiKey, kernelProxyId, falKey, start) {
-  const driver = new BrowserDriver();
+async function transcribeWithWizper(videoId, audio, startMs) {
+  const falKey = process.env.FAL_KEY;
+  if (!falKey) return null;
   try {
-    await driver.launch({ kernelApiKey, kernelProxyId, headless: true, viewport: { width: 1280, height: 800 }, locale: "en-US" });
-    const page = driver.getPage();
-    await driver.navigateTo(`https://www.youtube.com/watch?v=${videoId}`);
-    await page.waitForFunction(
-      () => !!window.ytcfg,
-      { timeout: 2e4 }
-    );
-    const playStatus = await page.evaluate(() => {
-      const w = window;
-      return w.ytInitialPlayerResponse?.playabilityStatus?.status;
-    });
-    if (playStatus !== "OK") {
-      await driver.close();
-      return null;
-    }
-    const audioInfo = await captureAudioViaMediaRecorder(page);
-    await driver.close();
-    if (!audioInfo || audioInfo.bytes.length < 1e4) return null;
     fal.config({ credentials: falKey });
-    const ext = audioInfo.mimeType.includes("ogg") ? "ogg" : "webm";
-    const audioFile = new File([new Uint8Array(audioInfo.bytes)], `audio.${ext}`, { type: audioInfo.mimeType });
+    const ext = audio.mimeType.includes("webm") ? "webm" : audio.mimeType.includes("mp4") ? "m4a" : "audio";
+    const audioFile = new File([new Uint8Array(audio.bytes)], `audio.${ext}`, { type: audio.mimeType });
     const uploadedUrl = await fal.storage.upload(audioFile);
+    const subscribeStartMs = Date.now();
     const result = await fal.subscribe("fal-ai/wizper", {
-      input: { audio_url: uploadedUrl, task: "transcribe", language: "en" },
+      input: { audio_url: uploadedUrl, task: "transcribe", language: "en", chunk_level: "segment" },
       logs: false,
       pollInterval: 3e3
     });
+    const subscribeDurationMs = Date.now() - subscribeStartMs;
     const data = result.data;
     const chunks = (data.chunks ?? []).map((c) => ({
       timestamp: c.timestamp,
       text: c.text.trim()
     }));
-    const audioSec = chunks.length ? chunks[chunks.length - 1]?.timestamp?.[1] ?? 0 : 0;
-    void recordVendorUsage({ vendor: "fal_wizper", model: "fal-ai/wizper", units: audioSec / 60, unitType: "audio_min" });
+    void recordVendorUsage({ vendor: "fal_wizper", model: "fal-ai/wizper", units: subscribeDurationMs / 1e3, unitType: "compute_sec" });
     const text = data.text ?? chunks.map((c) => c.text).join(" ");
     if (!text) return null;
-    return { videoId, text, chunks, durationMs: Date.now() - start, method: "browser-whisper" };
-  } catch {
-    await driver.close();
+    return { videoId, text, chunks, durationMs: Date.now() - startMs, method: "browser-whisper" };
+  } catch (err) {
+    console.warn(`[CaptionFetcher] wizper transcription failed for ${videoId}: ${err instanceof Error ? err.message : String(err)}`);
     return null;
   }
 }
-async function fetchViaKernelWhisper(videoId) {
+var YT_RESIDENTIAL_LOCATION = process.env.YOUTUBE_PROXY_LOCATION ?? "Austin, Texas";
+async function resolveResidentialProxy(kernelApiKey) {
+  try {
+    const resolved = await resolveKernelProxyId({
+      kernelApiKey,
+      proxyMode: "location",
+      location: YT_RESIDENTIAL_LOCATION,
+      gl: "us",
+      fresh: true
+    });
+    return { kernelProxyId: resolved.kernelProxyId, disposableProxyId: resolved.disposableProxyId };
+  } catch (err) {
+    console.warn(`[CaptionFetcher] residential proxy resolution failed: ${err instanceof Error ? err.message : String(err)}`);
+    return {};
+  }
+}
+async function probeAndExtract(videoId, opts) {
   const kernelApiKey = browserServiceApiKey();
-  const kernelProxyId = browserServiceProxyId();
-  const falKey = process.env.FAL_KEY;
-  if (!kernelApiKey || !falKey) return null;
+  if (!kernelApiKey) return null;
+  const kernelProxyId = opts?.kernelProxyId ?? browserServiceProxyId();
+  const driver = new BrowserDriver();
+  return runWithCostContext({ ...currentCostContext(), op: "yt_transcription", subOp: "browser-innertube" }, async () => {
+    let entries = [];
+    let audio = null;
+    let retryWithFreshIp = false;
+    try {
+      await driver.launch({ kernelApiKey, kernelProxyId, headless: true, headlessMode: "headless", viewport: { width: 1280, height: 800 }, locale: "en-US" });
+      const page = driver.getPage();
+      await driver.navigateTo(`https://www.youtube.com/watch?v=${videoId}`);
+      await page.waitForFunction(
+        () => !!window.ytcfg,
+        { timeout: 2e4 }
+      );
+      const probe = await probePlayer(page, videoId);
+      if (!probe) return null;
+      if (probe.captions && !opts?.skipCaptions) {
+        entries = parseTimedtextJson3(probe.captions);
+        if (entries.length === 0) entries = parseTimedtextXml(probe.captions);
+      }
+      if (entries.length === 0) {
+        if (!probe.audio) {
+          retryWithFreshIp = isBotBlockStatus(probe.status);
+          console.warn(`[CaptionFetcher] no captions and no audio stream for ${videoId} (playability=${probe.status ?? "unknown"}, proxied=${Boolean(kernelProxyId)})`);
+          return { entries, audio: null, retryWithFreshIp };
+        }
+        audio = await downloadWholeAudioStreamInOneRequestBecauseYouTubeAllowsOnlyOnePerIp(page, probe.audio);
+        if (!audio) return { entries, audio: null, retryWithFreshIp: true };
+      }
+    } catch (err) {
+      console.warn(`[CaptionFetcher] kernel session failed for ${videoId}: ${err instanceof Error ? err.message : String(err)}`);
+      return null;
+    } finally {
+      const servedSubOp = audio ? "browser-whisper" : "browser-innertube";
+      await runWithCostContext({ ...currentCostContext(), op: "yt_transcription", subOp: servedSubOp }, () => driver.close());
+    }
+    return { entries, audio, retryWithFreshIp };
+  });
+}
+async function extractWithProxyEscalation(videoId, opts) {
+  const direct = await probeAndExtract(videoId, opts);
+  if (direct && (direct.entries.length > 0 || direct.audio)) return direct;
+  const shouldEscalate = direct === null || direct.retryWithFreshIp;
+  if (!shouldEscalate) return direct;
+  const kernelApiKey = browserServiceApiKey();
+  if (!kernelApiKey) return direct;
+  const { kernelProxyId, disposableProxyId } = await resolveResidentialProxy(kernelApiKey);
+  if (!kernelProxyId) return direct;
+  try {
+    console.warn(`[CaptionFetcher] ${videoId}: retrying through residential proxy after bot block`);
+    const viaProxy = await probeAndExtract(videoId, { ...opts, kernelProxyId });
+    return viaProxy ?? direct;
+  } finally {
+    if (disposableProxyId) {
+      await deleteKernelProxyId(kernelApiKey, disposableProxyId).catch(
+        (err) => console.warn(`[CaptionFetcher] failed to delete disposable proxy: ${err instanceof Error ? err.message : String(err)}`)
+      );
+    }
+  }
+}
+async function fetchViaKernelSession(videoId) {
   const start = Date.now();
-  for (let attempt = 0; attempt < 4; attempt++) {
-    if (attempt > 0) await new Promise((r) => setTimeout(r, 2e3 * attempt));
-    const result = await attemptKernelWhisper(videoId, kernelApiKey, kernelProxyId, falKey, start);
-    if (result) return result;
+  const outcome = await extractWithProxyEscalation(videoId);
+  if (!outcome) return null;
+  if (outcome.entries.length > 0) {
+    const chunks = outcome.entries.map((e) => ({
+      timestamp: [e.start / 1e3, (e.start + e.dur) / 1e3],
+      text: e.text
+    }));
+    const text = chunks.map((c) => c.text).join(" ");
+    return { videoId, text, chunks, durationMs: Date.now() - start, method: "browser-innertube" };
+  }
+  if (outcome.audio) {
+    return runWithCostContext(
+      { ...currentCostContext(), op: "yt_transcription", subOp: "browser-whisper" },
+      () => transcribeWithWizper(videoId, outcome.audio, start)
+    );
   }
   return null;
 }
@@ -8889,11 +9004,11 @@ function finalizeCaptions(result) {
 async function fetchCaptions(videoId) {
   const primary = await fetchViaYoutubeTranscript(videoId);
   if (primary) return finalizeCaptions(primary);
-  const kernel = await fetchViaKernelInnertube(videoId);
-  if (kernel) return finalizeCaptions(kernel);
-  const whisper = await fetchViaKernelWhisper(videoId);
-  if (whisper) return finalizeCaptions(whisper);
-  throw new Error(`No captions available for ${videoId} \u2014 all three tiers failed`);
+  const result = await fetchViaKernelSession(videoId);
+  if (result) return finalizeCaptions(result);
+  throw new Error(
+    `Could not transcribe ${videoId}. It has no caption track, and its audio stream could not be downloaded \u2014 YouTube permits a single media request per IP, so streams that do not fit in one response are unreachable. Captioned videos of any length still work.`
+  );
 }
 
 // src/api/api-auth.ts
@@ -9221,7 +9336,7 @@ youtubeApp.post("/transcribe", createApiKeyAuth(), async (c) => {
     const { ok: trOk, balance_mc: trBal } = await debitMc(user.id, holdMc, LedgerOperation.TRANSCRIPTION_HOLD, id);
     if (!trOk) return c.json(insufficientBalanceResponse(trBal, holdMc), 402);
     debited = true;
-    const result = await fetchCaptions(id);
+    const result = await runWithCostContext({ ...currentCostContext(), op: "yt_transcription", userId: user.id }, () => fetchCaptions(id));
     const lastChunk = result.chunks?.[result.chunks.length - 1];
     const chunkSecs = lastChunk && Number.isFinite(lastChunk.timestamp[1]) ? lastChunk.timestamp[1] : 0;
     const durationSecs = Number.isFinite(result.durationMs) ? result.durationMs / 1e3 : 0;
@@ -10284,8 +10399,7 @@ async function transcribeMediaUrl(mediaUrl, markdownTitle = "# Media Transcript"
   const text = data.text ?? "";
   const chunks = data.chunks ?? [];
   const durationMs = Date.now() - startMs;
-  const audioSec = chunks.length ? chunks[chunks.length - 1]?.timestamp?.[1] ?? 0 : 0;
-  void recordVendorUsage({ vendor: "fal_wizper", model: "fal-ai/wizper", units: audioSec / 60, unitType: "audio_min" });
+  void recordVendorUsage({ vendor: "fal_wizper", model: "fal-ai/wizper", units: durationMs / 1e3, unitType: "compute_sec" });
   return {
     text,
     chunks,
@@ -11760,6 +11874,7 @@ redditApp.post("/thread", createApiKeyAuth(), async (c) => {
   if (!gate.ok) return c.json(concurrencyLimitExceededResponse(gate), 429, { "Retry-After": String(gate.retryAfterSeconds) });
   let debited = false;
   let refunded = false;
+  let commentDebitMc = 0;
   try {
     const { ok, balance_mc } = await debitMc(user.id, MC_COSTS.reddit_thread, LedgerOperation.REDDIT_THREAD, body.url);
     if (!ok) return c.json(insufficientBalanceResponse(balance_mc, MC_COSTS.reddit_thread), 402);
@@ -12077,6 +12192,21 @@ videoApp.post("/status", createApiKeyAuth(), async (c) => {
       effectiveFrames: st.frameCount ?? null
     };
   }
+  if ((st.status === "done" || st.status === "failed") && typeof st.costUsd === "number" && st.costUsd > 0) {
+    const llmOp = `${LedgerOperation.VIDEO_ANALYSIS_LLM}:${runId}`;
+    if (!await ledgerExistsForOperation(user.id, llmOp)) {
+      await runWithCostContext({ ...currentCostContext(), op: "video_analysis", userId: user.id }, async () => {
+        void recordVendorUsage({ vendor: "mcp_memory_video", model: null, units: st.costUsd, unitType: "usd", error: null });
+      });
+      const surchargeMc = Math.round(st.costUsd * VIDEO_ANALYSIS_LLM_MARGIN_MULTIPLE * MC_PER_USD);
+      if (surchargeMc > 0) {
+        const { ok } = await debitMc(user.id, surchargeMc, llmOp, `real-cost-surcharge:${runId}`);
+        if (!ok) {
+          console.warn(`[video-routes] VIDEO_ANALYSIS_LLM surcharge debit failed for user ${user.id} (run ${runId}): insufficient balance for ${surchargeMc}mc`);
+        }
+      }
+    }
+  }
   return c.json({
     ok: true,
     runId,
@@ -13247,7 +13377,7 @@ trustpilotApp.post("/reviews", createApiKeyAuth(), async (c) => {
     let pagesFetched = 0;
     try {
       await driver.launch({
-        headless: true,
+        headless: false,
         kernelApiKey: browserServiceApiKey(),
         viewport: { width: 1280, height: 900 },
         locale: "en-US"
@@ -13440,7 +13570,7 @@ g2App.post("/reviews", createApiKeyAuth(), async (c) => {
       const driver = new BrowserDriver();
       try {
         await driver.launch({
-          headless: true,
+          headless: false,
           kernelApiKey: browserServiceApiKey(),
           kernelProxyId: proxyId,
           viewport: { width: 1280, height: 900 },
@@ -20279,9 +20409,12 @@ function opForCostPath(p) {
   }
   if (p.startsWith("/instagram/")) return "instagram";
   if (p === "/reddit/thread") return "reddit_thread";
+  if (p.startsWith("/trustpilot/")) return "trustpilot_reviews";
+  if (p.startsWith("/g2/")) return "g2_reviews";
   if (p.startsWith("/api/internal/site-architecture-auditor")) return "audit_site";
   if (p.startsWith("/api/internal/memory")) return "memory_ai";
   if (p.startsWith("/agent/")) return "browser_session";
+  if (p.startsWith("/video/")) return "video_analysis";
   return null;
 }
 app.use("*", (c, next) => {
@@ -20993,16 +21126,20 @@ app.post("/harvest/sync", auth2, async (c) => {
     AbortSignal.timeout(syncTimeoutMs)
   ]);
   try {
-    const result = await harvest({
-      ...options,
-      kernelApiKey: browserServiceApiKey(),
-      headless: true,
-      format: "json",
-      outputDir: "/tmp/paa-output-api",
-      signal: syncSignal,
-      softDeadlineMs: Date.now() + Math.floor(syncTimeoutMs * 0.8),
-      onAttemptEvent: recordAttempt
-    });
+    const harvestCtx = currentCostContext();
+    const result = await runWithCostContext(
+      { ...harvestCtx, op: options.serpOnly ? "serp" : "paa", userId: user.id },
+      () => harvest({
+        ...options,
+        kernelApiKey: browserServiceApiKey(),
+        headless: true,
+        format: "json",
+        outputDir: "/tmp/paa-output-api",
+        signal: syncSignal,
+        softDeadlineMs: Date.now() + Math.floor(syncTimeoutMs * 0.8),
+        onAttemptEvent: recordAttempt
+      })
+    );
     await completeJob(jobId, result);
     const attempts = await listHarvestAttempts(jobId, user.id);
     if (!options.serpOnly) {
@@ -21681,7 +21818,7 @@ app.get("/cron/tick", async (c) => {
   if (!process.env.CRON_SECRET || secret2 !== `Bearer ${process.env.CRON_SECRET}`) {
     return c.json({ error: "Unauthorized" }, 401);
   }
-  const { drainQueue } = await import("./worker-VASAJ7FZ.js");
+  const { drainQueue } = await import("./worker-HTYZAYGB.js");
   const budget = { maxJobs: 10, deadlineMs: Date.now() + 28e4 };
   const workflowDispatchResult = await dispatchDueWorkflowSchedules(`${new URL(c.req.url).protocol}//${new URL(c.req.url).host}`);
   const [results, sweepResult, reapResult, expiredResult, blobCleanup] = await Promise.all([
@@ -21699,7 +21836,7 @@ app.post("/api/internal/extract-refinalize/:id", async (c) => {
     return c.json({ error: "Unauthorized" }, 401);
   }
   const jobId = c.req.param("id");
-  const { getExtractJob: getExtractJob2, completeExtractJob: completeExtractJob2 } = await import("./site-extract-repository-LI2S3S6E.js");
+  const { getExtractJob: getExtractJob2, completeExtractJob: completeExtractJob2 } = await import("./site-extract-repository-HTIY52MW.js");
   const { assembleExtractArtifacts } = await import("./extract-bundle-COS56ZDO.js");
   const job = await getExtractJob2(jobId);
   if (!job) return c.json({ error: "job not found" }, 404);
@@ -21707,7 +21844,7 @@ app.post("/api/internal/extract-refinalize/:id", async (c) => {
   await completeExtractJob2(jobId, stored);
   let settlement = "already_settled_or_refunded";
   if (job.billedMc == null && job.userId != null) {
-    const { settleExtractJob: settleExtractJob2, countSuccessfulPages: countSuccessfulPages2 } = await import("./site-extract-repository-LI2S3S6E.js");
+    const { settleExtractJob: settleExtractJob2, countSuccessfulPages: countSuccessfulPages2 } = await import("./site-extract-repository-HTIY52MW.js");
     const heldMc = Number(job.options.heldMc ?? 0);
     const successful = await countSuccessfulPages2(jobId);
     const usedMc = Math.min(successful * MC_COSTS.page_scrape, heldMc);
@@ -21871,4 +22008,4 @@ app.get("/blog/:slug/", (c) => {
 export {
   app
 };
-//# sourceMappingURL=server-R3KVQNOT.js.map
+//# sourceMappingURL=server-UNID3SJU.js.map