mcp-scraper 0.4.6 → 0.4.12

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (42) hide show
  1. package/dist/bin/api-server.cjs +1532 -1351
  2. package/dist/bin/api-server.cjs.map +1 -1
  3. package/dist/bin/api-server.js +2 -2
  4. package/dist/bin/mcp-scraper-cli.cjs +1 -1
  5. package/dist/bin/mcp-scraper-cli.cjs.map +1 -1
  6. package/dist/bin/mcp-scraper-cli.js +1 -1
  7. package/dist/bin/mcp-scraper-install.cjs +1 -1
  8. package/dist/bin/mcp-scraper-install.cjs.map +1 -1
  9. package/dist/bin/mcp-scraper-install.js +1 -1
  10. package/dist/bin/mcp-stdio-server.cjs +1 -1
  11. package/dist/bin/mcp-stdio-server.cjs.map +1 -1
  12. package/dist/bin/mcp-stdio-server.js +2 -2
  13. package/dist/bin/paa-harvest.cjs +13 -4
  14. package/dist/bin/paa-harvest.cjs.map +1 -1
  15. package/dist/bin/paa-harvest.js +1 -1
  16. package/dist/{chunk-5GAD2AN2.js → chunk-2YY46QYT.js} +27 -10
  17. package/dist/chunk-2YY46QYT.js.map +1 -0
  18. package/dist/{chunk-IFXACD4K.js → chunk-6EXP6DQG.js} +2 -2
  19. package/dist/{chunk-IVQYNY45.js → chunk-EMY7ELRU.js} +23 -6
  20. package/dist/chunk-EMY7ELRU.js.map +1 -0
  21. package/dist/{chunk-NYAWN7CZ.js → chunk-ONIOF5XW.js} +2 -2
  22. package/dist/chunk-SHXJQQOH.js +7 -0
  23. package/dist/chunk-SHXJQQOH.js.map +1 -0
  24. package/dist/index.cjs +59 -4
  25. package/dist/index.cjs.map +1 -1
  26. package/dist/index.js +8 -2
  27. package/dist/index.js.map +1 -1
  28. package/dist/{server-R3KVQNOT.js → server-UNID3SJU.js} +409 -272
  29. package/dist/server-UNID3SJU.js.map +1 -0
  30. package/dist/{site-extract-repository-LI2S3S6E.js → site-extract-repository-HTIY52MW.js} +3 -3
  31. package/dist/{worker-VASAJ7FZ.js → worker-HTYZAYGB.js} +16 -12
  32. package/dist/worker-HTYZAYGB.js.map +1 -0
  33. package/package.json +1 -2
  34. package/dist/chunk-5BB6Y3BB.js +0 -7
  35. package/dist/chunk-5BB6Y3BB.js.map +0 -1
  36. package/dist/chunk-5GAD2AN2.js.map +0 -1
  37. package/dist/chunk-IVQYNY45.js.map +0 -1
  38. package/dist/server-R3KVQNOT.js.map +0 -1
  39. package/dist/worker-VASAJ7FZ.js.map +0 -1
  40. /package/dist/{chunk-IFXACD4K.js.map → chunk-6EXP6DQG.js.map} +0 -0
  41. /package/dist/{chunk-NYAWN7CZ.js.map → chunk-ONIOF5XW.js.map} +0 -0
  42. /package/dist/{site-extract-repository-LI2S3S6E.js.map → site-extract-repository-HTIY52MW.js.map} +0 -0
@@ -5341,18 +5341,25 @@ function kernelCostUsd(ms, headful) {
5341
5341
  return sec * (headful ? KERNEL_HEADFUL_USD_PER_SEC : KERNEL_HEADLESS_USD_PER_SEC);
5342
5342
  }
5343
5343
  function vendorCostUsd(vendor, units) {
5344
- if (vendor === "fal_wizper") return Math.max(0, units) * FAL_WIZPER_USD_PER_AUDIO_MIN;
5344
+ if (vendor === "fal_wizper") {
5345
+ const estimatedComputeSec = Math.max(0, units) / FAL_WIZPER_WALLCLOCK_TO_COMPUTE_SEC_FACTOR;
5346
+ return estimatedComputeSec * FAL_WIZPER_USD_PER_COMPUTE_SEC;
5347
+ }
5345
5348
  if (vendor === "deepinfra_qwen") return Math.max(0, units) / 1e3 * DEEPINFRA_QWEN_USD_PER_1K_OUT_TOKENS;
5349
+ if (vendor === "openrouter") return Math.max(0, units);
5350
+ if (vendor === "mcp_memory_video") return Math.max(0, units);
5351
+ if (vendor === "mcp_memory_ai") return Math.max(0, units);
5346
5352
  return 0;
5347
5353
  }
5348
- var KERNEL_HEADLESS_USD_PER_SEC, KERNEL_HEADFUL_USD_PER_SEC, HEADLESS_OPS, FAL_WIZPER_USD_PER_AUDIO_MIN, DEEPINFRA_QWEN_USD_PER_1K_OUT_TOKENS;
5354
+ var KERNEL_HEADLESS_USD_PER_SEC, KERNEL_HEADFUL_USD_PER_SEC, HEADLESS_OPS, FAL_WIZPER_USD_PER_COMPUTE_SEC, FAL_WIZPER_WALLCLOCK_TO_COMPUTE_SEC_FACTOR, DEEPINFRA_QWEN_USD_PER_1K_OUT_TOKENS;
5349
5355
  var init_cost_rates = __esm({
5350
5356
  "src/api/cost-rates.ts"() {
5351
5357
  "use strict";
5352
5358
  KERNEL_HEADLESS_USD_PER_SEC = 166667e-10;
5353
5359
  KERNEL_HEADFUL_USD_PER_SEC = 1333336e-10;
5354
- HEADLESS_OPS = /* @__PURE__ */ new Set(["harvest", "maps_search", "yt_channel", "fb_search", "fb_ad", "instagram"]);
5355
- FAL_WIZPER_USD_PER_AUDIO_MIN = 5e-3;
5360
+ HEADLESS_OPS = /* @__PURE__ */ new Set(["serp", "maps_search", "yt_channel", "yt_transcription", "fb_search", "fb_ad", "instagram"]);
5361
+ FAL_WIZPER_USD_PER_COMPUTE_SEC = 111e-5;
5362
+ FAL_WIZPER_WALLCLOCK_TO_COMPUTE_SEC_FACTOR = 4;
5356
5363
  DEEPINFRA_QWEN_USD_PER_1K_OUT_TOKENS = 4e-4;
5357
5364
  }
5358
5365
  });
@@ -5393,6 +5400,10 @@ async function migrateCostTelemetry() {
5393
5400
  await db.execute(`ALTER TABLE kernel_session_log ADD COLUMN proxy_type TEXT`);
5394
5401
  } catch {
5395
5402
  }
5403
+ try {
5404
+ await db.execute(`ALTER TABLE kernel_session_log ADD COLUMN method TEXT`);
5405
+ } catch {
5406
+ }
5396
5407
  await db.execute(`
5397
5408
  CREATE TABLE IF NOT EXISTS vendor_usage_log (
5398
5409
  id TEXT PRIMARY KEY,
@@ -5411,6 +5422,10 @@ async function migrateCostTelemetry() {
5411
5422
  await db.execute(`CREATE INDEX IF NOT EXISTS vendor_usage_log_op ON vendor_usage_log(op)`);
5412
5423
  await db.execute(`CREATE INDEX IF NOT EXISTS vendor_usage_log_probe ON vendor_usage_log(probe_run_id)`);
5413
5424
  await db.execute(`CREATE INDEX IF NOT EXISTS vendor_usage_log_created ON vendor_usage_log(created_at)`);
5425
+ try {
5426
+ await db.execute(`ALTER TABLE vendor_usage_log ADD COLUMN method TEXT`);
5427
+ } catch {
5428
+ }
5414
5429
  await db.execute(`
5415
5430
  CREATE TABLE IF NOT EXISTS cost_probe_runs (
5416
5431
  id TEXT PRIMARY KEY,
@@ -5457,8 +5472,8 @@ async function recordKernelSession(r) {
5457
5472
  const db = getDb();
5458
5473
  await db.execute({
5459
5474
  sql: `INSERT INTO kernel_session_log
5460
- (id, kernel_session_id, op, source, probe_run_id, user_id, stealth, headless_sent, proxy_used, proxy_source, proxy_type, fallback, opened_at, closed_at, duration_ms, est_cost_usd_headless, est_cost_usd_headful, error)
5461
- VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)`,
5475
+ (id, kernel_session_id, op, source, probe_run_id, user_id, stealth, headless_sent, proxy_used, proxy_source, proxy_type, fallback, opened_at, closed_at, duration_ms, est_cost_usd_headless, est_cost_usd_headful, error, method)
5476
+ VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)`,
5462
5477
  args: [
5463
5478
  (0, import_node_crypto2.randomUUID)(),
5464
5479
  r.kernelSessionId ?? null,
@@ -5477,7 +5492,8 @@ async function recordKernelSession(r) {
5477
5492
  durationMs,
5478
5493
  kernelCostUsd(durationMs, false),
5479
5494
  kernelCostUsd(durationMs, true),
5480
- r.error ?? null
5495
+ r.error ?? null,
5496
+ ctx?.subOp ?? null
5481
5497
  ]
5482
5498
  });
5483
5499
  } catch (err) {
@@ -5491,8 +5507,8 @@ async function recordVendorUsage(r) {
5491
5507
  const db = getDb();
5492
5508
  await db.execute({
5493
5509
  sql: `INSERT INTO vendor_usage_log
5494
- (id, op, probe_run_id, user_id, vendor, model, units, unit_type, est_cost_usd, error)
5495
- VALUES (?,?,?,?,?,?,?,?,?,?)`,
5510
+ (id, op, probe_run_id, user_id, vendor, model, units, unit_type, est_cost_usd, error, method)
5511
+ VALUES (?,?,?,?,?,?,?,?,?,?,?)`,
5496
5512
  args: [
5497
5513
  (0, import_node_crypto2.randomUUID)(),
5498
5514
  ctx?.op ?? null,
@@ -5503,7 +5519,8 @@ async function recordVendorUsage(r) {
5503
5519
  r.units,
5504
5520
  r.unitType,
5505
5521
  vendorCostUsd(r.vendor, r.units),
5506
- r.error ?? null
5522
+ r.error ?? null,
5523
+ ctx?.subOp ?? null
5507
5524
  ]
5508
5525
  });
5509
5526
  } catch (err) {
@@ -7405,6 +7422,7 @@ var init_llm_parse_with_retry = __esm({
7405
7422
  DeepInfraLlmClient = class {
7406
7423
  apiKey;
7407
7424
  model;
7425
+ totalCostUsd = 0;
7408
7426
  constructor(apiKey, model = "Qwen/Qwen3.6-35B-A3B") {
7409
7427
  this.apiKey = apiKey;
7410
7428
  this.model = model;
@@ -7437,6 +7455,7 @@ ${raw}`
7437
7455
  OpenRouterLlmClient = class {
7438
7456
  apiKey;
7439
7457
  model;
7458
+ totalCostUsd = 0;
7440
7459
  constructor(apiKey, model = "qwen/qwen3-235b-a22b") {
7441
7460
  this.apiKey = apiKey;
7442
7461
  this.model = model;
@@ -7449,6 +7468,9 @@ ${raw}`
7449
7468
  });
7450
7469
  if (!resp.ok) throw new Error(`OpenRouter error ${resp.status}: ${await resp.text()}`);
7451
7470
  const data = await resp.json();
7471
+ const cost = data.usage?.cost ?? 0;
7472
+ this.totalCostUsd += cost;
7473
+ void recordVendorUsage({ vendor: "openrouter", model: this.model, units: cost, unitType: "usd" });
7452
7474
  return data.choices[0].message.content;
7453
7475
  }
7454
7476
  async completeJson(prompt) {
@@ -7474,6 +7496,9 @@ ${raw}`
7474
7496
  secondary;
7475
7497
  lastServedBy = null;
7476
7498
  lastPrimaryError = null;
7499
+ get totalCostUsd() {
7500
+ return this.primary.totalCostUsd + this.secondary.totalCostUsd;
7501
+ }
7477
7502
  async complete(prompt) {
7478
7503
  try {
7479
7504
  const result = await this.primary.complete(prompt);
@@ -7928,6 +7953,390 @@ var init_schemas = __esm({
7928
7953
  }
7929
7954
  });
7930
7955
 
7956
+ // src/api/rates.ts
7957
+ function browserActiveCostMc(activeMs) {
7958
+ return Math.round(activeMs * MC_PER_BROWSER_MS);
7959
+ }
7960
+ function concurrencySlotBillingInfo() {
7961
+ const billingUrl = process.env.CONCURRENCY_BILLING_URL ?? "https://mcpscraper.dev/billing";
7962
+ return {
7963
+ product: "Extra concurrency slot",
7964
+ price_label: CONCURRENCY_SLOT_PRICE_LABEL,
7965
+ unit_amount_usd: CONCURRENCY_SLOT_PRICE_USD,
7966
+ currency: CONCURRENCY_SLOT_PRICE_CURRENCY,
7967
+ interval: CONCURRENCY_SLOT_PRICE_INTERVAL,
7968
+ billing_url: billingUrl,
7969
+ terminal_command: CONCURRENCY_SLOT_TERMINAL_COMMAND,
7970
+ terminal_command_with_api_key_env: `MCP_SCRAPER_API_KEY=sk_live_your_key ${CONCURRENCY_SLOT_TERMINAL_COMMAND}`
7971
+ };
7972
+ }
7973
+ function mcToCredits(mc) {
7974
+ return mc / MC_PER_CREDIT;
7975
+ }
7976
+ function insufficientBalanceResponse(balanceMc, requiredMc) {
7977
+ const topupUrl = process.env.TOPUP_URL ?? "https://mcpscraper.dev/billing";
7978
+ const balanceCredits = mcToCredits(balanceMc);
7979
+ const requiredCredits = mcToCredits(requiredMc);
7980
+ return {
7981
+ error: "insufficient_balance",
7982
+ error_code: "insufficient_balance",
7983
+ message: `Insufficient credits. Balance: ${balanceCredits} credits. This call requires ${requiredCredits} credits. Top up at ${topupUrl}`,
7984
+ balance_credits: balanceCredits,
7985
+ required_credits: requiredCredits,
7986
+ topup_url: topupUrl
7987
+ };
7988
+ }
7989
+ var MC_COSTS, MC_PER_BROWSER_MS, BROWSER_OPEN_MIN_BALANCE_MC, MC_PER_CREDIT, CREDIT_COST_CATALOG, CONCURRENCY_PRICE_ID, SUBSCRIPTION_TIERS, SUBSCRIPTION_TIER_BY_KEY, CONCURRENCY_SLOT_PRICE_USD, CONCURRENCY_SLOT_PRICE_CURRENCY, CONCURRENCY_SLOT_PRICE_INTERVAL, CONCURRENCY_SLOT_PRICE_LABEL, CONCURRENCY_SLOT_TERMINAL_COMMAND, FREE_SIGNUP_MC, MEMORY_FREE_QUOTA_BYTES, MEMORY_PLANS, MEMORY_PLAN_QUOTA, MEMORY_MARGIN_MULTIPLE, MEMORY_PRO_COST_BUDGET_USD, SCHEDULING_PLANS, LedgerOperation, MEMORY_AI_MARGIN_MULTIPLE, SITE_AUDIT_LLM_MARGIN_MULTIPLE, VIDEO_ANALYSIS_LLM_MARGIN_MULTIPLE, MC_PER_USD;
7990
+ var init_rates = __esm({
7991
+ "src/api/rates.ts"() {
7992
+ "use strict";
7993
+ MC_COSTS = {
7994
+ serp: 100,
7995
+ paa: 200,
7996
+ paa_base: 1e3,
7997
+ page_scrape: 100,
7998
+ url_map: 500,
7999
+ yt_channel: 200,
8000
+ yt_transcription: 200,
8001
+ fb_ad: 200,
8002
+ maps_search: 500,
8003
+ maps_place: 6e3,
8004
+ maps_review: 100,
8005
+ fb_search: 600,
8006
+ fb_transcribe: 200,
8007
+ google_ads_search: 600,
8008
+ google_ads_intel: 200,
8009
+ google_ads_transcribe: 200,
8010
+ instagram_profile: 400,
8011
+ instagram_media: 400,
8012
+ instagram_transcribe: 50,
8013
+ browser_minute: 12e3,
8014
+ reddit_thread: 3e3,
8015
+ reddit_comment: 200,
8016
+ video_analysis: 666667,
8017
+ trustpilot_reviews: 500,
8018
+ trustpilot_review: 100,
8019
+ g2_reviews: 500,
8020
+ g2_review: 150,
8021
+ diff_page: 100
8022
+ };
8023
+ MC_PER_BROWSER_MS = MC_COSTS.browser_minute / 6e4;
8024
+ BROWSER_OPEN_MIN_BALANCE_MC = 1e3;
8025
+ MC_PER_CREDIT = 1e3;
8026
+ CREDIT_COST_CATALOG = [
8027
+ {
8028
+ key: "serp",
8029
+ label: "SERP search",
8030
+ aliases: ["search_serp", "serp", "google search", "organic results"],
8031
+ credits: mcToCredits(MC_COSTS.serp),
8032
+ unit: "per search",
8033
+ notes: "Returns AI Overview, PAA snippet, videos, forums, and local pack."
8034
+ },
8035
+ {
8036
+ key: "paa",
8037
+ label: "PAA harvest",
8038
+ aliases: ["harvest_paa", "paa", "people also ask", "questions"],
8039
+ credits: mcToCredits(MC_COSTS.paa),
8040
+ unit: `per question (+${mcToCredits(MC_COSTS.paa_base)} credit base)`,
8041
+ notes: `Full SERP feature extraction. Billed ${mcToCredits(MC_COSTS.paa_base)} credit base + ${mcToCredits(MC_COSTS.paa)} per question actually returned (unused estimate refunded).`
8042
+ },
8043
+ {
8044
+ key: "page_scrape",
8045
+ label: "Page crawl / extract",
8046
+ aliases: ["extract_url", "extract_site", "page scrape", "url scrape", "single page", "site crawl"],
8047
+ credits: mcToCredits(MC_COSTS.page_scrape),
8048
+ unit: "per page",
8049
+ notes: "Applies to both single-URL extraction and per-page site crawls."
8050
+ },
8051
+ {
8052
+ key: "url_map",
8053
+ label: "Site URL mapping",
8054
+ aliases: ["map_site_urls", "url map", "site map", "crawl urls"],
8055
+ credits: mcToCredits(MC_COSTS.url_map),
8056
+ unit: "per mapping operation",
8057
+ notes: "Flat rate for the full /map-urls call regardless of URL count discovered."
8058
+ },
8059
+ {
8060
+ key: "yt_channel",
8061
+ label: "YouTube search / channel harvest",
8062
+ aliases: ["youtube_harvest", "youtube search", "youtube channel", "yt_channel"],
8063
+ credits: mcToCredits(MC_COSTS.yt_channel),
8064
+ unit: "per call"
8065
+ },
8066
+ {
8067
+ key: "yt_transcription",
8068
+ label: "YouTube transcription",
8069
+ aliases: ["youtube_transcribe", "youtube transcript", "transcription", "yt_transcription"],
8070
+ credits: mcToCredits(MC_COSTS.yt_transcription),
8071
+ unit: "per minute",
8072
+ notes: "A 5-minute hold is taken, then reconciled to actual video duration."
8073
+ },
8074
+ {
8075
+ key: "fb_ad",
8076
+ label: "Facebook search / ad lookup",
8077
+ aliases: ["facebook_page_intel", "facebook_ad_search", "facebook_ad", "facebook ads", "fb ads"],
8078
+ credits: mcToCredits(MC_COSTS.fb_ad),
8079
+ unit: "per call"
8080
+ },
8081
+ {
8082
+ key: "maps_search",
8083
+ label: "Maps business search",
8084
+ aliases: ["maps_search", "google maps search", "gmb search", "gbp search", "business profiles"],
8085
+ credits: mcToCredits(MC_COSTS.maps_search),
8086
+ unit: "per search",
8087
+ notes: "Returns up to 50 Google Maps business/profile candidates. Use maps_place_intel to hydrate selected businesses."
8088
+ },
8089
+ {
8090
+ key: "maps_place",
8091
+ label: "Maps business lookup",
8092
+ aliases: ["maps_place_intel", "google maps", "maps place", "place intel"],
8093
+ credits: mcToCredits(MC_COSTS.maps_place),
8094
+ unit: "per business",
8095
+ notes: "Base lookup. Reviews billed separately per card at maps_review rate."
8096
+ },
8097
+ {
8098
+ key: "maps_review",
8099
+ label: "Maps review",
8100
+ aliases: ["maps_reviews", "google reviews", "review cards", "reviews"],
8101
+ credits: mcToCredits(MC_COSTS.maps_review),
8102
+ unit: "per review card",
8103
+ notes: "Charged after extraction when includeReviews is true."
8104
+ },
8105
+ {
8106
+ key: "fb_search",
8107
+ label: "Facebook ad library search",
8108
+ aliases: ["facebook_search", "fb_search", "fb ad search"],
8109
+ credits: mcToCredits(MC_COSTS.fb_search),
8110
+ unit: "per search",
8111
+ notes: "Browser automation to search Facebook Ads Library by keyword."
8112
+ },
8113
+ {
8114
+ key: "google_ads_search",
8115
+ label: "Google Ads Transparency search",
8116
+ aliases: ["google_ads_search", "google ads search", "ads transparency search"],
8117
+ credits: mcToCredits(MC_COSTS.google_ads_search),
8118
+ unit: "per search",
8119
+ notes: "Browser automation to find advertisers in Google Ads Transparency Center by domain or name."
8120
+ },
8121
+ {
8122
+ key: "google_ads_intel",
8123
+ label: "Google Ads Transparency advertiser intel",
8124
+ aliases: ["google_ads_page_intel", "google ads intel", "ads transparency intel"],
8125
+ credits: mcToCredits(MC_COSTS.google_ads_intel),
8126
+ unit: "per call",
8127
+ notes: "Lists and hydrates an advertiser's creatives with image URLs and video references."
8128
+ },
8129
+ {
8130
+ key: "google_ads_transcribe",
8131
+ label: "Google ad video transcription",
8132
+ aliases: ["google_ads_transcribe", "google ad transcript"],
8133
+ credits: mcToCredits(MC_COSTS.google_ads_transcribe),
8134
+ unit: "per minute",
8135
+ notes: "A hold is taken, then reconciled to actual video duration."
8136
+ },
8137
+ {
8138
+ key: "fb_transcribe",
8139
+ label: "Facebook video / ad transcription",
8140
+ aliases: ["facebook_transcribe", "facebook_video_transcribe", "fb_transcribe", "fb ad transcript"],
8141
+ credits: mcToCredits(MC_COSTS.fb_transcribe),
8142
+ unit: "per minute",
8143
+ notes: "A hold is taken, then reconciled to actual video duration."
8144
+ },
8145
+ {
8146
+ key: "instagram_profile",
8147
+ label: "Instagram profile content discovery",
8148
+ aliases: ["instagram_profile_content", "instagram profile", "instagram content list", "ig profile"],
8149
+ credits: mcToCredits(MC_COSTS.instagram_profile),
8150
+ unit: "per profile scan",
8151
+ notes: "Browser extraction of public Instagram profile grid links. Complete history may require a logged-in profile."
8152
+ },
8153
+ {
8154
+ key: "instagram_media",
8155
+ label: "Instagram media download",
8156
+ aliases: ["instagram_media_download", "instagram reel download", "instagram post download", "ig media"],
8157
+ credits: mcToCredits(MC_COSTS.instagram_media),
8158
+ unit: "per post or reel",
8159
+ notes: "Extracts post text, image URLs, and reel audio/video tracks, with local downloads when the server can write files."
8160
+ },
8161
+ {
8162
+ key: "instagram_transcribe",
8163
+ label: "Instagram media transcription",
8164
+ aliases: ["instagram transcript", "instagram reel transcript", "ig transcribe"],
8165
+ credits: mcToCredits(MC_COSTS.instagram_transcribe),
8166
+ unit: "per call",
8167
+ notes: "Whisper transcription of selected Instagram audio/video media via fal.ai."
8168
+ },
8169
+ {
8170
+ key: "browser_minute",
8171
+ label: "Interactive browser session",
8172
+ aliases: ["browser_open", "browser agent", "browser_agent", "live browser", "browse", "browser control", "interactive browser"],
8173
+ credits: mcToCredits(MC_COSTS.browser_minute),
8174
+ unit: "per minute of use",
8175
+ notes: "Metered per minute of use for the whole time the browser session is open. Close the session to stop the meter; abandoned sessions are auto-closed after a short idle window."
8176
+ },
8177
+ {
8178
+ key: "reddit_thread",
8179
+ label: "Reddit thread base lookup",
8180
+ aliases: ["reddit", "reddit_thread", "reddit comments", "subreddit", "reddit post"],
8181
+ credits: mcToCredits(MC_COSTS.reddit_thread),
8182
+ unit: "per thread",
8183
+ notes: "Base lookup for the post itself. Comments billed separately per comment at reddit_comment rate. Refunded if the thread cannot be retrieved."
8184
+ },
8185
+ {
8186
+ key: "reddit_comment",
8187
+ label: "Reddit comment",
8188
+ aliases: ["reddit_comment", "reddit comments"],
8189
+ credits: mcToCredits(MC_COSTS.reddit_comment),
8190
+ unit: "per comment",
8191
+ notes: "Charged per comment actually extracted, billed down automatically if the balance runs out mid-thread."
8192
+ },
8193
+ {
8194
+ key: "video_analysis",
8195
+ label: "Video breakdown (frame-by-frame + transcript)",
8196
+ aliases: ["video_frame_analysis", "video breakdown", "video analysis", "analyze video"],
8197
+ credits: mcToCredits(MC_COSTS.video_analysis),
8198
+ unit: "per 120 frames (max 480)",
8199
+ notes: "Full multi-lens video breakdown: samples frames across a video (up to 30 minutes), analyzes each with vision AI, transcribes the audio, then produces summary, pacing/energy, words-per-minute, topic outline, key points, hook analysis, visual style, and a how-to-replicate recipe with a quality-control pass. $1 per 120 frames requested (max 480 = $4); billed down automatically if the video cannot use the requested frames, and refunded fully if the run fails."
8200
+ },
8201
+ {
8202
+ key: "trustpilot_reviews",
8203
+ label: "Trustpilot review harvest",
8204
+ aliases: ["trustpilot_reviews", "trustpilot", "trustpilot reviews"],
8205
+ credits: mcToCredits(MC_COSTS.trustpilot_reviews),
8206
+ unit: "per call",
8207
+ notes: "Base lookup. Reviews billed separately per card at trustpilot_review rate. Refunded if no reviews are found."
8208
+ },
8209
+ {
8210
+ key: "trustpilot_review",
8211
+ label: "Trustpilot review card",
8212
+ aliases: ["trustpilot_review", "trustpilot card"],
8213
+ credits: mcToCredits(MC_COSTS.trustpilot_review),
8214
+ unit: "per review card",
8215
+ notes: "Charged per review actually extracted, billed down automatically if the balance runs out mid-page."
8216
+ },
8217
+ {
8218
+ key: "g2_reviews",
8219
+ label: "G2 review harvest",
8220
+ aliases: ["g2_reviews", "g2", "g2 reviews"],
8221
+ credits: mcToCredits(MC_COSTS.g2_reviews),
8222
+ unit: "per call",
8223
+ notes: "Base lookup. Reviews billed separately per card at g2_review rate. Refunded if no reviews are found."
8224
+ },
8225
+ {
8226
+ key: "g2_review",
8227
+ label: "G2 review card",
8228
+ aliases: ["g2_review", "g2 card"],
8229
+ credits: mcToCredits(MC_COSTS.g2_review),
8230
+ unit: "per review card",
8231
+ notes: "Charged per review actually extracted (each card carries up to 3 Q&A sections), billed down automatically if the balance runs out mid-page."
8232
+ },
8233
+ {
8234
+ key: "diff_page",
8235
+ label: "Page change check",
8236
+ aliases: ["diff_page", "page diff", "change detection"],
8237
+ credits: mcToCredits(MC_COSTS.diff_page),
8238
+ unit: "per check",
8239
+ notes: "Same cost as a single page extract \u2014 one scrape is performed per check, then compared against your last stored snapshot for that URL."
8240
+ }
8241
+ ];
8242
+ CONCURRENCY_PRICE_ID = "price_1Ta1NRS8aAcsk3TGwsRnYbix";
8243
+ SUBSCRIPTION_TIERS = {
8244
+ "price_1TmiHRS8aAcsk3TGwmSNfNIa": { tier: "starter", label: "Starter", price_id: "price_1TmiHRS8aAcsk3TGwmSNfNIa", monthly_usd: 12, credits_mc: 8e6, concurrency: 3, intro_coupon: "mcp-starter-1dollar-intro-12" },
8245
+ "price_1Tmg1nS8aAcsk3TGe8zcnGTM": { tier: "growth", label: "Growth", price_id: "price_1Tmg1nS8aAcsk3TGe8zcnGTM", monthly_usd: 100, credits_mc: 8e7, concurrency: 10, intro_coupon: null },
8246
+ "price_1Tmg1nS8aAcsk3TGsZw34iXS": { tier: "scale", label: "Scale", price_id: "price_1Tmg1nS8aAcsk3TGsZw34iXS", monthly_usd: 250, credits_mc: 24e7, concurrency: 20, intro_coupon: null }
8247
+ };
8248
+ SUBSCRIPTION_TIER_BY_KEY = Object.fromEntries(
8249
+ Object.values(SUBSCRIPTION_TIERS).map((t) => [t.tier, t])
8250
+ );
8251
+ CONCURRENCY_SLOT_PRICE_USD = 5;
8252
+ CONCURRENCY_SLOT_PRICE_CURRENCY = "usd";
8253
+ CONCURRENCY_SLOT_PRICE_INTERVAL = "month";
8254
+ CONCURRENCY_SLOT_PRICE_LABEL = "$5/month";
8255
+ CONCURRENCY_SLOT_TERMINAL_COMMAND = "npx -y -p mcp-scraper@latest mcp-scraper-cli billing concurrency checkout";
8256
+ FREE_SIGNUP_MC = 1e5;
8257
+ MEMORY_FREE_QUOTA_BYTES = 1e7 * 1;
8258
+ MEMORY_PLANS = {
8259
+ "price_1TnMSTS8aAcsk3TGBgiwuvqL": { plan: "pro", label: "Pro", price_id: "price_1TnMSTS8aAcsk3TGBgiwuvqL", interval: "month", monthly_usd: 19, quota_bytes: 5e9 },
8260
+ "price_1TnMSTS8aAcsk3TGWBVU2agY": { plan: "pro", label: "Pro", price_id: "price_1TnMSTS8aAcsk3TGWBVU2agY", interval: "year", monthly_usd: 19, quota_bytes: 5e9 }
8261
+ };
8262
+ MEMORY_PLAN_QUOTA = { free: MEMORY_FREE_QUOTA_BYTES, pro: 5e9, team: 5e10 };
8263
+ MEMORY_MARGIN_MULTIPLE = 3;
8264
+ MEMORY_PRO_COST_BUDGET_USD = 19 / MEMORY_MARGIN_MULTIPLE;
8265
+ SCHEDULING_PLANS = {
8266
+ "price_1ToXcHS8aAcsk3TGf3cW7zHx": { price_id: "price_1ToXcHS8aAcsk3TGf3cW7zHx", label: "Scheduled Actions", interval: "month", monthly_usd: 10, quota_per_period: 1e3 }
8267
+ };
8268
+ LedgerOperation = {
8269
+ TOPUP: "topup",
8270
+ SUBSCRIPTION: "subscription",
8271
+ SIGNUP_GRANT: "signup_grant",
8272
+ MONTHLY_REFRESH: "monthly_free_refresh",
8273
+ PAA: "paa",
8274
+ PAA_REFUND: "paa_refund",
8275
+ SERP: "serp",
8276
+ REFUND: "refund",
8277
+ TRANSCRIPTION: "transcription",
8278
+ TRANSCRIPTION_HOLD: "transcription_hold",
8279
+ TRANSCRIPTION_REFUND: "transcription_refund",
8280
+ YT_CHANNEL: "yt_channel",
8281
+ FB_AD: "fb_ad",
8282
+ MAPS_SEARCH: "maps_search",
8283
+ MAPS_PLACE: "maps_place",
8284
+ MAPS_REVIEW: "maps_review",
8285
+ MAPS_REVIEW_REFUND: "maps_review_refund",
8286
+ EXTRACT_SITE: "extract_site",
8287
+ EXTRACT_SITE_REFUND: "extract_site_refund",
8288
+ EXTRACT_URL: "page_scrape",
8289
+ URL_MAP: "url_map",
8290
+ EXTRACT_SITE_HOLD: "extract_site_hold",
8291
+ YT_CHANNEL_REFUND: "yt_channel_refund",
8292
+ FB_AD_REFUND: "fb_ad_refund",
8293
+ URL_MAP_REFUND: "url_map_refund",
8294
+ FB_SEARCH: "fb_search",
8295
+ FB_TRANSCRIBE: "fb_transcribe",
8296
+ FB_SEARCH_REFUND: "fb_search_refund",
8297
+ FB_TRANSCRIBE_REFUND: "fb_transcribe_refund",
8298
+ GOOGLE_ADS_SEARCH: "google_ads_search",
8299
+ GOOGLE_ADS_SEARCH_REFUND: "google_ads_search_refund",
8300
+ GOOGLE_ADS_INTEL: "google_ads_intel",
8301
+ GOOGLE_ADS_INTEL_REFUND: "google_ads_intel_refund",
8302
+ GOOGLE_ADS_TRANSCRIBE: "google_ads_transcribe",
8303
+ GOOGLE_ADS_TRANSCRIBE_REFUND: "google_ads_transcribe_refund",
8304
+ INSTAGRAM_PROFILE: "instagram_profile",
8305
+ INSTAGRAM_PROFILE_REFUND: "instagram_profile_refund",
8306
+ INSTAGRAM_MEDIA: "instagram_media",
8307
+ INSTAGRAM_MEDIA_REFUND: "instagram_media_refund",
8308
+ INSTAGRAM_TRANSCRIBE: "instagram_transcribe",
8309
+ INSTAGRAM_TRANSCRIBE_REFUND: "instagram_transcribe_refund",
8310
+ BROWSER_SESSION: "browser_session",
8311
+ REDDIT_THREAD: "reddit_thread",
8312
+ REDDIT_THREAD_REFUND: "reddit_thread_refund",
8313
+ REDDIT_COMMENT: "reddit_comment",
8314
+ REDDIT_COMMENT_REFUND: "reddit_comment_refund",
8315
+ VIDEO_ANALYSIS: "video_analysis",
8316
+ VIDEO_ANALYSIS_REFUND: "video_analysis_refund",
8317
+ VIDEO_ANALYSIS_ADJUST: "video_analysis_adjust",
8318
+ VIDEO_ANALYSIS_LLM: "video_analysis_llm",
8319
+ MEMORY_AI: "memory_ai",
8320
+ MEMORY_AI_REFUND: "memory_ai_refund",
8321
+ TRUSTPILOT_REVIEWS: "trustpilot_reviews",
8322
+ TRUSTPILOT_REVIEWS_REFUND: "trustpilot_reviews_refund",
8323
+ TRUSTPILOT_REVIEW: "trustpilot_review",
8324
+ TRUSTPILOT_REVIEW_REFUND: "trustpilot_review_refund",
8325
+ G2_REVIEWS: "g2_reviews",
8326
+ G2_REVIEWS_REFUND: "g2_reviews_refund",
8327
+ G2_REVIEW: "g2_review",
8328
+ G2_REVIEW_REFUND: "g2_review_refund",
8329
+ DIFF_PAGE: "diff_page",
8330
+ DIFF_PAGE_REFUND: "diff_page_refund",
8331
+ SITE_AUDIT_LLM: "site_audit_llm"
8332
+ };
8333
+ MEMORY_AI_MARGIN_MULTIPLE = 3;
8334
+ SITE_AUDIT_LLM_MARGIN_MULTIPLE = 1.5;
8335
+ VIDEO_ANALYSIS_LLM_MARGIN_MULTIPLE = 1.5;
8336
+ MC_PER_USD = SUBSCRIPTION_TIERS["price_1TmiHRS8aAcsk3TGwmSNfNIa"].credits_mc / SUBSCRIPTION_TIERS["price_1TmiHRS8aAcsk3TGwmSNfNIa"].monthly_usd;
8337
+ }
8338
+ });
8339
+
7931
8340
  // src/services/site-architecture-auditor/prompts/ingest-validate.ts
7932
8341
  function buildIngestValidatePrompt(input) {
7933
8342
  const sessionPath = `<session-path-from-job>`;
@@ -8951,6 +9360,9 @@ var init_site_audit_service = __esm({
8951
9360
  import_p_limit = __toESM(require("p-limit"), 1);
8952
9361
  init_llm_parse_with_retry();
8953
9362
  init_schemas();
9363
+ init_db();
9364
+ init_rates();
9365
+ init_cost_context();
8954
9366
  init_ingest_validate();
8955
9367
  init_enrich_competitor();
8956
9368
  init_build_graph_per_site();
@@ -9009,8 +9421,23 @@ var init_site_audit_service = __esm({
9009
9421
  async getJobsForUser(userId) {
9010
9422
  return this.deps.repo.listSiteAuditJobs(userId);
9011
9423
  }
9424
+ async runLlmBilled(userId, source, fn) {
9425
+ const before = this.deps.llm.totalCostUsd;
9426
+ const result = await runWithCostContext({ ...currentCostContext(), op: "audit_site", userId }, fn);
9427
+ const deltaCostUsd = this.deps.llm.totalCostUsd - before;
9428
+ if (deltaCostUsd <= 0) return result;
9429
+ const mc = Math.round(deltaCostUsd * SITE_AUDIT_LLM_MARGIN_MULTIPLE * MC_PER_USD);
9430
+ if (mc <= 0) return result;
9431
+ const { ok } = await debitMc(userId, mc, LedgerOperation.SITE_AUDIT_LLM, source);
9432
+ if (!ok) {
9433
+ console.warn(`[site-audit-service] SITE_AUDIT_LLM surcharge debit failed for user ${userId} (${source}): insufficient balance for ${mc}mc`);
9434
+ }
9435
+ return result;
9436
+ }
9012
9437
  async runIngestPhase(jobId, payload) {
9013
9438
  const { request } = payload;
9439
+ const job = await this.deps.repo.getSiteAuditJob(jobId);
9440
+ if (!job) throw new Error(`[site-audit-service] runIngestPhase: job ${jobId} not found`);
9014
9441
  const input = {
9015
9442
  clientDomain: request.clientDomain,
9016
9443
  sfInternalPath: request.sfInternalPath,
@@ -9025,14 +9452,18 @@ var init_site_audit_service = __esm({
9025
9452
  businessContext: request.businessContext,
9026
9453
  priorAuditMemoryEnabled: request.priorAuditMemoryEnabled
9027
9454
  };
9028
- const result = await llmParseWithRetry(
9029
- ValidatedFileManifestSchema,
9030
- (attempt, error) => {
9031
- void attempt;
9032
- void error;
9033
- return buildIngestValidatePrompt(input);
9034
- },
9035
- this.deps.llm
9455
+ const result = await this.runLlmBilled(
9456
+ job.user_id,
9457
+ `${jobId}:phase1-ingest`,
9458
+ () => llmParseWithRetry(
9459
+ ValidatedFileManifestSchema,
9460
+ (attempt, error) => {
9461
+ void attempt;
9462
+ void error;
9463
+ return buildIngestValidatePrompt(input);
9464
+ },
9465
+ this.deps.llm
9466
+ )
9036
9467
  );
9037
9468
  await this.deps.repo.logSiteAuditPhaseComplete(jobId, "phase1-ingest", result);
9038
9469
  }
@@ -9047,23 +9478,27 @@ var init_site_audit_service = __esm({
9047
9478
  const competitorDomainsCsvPath = request.competitorDomainsCsvPath;
9048
9479
  void competitorDomainsCsvPath;
9049
9480
  const competitors = await this._readCompetitorDomains(request);
9050
- const results = await Promise.all(
9051
- competitors.map(
9052
- (domain, idx) => limit(async () => {
9053
- const competitorIndex = idx + 1;
9054
- const rawPrompt = buildEnrichCompetitorPrompt({ domain, sessionPath });
9055
- const prompt = rawPrompt.replace("<competitor-index>", String(competitorIndex));
9056
- const result = await llmParseWithRetry(
9057
- EnrichCompetitorOutputSchema,
9058
- (attempt, error) => {
9059
- void attempt;
9060
- void error;
9061
- return prompt;
9062
- },
9063
- this.deps.llm
9064
- );
9065
- return result;
9066
- })
9481
+ const results = await this.runLlmBilled(
9482
+ job.user_id,
9483
+ `${jobId}:phase1-enrich`,
9484
+ () => Promise.all(
9485
+ competitors.map(
9486
+ (domain, idx) => limit(async () => {
9487
+ const competitorIndex = idx + 1;
9488
+ const rawPrompt = buildEnrichCompetitorPrompt({ domain, sessionPath });
9489
+ const prompt = rawPrompt.replace("<competitor-index>", String(competitorIndex));
9490
+ const result = await llmParseWithRetry(
9491
+ EnrichCompetitorOutputSchema,
9492
+ (attempt, error) => {
9493
+ void attempt;
9494
+ void error;
9495
+ return prompt;
9496
+ },
9497
+ this.deps.llm
9498
+ );
9499
+ return result;
9500
+ })
9501
+ )
9067
9502
  )
9068
9503
  );
9069
9504
  await this.deps.repo.logSiteAuditPhaseComplete(jobId, "phase1-ingest", { competitors: results });
@@ -9077,19 +9512,23 @@ var init_site_audit_service = __esm({
9077
9512
  const clientDomain = request.clientDomain;
9078
9513
  const urlInventory = [];
9079
9514
  const directedEdgeCount = 0;
9080
- const orphanAnnotation = await llmParseWithRetry(
9081
- OrphanAnnotationOutputSchema,
9082
- (attempt, error) => {
9083
- void attempt;
9084
- void error;
9085
- return buildGraphPerSitePrompt({
9086
- siteId: "client",
9087
- domain: clientDomain,
9088
- urlInventory,
9089
- directedEdgeCount
9090
- });
9091
- },
9092
- this.deps.llm
9515
+ const orphanAnnotation = await this.runLlmBilled(
9516
+ job.user_id,
9517
+ `${jobId}:phase2-build-graph`,
9518
+ () => llmParseWithRetry(
9519
+ OrphanAnnotationOutputSchema,
9520
+ (attempt, error) => {
9521
+ void attempt;
9522
+ void error;
9523
+ return buildGraphPerSitePrompt({
9524
+ siteId: "client",
9525
+ domain: clientDomain,
9526
+ urlInventory,
9527
+ directedEdgeCount
9528
+ });
9529
+ },
9530
+ this.deps.llm
9531
+ )
9093
9532
  );
9094
9533
  const graphMetrics = await this.deps.python.runGraphMetrics({
9095
9534
  directedLinkGraphPath: `${sessionPath}/phase1/directed_link_graph.json`,
@@ -9134,50 +9573,52 @@ var init_site_audit_service = __esm({
9134
9573
  }
9135
9574
  if (currentBatch.length > 0) batches.push(currentBatch);
9136
9575
  const usZipsCsvPath = process.env["US_ZIPS_CSV_PATH"] ?? "/Users/vilovieta/Downloads/sales-magician-api-leads-magician-01c6cff78e31/tools/analytics/data/uszips.csv";
9137
- for (const batch of batches) {
9138
- let classified = [];
9139
- let unclassifiedUrls = batch.map((u) => u.url);
9140
- try {
9141
- const pythonResult = await this.deps.python.runLocationClassifier({
9142
- urls: batch.map((u) => u.url),
9143
- usZipsCsvPath
9144
- });
9145
- classified = pythonResult.classified;
9146
- unclassifiedUrls = pythonResult.unclassified;
9147
- } catch (err) {
9148
- console.warn(
9149
- "[site-audit-service] runClassifyPhase: python.runLocationClassifier failed (falling back to LLM):",
9150
- err instanceof Error ? err.message : String(err)
9151
- );
9152
- unclassifiedUrls = batch.map((u) => u.url);
9153
- }
9154
- for (const c of classified) {
9155
- allClassifications.push({
9156
- url: c.url,
9157
- page_type: c.pageType,
9158
- confidence: c.confidence > 0.6 ? "high" : c.confidence > 0.4 ? "medium" : "low",
9159
- _classification_label: "[COMPUTED:python/location-classifier]",
9160
- topic_cluster: null,
9161
- location_signals: c.locationSignals
9162
- });
9163
- }
9164
- if (unclassifiedUrls.length > 0) {
9165
- const unclassifiedBatch = batch.filter((u) => unclassifiedUrls.includes(u.url));
9166
- const llmResults = await llmParseWithRetry(
9167
- import_zod10.z.array(PageTypeClassificationRowSchema),
9168
- (attempt, error) => {
9169
- void attempt;
9170
- void error;
9171
- return buildMetricsClassifyPrompt({
9172
- urls: unclassifiedBatch,
9173
- siteDomain: clientDomain
9174
- });
9175
- },
9176
- this.deps.llm
9177
- );
9178
- allClassifications.push(...llmResults);
9576
+ await this.runLlmBilled(job.user_id, `${jobId}:phase3-classify`, async () => {
9577
+ for (const batch of batches) {
9578
+ let classified = [];
9579
+ let unclassifiedUrls = batch.map((u) => u.url);
9580
+ try {
9581
+ const pythonResult = await this.deps.python.runLocationClassifier({
9582
+ urls: batch.map((u) => u.url),
9583
+ usZipsCsvPath
9584
+ });
9585
+ classified = pythonResult.classified;
9586
+ unclassifiedUrls = pythonResult.unclassified;
9587
+ } catch (err) {
9588
+ console.warn(
9589
+ "[site-audit-service] runClassifyPhase: python.runLocationClassifier failed (falling back to LLM):",
9590
+ err instanceof Error ? err.message : String(err)
9591
+ );
9592
+ unclassifiedUrls = batch.map((u) => u.url);
9593
+ }
9594
+ for (const c of classified) {
9595
+ allClassifications.push({
9596
+ url: c.url,
9597
+ page_type: c.pageType,
9598
+ confidence: c.confidence > 0.6 ? "high" : c.confidence > 0.4 ? "medium" : "low",
9599
+ _classification_label: "[COMPUTED:python/location-classifier]",
9600
+ topic_cluster: null,
9601
+ location_signals: c.locationSignals
9602
+ });
9603
+ }
9604
+ if (unclassifiedUrls.length > 0) {
9605
+ const unclassifiedBatch = batch.filter((u) => unclassifiedUrls.includes(u.url));
9606
+ const llmResults = await llmParseWithRetry(
9607
+ import_zod10.z.array(PageTypeClassificationRowSchema),
9608
+ (attempt, error) => {
9609
+ void attempt;
9610
+ void error;
9611
+ return buildMetricsClassifyPrompt({
9612
+ urls: unclassifiedBatch,
9613
+ siteDomain: clientDomain
9614
+ });
9615
+ },
9616
+ this.deps.llm
9617
+ );
9618
+ allClassifications.push(...llmResults);
9619
+ }
9179
9620
  }
9180
- }
9621
+ });
9181
9622
  void sessionPath;
9182
9623
  await this.deps.repo.logSiteAuditPhaseComplete(jobId, "phase3-classify", {
9183
9624
  classifications: allClassifications,
@@ -9190,20 +9631,24 @@ var init_site_audit_service = __esm({
9190
9631
  if (!job) throw new Error(`[site-audit-service] runComparePhase: job ${jobId} not found`);
9191
9632
  const request = JSON.parse(job.request);
9192
9633
  const clientDomain = request.clientDomain;
9193
- const result = await llmParseWithRetry(
9194
- CompareRecommendOutputSchema,
9195
- (attempt, error) => {
9196
- void attempt;
9197
- void error;
9198
- return buildCompareRecommendPrompt({
9199
- clientDomain,
9200
- clientMoneyPages: [],
9201
- competitorSites: [],
9202
- clientUrlMetrics: [],
9203
- clientPageTypes: []
9204
- });
9205
- },
9206
- this.deps.llm
9634
+ const result = await this.runLlmBilled(
9635
+ job.user_id,
9636
+ `${jobId}:phase4-compare`,
9637
+ () => llmParseWithRetry(
9638
+ CompareRecommendOutputSchema,
9639
+ (attempt, error) => {
9640
+ void attempt;
9641
+ void error;
9642
+ return buildCompareRecommendPrompt({
9643
+ clientDomain,
9644
+ clientMoneyPages: [],
9645
+ competitorSites: [],
9646
+ clientUrlMetrics: [],
9647
+ clientPageTypes: []
9648
+ });
9649
+ },
9650
+ this.deps.llm
9651
+ )
9207
9652
  );
9208
9653
  await this.deps.repo.logSiteAuditPhaseComplete(jobId, "phase4-compare", result);
9209
9654
  return result;
@@ -9233,20 +9678,24 @@ var init_site_audit_service = __esm({
9233
9678
  total: ilesTotal,
9234
9679
  components: ilesComponents
9235
9680
  };
9236
- const reportMarkdown = await llmParseWithRetry(
9237
- import_zod10.z.string(),
9238
- (attempt, error) => {
9239
- void attempt;
9240
- void error;
9241
- return buildScoreSynthesizePrompt({
9242
- preComputedArchitectureScore,
9243
- preComputedIlesScore,
9244
- rankedBacklog: [],
9245
- clientDomain,
9246
- sessionPath
9247
- });
9248
- },
9249
- this.deps.llm
9681
+ const reportMarkdown = await this.runLlmBilled(
9682
+ job.user_id,
9683
+ `${jobId}:phase5-synthesize`,
9684
+ () => llmParseWithRetry(
9685
+ import_zod10.z.string(),
9686
+ (attempt, error) => {
9687
+ void attempt;
9688
+ void error;
9689
+ return buildScoreSynthesizePrompt({
9690
+ preComputedArchitectureScore,
9691
+ preComputedIlesScore,
9692
+ rankedBacklog: [],
9693
+ clientDomain,
9694
+ sessionPath
9695
+ });
9696
+ },
9697
+ this.deps.llm
9698
+ )
9250
9699
  );
9251
9700
  const auditResult = SiteAuditResultSchema.parse({
9252
9701
  jobId,
@@ -10031,375 +10480,6 @@ var init_image_audit = __esm({
10031
10480
  }
10032
10481
  });
10033
10482
 
10034
- // src/api/rates.ts
10035
- function browserActiveCostMc(activeMs) {
10036
- return Math.round(activeMs * MC_PER_BROWSER_MS);
10037
- }
10038
- function concurrencySlotBillingInfo() {
10039
- const billingUrl = process.env.CONCURRENCY_BILLING_URL ?? "https://mcpscraper.dev/billing";
10040
- return {
10041
- product: "Extra concurrency slot",
10042
- price_label: CONCURRENCY_SLOT_PRICE_LABEL,
10043
- unit_amount_usd: CONCURRENCY_SLOT_PRICE_USD,
10044
- currency: CONCURRENCY_SLOT_PRICE_CURRENCY,
10045
- interval: CONCURRENCY_SLOT_PRICE_INTERVAL,
10046
- billing_url: billingUrl,
10047
- terminal_command: CONCURRENCY_SLOT_TERMINAL_COMMAND,
10048
- terminal_command_with_api_key_env: `MCP_SCRAPER_API_KEY=sk_live_your_key ${CONCURRENCY_SLOT_TERMINAL_COMMAND}`
10049
- };
10050
- }
10051
- function mcToCredits(mc) {
10052
- return mc / MC_PER_CREDIT;
10053
- }
10054
- function insufficientBalanceResponse(balanceMc, requiredMc) {
10055
- const topupUrl = process.env.TOPUP_URL ?? "https://mcpscraper.dev/billing";
10056
- const balanceCredits = mcToCredits(balanceMc);
10057
- const requiredCredits = mcToCredits(requiredMc);
10058
- return {
10059
- error: "insufficient_balance",
10060
- error_code: "insufficient_balance",
10061
- message: `Insufficient credits. Balance: ${balanceCredits} credits. This call requires ${requiredCredits} credits. Top up at ${topupUrl}`,
10062
- balance_credits: balanceCredits,
10063
- required_credits: requiredCredits,
10064
- topup_url: topupUrl
10065
- };
10066
- }
10067
- var MC_COSTS, MC_PER_BROWSER_MS, BROWSER_OPEN_MIN_BALANCE_MC, MC_PER_CREDIT, CREDIT_COST_CATALOG, CONCURRENCY_PRICE_ID, SUBSCRIPTION_TIERS, SUBSCRIPTION_TIER_BY_KEY, CONCURRENCY_SLOT_PRICE_USD, CONCURRENCY_SLOT_PRICE_CURRENCY, CONCURRENCY_SLOT_PRICE_INTERVAL, CONCURRENCY_SLOT_PRICE_LABEL, CONCURRENCY_SLOT_TERMINAL_COMMAND, FREE_SIGNUP_MC, MEMORY_FREE_QUOTA_BYTES, MEMORY_PLANS, MEMORY_PLAN_QUOTA, MEMORY_MARGIN_MULTIPLE, MEMORY_PRO_COST_BUDGET_USD, SCHEDULING_PLANS, LedgerOperation, MEMORY_AI_MARGIN_MULTIPLE, MC_PER_USD;
10068
- var init_rates = __esm({
10069
- "src/api/rates.ts"() {
10070
- "use strict";
10071
- MC_COSTS = {
10072
- serp: 100,
10073
- paa: 200,
10074
- paa_base: 1e3,
10075
- page_scrape: 100,
10076
- url_map: 500,
10077
- yt_channel: 200,
10078
- yt_transcription: 200,
10079
- fb_ad: 200,
10080
- maps_search: 500,
10081
- maps_place: 3e3,
10082
- maps_review: 100,
10083
- fb_search: 600,
10084
- fb_transcribe: 200,
10085
- google_ads_search: 600,
10086
- google_ads_intel: 200,
10087
- google_ads_transcribe: 200,
10088
- instagram_profile: 400,
10089
- instagram_media: 400,
10090
- instagram_transcribe: 50,
10091
- browser_minute: 12e3,
10092
- reddit_thread: 2e3,
10093
- video_analysis: 666667,
10094
- trustpilot_reviews: 500,
10095
- trustpilot_review: 100,
10096
- g2_reviews: 500,
10097
- g2_review: 150,
10098
- diff_page: 100
10099
- };
10100
- MC_PER_BROWSER_MS = MC_COSTS.browser_minute / 6e4;
10101
- BROWSER_OPEN_MIN_BALANCE_MC = 1e3;
10102
- MC_PER_CREDIT = 1e3;
10103
- CREDIT_COST_CATALOG = [
10104
- {
10105
- key: "serp",
10106
- label: "SERP search",
10107
- aliases: ["search_serp", "serp", "google search", "organic results"],
10108
- credits: mcToCredits(MC_COSTS.serp),
10109
- unit: "per search",
10110
- notes: "Returns AI Overview, PAA snippet, videos, forums, and local pack."
10111
- },
10112
- {
10113
- key: "paa",
10114
- label: "PAA harvest",
10115
- aliases: ["harvest_paa", "paa", "people also ask", "questions"],
10116
- credits: mcToCredits(MC_COSTS.paa),
10117
- unit: `per question (+${mcToCredits(MC_COSTS.paa_base)} credit base)`,
10118
- notes: `Full SERP feature extraction. Billed ${mcToCredits(MC_COSTS.paa_base)} credit base + ${mcToCredits(MC_COSTS.paa)} per question actually returned (unused estimate refunded).`
10119
- },
10120
- {
10121
- key: "page_scrape",
10122
- label: "Page crawl / extract",
10123
- aliases: ["extract_url", "extract_site", "page scrape", "url scrape", "single page", "site crawl"],
10124
- credits: mcToCredits(MC_COSTS.page_scrape),
10125
- unit: "per page",
10126
- notes: "Applies to both single-URL extraction and per-page site crawls."
10127
- },
10128
- {
10129
- key: "url_map",
10130
- label: "Site URL mapping",
10131
- aliases: ["map_site_urls", "url map", "site map", "crawl urls"],
10132
- credits: mcToCredits(MC_COSTS.url_map),
10133
- unit: "per mapping operation",
10134
- notes: "Flat rate for the full /map-urls call regardless of URL count discovered."
10135
- },
10136
- {
10137
- key: "yt_channel",
10138
- label: "YouTube search / channel harvest",
10139
- aliases: ["youtube_harvest", "youtube search", "youtube channel", "yt_channel"],
10140
- credits: mcToCredits(MC_COSTS.yt_channel),
10141
- unit: "per call"
10142
- },
10143
- {
10144
- key: "yt_transcription",
10145
- label: "YouTube transcription",
10146
- aliases: ["youtube_transcribe", "youtube transcript", "transcription", "yt_transcription"],
10147
- credits: mcToCredits(MC_COSTS.yt_transcription),
10148
- unit: "per minute",
10149
- notes: "A 5-minute hold is taken, then reconciled to actual video duration."
10150
- },
10151
- {
10152
- key: "fb_ad",
10153
- label: "Facebook search / ad lookup",
10154
- aliases: ["facebook_page_intel", "facebook_ad_search", "facebook_ad", "facebook ads", "fb ads"],
10155
- credits: mcToCredits(MC_COSTS.fb_ad),
10156
- unit: "per call"
10157
- },
10158
- {
10159
- key: "maps_search",
10160
- label: "Maps business search",
10161
- aliases: ["maps_search", "google maps search", "gmb search", "gbp search", "business profiles"],
10162
- credits: mcToCredits(MC_COSTS.maps_search),
10163
- unit: "per search",
10164
- notes: "Returns up to 50 Google Maps business/profile candidates. Use maps_place_intel to hydrate selected businesses."
10165
- },
10166
- {
10167
- key: "maps_place",
10168
- label: "Maps business lookup",
10169
- aliases: ["maps_place_intel", "google maps", "maps place", "place intel"],
10170
- credits: mcToCredits(MC_COSTS.maps_place),
10171
- unit: "per business",
10172
- notes: "Base lookup. Reviews billed separately per card at maps_review rate."
10173
- },
10174
- {
10175
- key: "maps_review",
10176
- label: "Maps review",
10177
- aliases: ["maps_reviews", "google reviews", "review cards", "reviews"],
10178
- credits: mcToCredits(MC_COSTS.maps_review),
10179
- unit: "per review card",
10180
- notes: "Charged after extraction when includeReviews is true."
10181
- },
10182
- {
10183
- key: "fb_search",
10184
- label: "Facebook ad library search",
10185
- aliases: ["facebook_search", "fb_search", "fb ad search"],
10186
- credits: mcToCredits(MC_COSTS.fb_search),
10187
- unit: "per search",
10188
- notes: "Browser automation to search Facebook Ads Library by keyword."
10189
- },
10190
- {
10191
- key: "google_ads_search",
10192
- label: "Google Ads Transparency search",
10193
- aliases: ["google_ads_search", "google ads search", "ads transparency search"],
10194
- credits: mcToCredits(MC_COSTS.google_ads_search),
10195
- unit: "per search",
10196
- notes: "Browser automation to find advertisers in Google Ads Transparency Center by domain or name."
10197
- },
10198
- {
10199
- key: "google_ads_intel",
10200
- label: "Google Ads Transparency advertiser intel",
10201
- aliases: ["google_ads_page_intel", "google ads intel", "ads transparency intel"],
10202
- credits: mcToCredits(MC_COSTS.google_ads_intel),
10203
- unit: "per call",
10204
- notes: "Lists and hydrates an advertiser's creatives with image URLs and video references."
10205
- },
10206
- {
10207
- key: "google_ads_transcribe",
10208
- label: "Google ad video transcription",
10209
- aliases: ["google_ads_transcribe", "google ad transcript"],
10210
- credits: mcToCredits(MC_COSTS.google_ads_transcribe),
10211
- unit: "per minute",
10212
- notes: "A hold is taken, then reconciled to actual video duration."
10213
- },
10214
- {
10215
- key: "fb_transcribe",
10216
- label: "Facebook video / ad transcription",
10217
- aliases: ["facebook_transcribe", "facebook_video_transcribe", "fb_transcribe", "fb ad transcript"],
10218
- credits: mcToCredits(MC_COSTS.fb_transcribe),
10219
- unit: "per minute",
10220
- notes: "A hold is taken, then reconciled to actual video duration."
10221
- },
10222
- {
10223
- key: "instagram_profile",
10224
- label: "Instagram profile content discovery",
10225
- aliases: ["instagram_profile_content", "instagram profile", "instagram content list", "ig profile"],
10226
- credits: mcToCredits(MC_COSTS.instagram_profile),
10227
- unit: "per profile scan",
10228
- notes: "Browser extraction of public Instagram profile grid links. Complete history may require a logged-in profile."
10229
- },
10230
- {
10231
- key: "instagram_media",
10232
- label: "Instagram media download",
10233
- aliases: ["instagram_media_download", "instagram reel download", "instagram post download", "ig media"],
10234
- credits: mcToCredits(MC_COSTS.instagram_media),
10235
- unit: "per post or reel",
10236
- notes: "Extracts post text, image URLs, and reel audio/video tracks, with local downloads when the server can write files."
10237
- },
10238
- {
10239
- key: "instagram_transcribe",
10240
- label: "Instagram media transcription",
10241
- aliases: ["instagram transcript", "instagram reel transcript", "ig transcribe"],
10242
- credits: mcToCredits(MC_COSTS.instagram_transcribe),
10243
- unit: "per call",
10244
- notes: "Whisper transcription of selected Instagram audio/video media via fal.ai."
10245
- },
10246
- {
10247
- key: "browser_minute",
10248
- label: "Interactive browser session",
10249
- aliases: ["browser_open", "browser agent", "browser_agent", "live browser", "browse", "browser control", "interactive browser"],
10250
- credits: mcToCredits(MC_COSTS.browser_minute),
10251
- unit: "per minute of use",
10252
- notes: "Metered per minute of use for the whole time the browser session is open. Close the session to stop the meter; abandoned sessions are auto-closed after a short idle window."
10253
- },
10254
- {
10255
- key: "reddit_thread",
10256
- label: "Reddit thread + comments",
10257
- aliases: ["reddit", "reddit_thread", "reddit comments", "subreddit", "reddit post"],
10258
- credits: mcToCredits(MC_COSTS.reddit_thread),
10259
- unit: "per thread",
10260
- notes: "Captures a Reddit post and its comment tree. Charged on success; refunded if the thread cannot be retrieved."
10261
- },
10262
- {
10263
- key: "video_analysis",
10264
- label: "Video breakdown (frame-by-frame + transcript)",
10265
- aliases: ["video_frame_analysis", "video breakdown", "video analysis", "analyze video"],
10266
- credits: mcToCredits(MC_COSTS.video_analysis),
10267
- unit: "per 120 frames (max 480)",
10268
- notes: "Full multi-lens video breakdown: samples frames across a video (up to 30 minutes), analyzes each with vision AI, transcribes the audio, then produces summary, pacing/energy, words-per-minute, topic outline, key points, hook analysis, visual style, and a how-to-replicate recipe with a quality-control pass. $1 per 120 frames requested (max 480 = $4); billed down automatically if the video cannot use the requested frames, and refunded fully if the run fails."
10269
- },
10270
- {
10271
- key: "trustpilot_reviews",
10272
- label: "Trustpilot review harvest",
10273
- aliases: ["trustpilot_reviews", "trustpilot", "trustpilot reviews"],
10274
- credits: mcToCredits(MC_COSTS.trustpilot_reviews),
10275
- unit: "per call",
10276
- notes: "Base lookup. Reviews billed separately per card at trustpilot_review rate. Refunded if no reviews are found."
10277
- },
10278
- {
10279
- key: "trustpilot_review",
10280
- label: "Trustpilot review card",
10281
- aliases: ["trustpilot_review", "trustpilot card"],
10282
- credits: mcToCredits(MC_COSTS.trustpilot_review),
10283
- unit: "per review card",
10284
- notes: "Charged per review actually extracted, billed down automatically if the balance runs out mid-page."
10285
- },
10286
- {
10287
- key: "g2_reviews",
10288
- label: "G2 review harvest",
10289
- aliases: ["g2_reviews", "g2", "g2 reviews"],
10290
- credits: mcToCredits(MC_COSTS.g2_reviews),
10291
- unit: "per call",
10292
- notes: "Base lookup. Reviews billed separately per card at g2_review rate. Refunded if no reviews are found."
10293
- },
10294
- {
10295
- key: "g2_review",
10296
- label: "G2 review card",
10297
- aliases: ["g2_review", "g2 card"],
10298
- credits: mcToCredits(MC_COSTS.g2_review),
10299
- unit: "per review card",
10300
- notes: "Charged per review actually extracted (each card carries up to 3 Q&A sections), billed down automatically if the balance runs out mid-page."
10301
- },
10302
- {
10303
- key: "diff_page",
10304
- label: "Page change check",
10305
- aliases: ["diff_page", "page diff", "change detection"],
10306
- credits: mcToCredits(MC_COSTS.diff_page),
10307
- unit: "per check",
10308
- notes: "Same cost as a single page extract \u2014 one scrape is performed per check, then compared against your last stored snapshot for that URL."
10309
- }
10310
- ];
10311
- CONCURRENCY_PRICE_ID = "price_1Ta1NRS8aAcsk3TGwsRnYbix";
10312
- SUBSCRIPTION_TIERS = {
10313
- "price_1TmiHRS8aAcsk3TGwmSNfNIa": { tier: "starter", label: "Starter", price_id: "price_1TmiHRS8aAcsk3TGwmSNfNIa", monthly_usd: 12, credits_mc: 8e6, concurrency: 3, intro_coupon: "mcp-starter-1dollar-intro-12" },
10314
- "price_1Tmg1nS8aAcsk3TGe8zcnGTM": { tier: "growth", label: "Growth", price_id: "price_1Tmg1nS8aAcsk3TGe8zcnGTM", monthly_usd: 100, credits_mc: 8e7, concurrency: 10, intro_coupon: null },
10315
- "price_1Tmg1nS8aAcsk3TGsZw34iXS": { tier: "scale", label: "Scale", price_id: "price_1Tmg1nS8aAcsk3TGsZw34iXS", monthly_usd: 250, credits_mc: 24e7, concurrency: 20, intro_coupon: null }
10316
- };
10317
- SUBSCRIPTION_TIER_BY_KEY = Object.fromEntries(
10318
- Object.values(SUBSCRIPTION_TIERS).map((t) => [t.tier, t])
10319
- );
10320
- CONCURRENCY_SLOT_PRICE_USD = 5;
10321
- CONCURRENCY_SLOT_PRICE_CURRENCY = "usd";
10322
- CONCURRENCY_SLOT_PRICE_INTERVAL = "month";
10323
- CONCURRENCY_SLOT_PRICE_LABEL = "$5/month";
10324
- CONCURRENCY_SLOT_TERMINAL_COMMAND = "npx -y -p mcp-scraper@latest mcp-scraper-cli billing concurrency checkout";
10325
- FREE_SIGNUP_MC = 1e5;
10326
- MEMORY_FREE_QUOTA_BYTES = 1e7 * 1;
10327
- MEMORY_PLANS = {
10328
- "price_1TnMSTS8aAcsk3TGBgiwuvqL": { plan: "pro", label: "Pro", price_id: "price_1TnMSTS8aAcsk3TGBgiwuvqL", interval: "month", monthly_usd: 19, quota_bytes: 5e9 },
10329
- "price_1TnMSTS8aAcsk3TGWBVU2agY": { plan: "pro", label: "Pro", price_id: "price_1TnMSTS8aAcsk3TGWBVU2agY", interval: "year", monthly_usd: 19, quota_bytes: 5e9 }
10330
- };
10331
- MEMORY_PLAN_QUOTA = { free: MEMORY_FREE_QUOTA_BYTES, pro: 5e9, team: 5e10 };
10332
- MEMORY_MARGIN_MULTIPLE = 3;
10333
- MEMORY_PRO_COST_BUDGET_USD = 19 / MEMORY_MARGIN_MULTIPLE;
10334
- SCHEDULING_PLANS = {
10335
- "price_1ToXcHS8aAcsk3TGf3cW7zHx": { price_id: "price_1ToXcHS8aAcsk3TGf3cW7zHx", label: "Scheduled Actions", interval: "month", monthly_usd: 10, quota_per_period: 1e3 }
10336
- };
10337
- LedgerOperation = {
10338
- TOPUP: "topup",
10339
- SUBSCRIPTION: "subscription",
10340
- SIGNUP_GRANT: "signup_grant",
10341
- MONTHLY_REFRESH: "monthly_free_refresh",
10342
- PAA: "paa",
10343
- PAA_REFUND: "paa_refund",
10344
- SERP: "serp",
10345
- REFUND: "refund",
10346
- TRANSCRIPTION: "transcription",
10347
- TRANSCRIPTION_HOLD: "transcription_hold",
10348
- TRANSCRIPTION_REFUND: "transcription_refund",
10349
- YT_CHANNEL: "yt_channel",
10350
- FB_AD: "fb_ad",
10351
- MAPS_SEARCH: "maps_search",
10352
- MAPS_PLACE: "maps_place",
10353
- MAPS_REVIEW: "maps_review",
10354
- MAPS_REVIEW_REFUND: "maps_review_refund",
10355
- EXTRACT_SITE: "extract_site",
10356
- EXTRACT_SITE_REFUND: "extract_site_refund",
10357
- EXTRACT_URL: "page_scrape",
10358
- URL_MAP: "url_map",
10359
- EXTRACT_SITE_HOLD: "extract_site_hold",
10360
- YT_CHANNEL_REFUND: "yt_channel_refund",
10361
- FB_AD_REFUND: "fb_ad_refund",
10362
- URL_MAP_REFUND: "url_map_refund",
10363
- FB_SEARCH: "fb_search",
10364
- FB_TRANSCRIBE: "fb_transcribe",
10365
- FB_SEARCH_REFUND: "fb_search_refund",
10366
- FB_TRANSCRIBE_REFUND: "fb_transcribe_refund",
10367
- GOOGLE_ADS_SEARCH: "google_ads_search",
10368
- GOOGLE_ADS_SEARCH_REFUND: "google_ads_search_refund",
10369
- GOOGLE_ADS_INTEL: "google_ads_intel",
10370
- GOOGLE_ADS_INTEL_REFUND: "google_ads_intel_refund",
10371
- GOOGLE_ADS_TRANSCRIBE: "google_ads_transcribe",
10372
- GOOGLE_ADS_TRANSCRIBE_REFUND: "google_ads_transcribe_refund",
10373
- INSTAGRAM_PROFILE: "instagram_profile",
10374
- INSTAGRAM_PROFILE_REFUND: "instagram_profile_refund",
10375
- INSTAGRAM_MEDIA: "instagram_media",
10376
- INSTAGRAM_MEDIA_REFUND: "instagram_media_refund",
10377
- INSTAGRAM_TRANSCRIBE: "instagram_transcribe",
10378
- INSTAGRAM_TRANSCRIBE_REFUND: "instagram_transcribe_refund",
10379
- BROWSER_SESSION: "browser_session",
10380
- REDDIT_THREAD: "reddit_thread",
10381
- REDDIT_THREAD_REFUND: "reddit_thread_refund",
10382
- VIDEO_ANALYSIS: "video_analysis",
10383
- VIDEO_ANALYSIS_REFUND: "video_analysis_refund",
10384
- VIDEO_ANALYSIS_ADJUST: "video_analysis_adjust",
10385
- MEMORY_AI: "memory_ai",
10386
- MEMORY_AI_REFUND: "memory_ai_refund",
10387
- TRUSTPILOT_REVIEWS: "trustpilot_reviews",
10388
- TRUSTPILOT_REVIEWS_REFUND: "trustpilot_reviews_refund",
10389
- TRUSTPILOT_REVIEW: "trustpilot_review",
10390
- TRUSTPILOT_REVIEW_REFUND: "trustpilot_review_refund",
10391
- G2_REVIEWS: "g2_reviews",
10392
- G2_REVIEWS_REFUND: "g2_reviews_refund",
10393
- G2_REVIEW: "g2_review",
10394
- G2_REVIEW_REFUND: "g2_review_refund",
10395
- DIFF_PAGE: "diff_page",
10396
- DIFF_PAGE_REFUND: "diff_page_refund"
10397
- };
10398
- MEMORY_AI_MARGIN_MULTIPLE = 3;
10399
- MC_PER_USD = SUBSCRIPTION_TIERS["price_1TmiHRS8aAcsk3TGwmSNfNIa"].credits_mc / SUBSCRIPTION_TIERS["price_1TmiHRS8aAcsk3TGwmSNfNIa"].monthly_usd;
10400
- }
10401
- });
10402
-
10403
10483
  // src/api/site-extract-repository.ts
10404
10484
  var site_extract_repository_exports = {};
10405
10485
  __export(site_extract_repository_exports, {
@@ -11619,6 +11699,8 @@ var init_internal_memory_routes = __esm({
11619
11699
  import_hono2 = require("hono");
11620
11700
  init_db();
11621
11701
  init_rates();
11702
+ init_cost_telemetry();
11703
+ init_cost_context();
11622
11704
  internalMemoryApp = new import_hono2.Hono();
11623
11705
  internalMemoryApp.post("/ai-debit", async (c) => {
11624
11706
  const secret2 = c.req.header("x-internal-secret");
@@ -11631,6 +11713,11 @@ var init_internal_memory_routes = __esm({
11631
11713
  }
11632
11714
  const user = await getUserByEmail(body.identity.trim().toLowerCase());
11633
11715
  if (!user) return c.json({ ok: false, reason: "no_user" }, 200);
11716
+ if (body.costUsd > 0) {
11717
+ await runWithCostContext({ ...currentCostContext(), op: "memory_ai", userId: user.id }, async () => {
11718
+ void recordVendorUsage({ vendor: "mcp_memory_ai", model: null, units: body.costUsd, unitType: "usd", error: null });
11719
+ });
11720
+ }
11634
11721
  const mc = Math.round(body.costUsd * MEMORY_AI_MARGIN_MULTIPLE * MC_PER_USD);
11635
11722
  if (mc <= 0) return c.json({ ok: true, mc: 0, balance_mc: null }, 200);
11636
11723
  const { ok, balance_mc } = await debitMc(user.id, mc, LedgerOperation.MEMORY_AI, `${body.source ?? "ai"}:${body.op ?? ""}`);
@@ -12893,6 +12980,563 @@ var init_youtube_harvest = __esm({
12893
12980
  }
12894
12981
  });
12895
12982
 
12983
+ // src/locations.ts
12984
+ var LOCATIONS;
12985
+ var init_locations = __esm({
12986
+ "src/locations.ts"() {
12987
+ "use strict";
12988
+ LOCATIONS = {
12989
+ "austin": "Austin,Texas,United States",
12990
+ "new york": "New York,New York,United States",
12991
+ "new york city": "New York,New York,United States",
12992
+ "nyc": "New York,New York,United States",
12993
+ "los angeles": "Los Angeles,California,United States",
12994
+ "la": "Los Angeles,California,United States",
12995
+ "chicago": "Chicago,Illinois,United States",
12996
+ "houston": "Houston,Texas,United States",
12997
+ "phoenix": "Phoenix,Arizona,United States",
12998
+ "philadelphia": "Philadelphia,Pennsylvania,United States",
12999
+ "philly": "Philadelphia,Pennsylvania,United States",
13000
+ "san antonio": "San Antonio,Texas,United States",
13001
+ "dallas": "Dallas,Texas,United States",
13002
+ "miami": "Miami,Florida,United States",
13003
+ "seattle": "Seattle,Washington,United States",
13004
+ "denver": "Denver,Colorado,United States",
13005
+ "loveland": "Loveland,Colorado,United States",
13006
+ "loveland co": "Loveland,Colorado,United States",
13007
+ "fort collins": "Fort Collins,Colorado,United States",
13008
+ "boulder": "Boulder,Colorado,United States",
13009
+ "colorado springs": "Colorado Springs,Colorado,United States",
13010
+ "boston": "Boston,Massachusetts,United States",
13011
+ "atlanta": "Atlanta,Georgia,United States",
13012
+ "san francisco": "San Francisco,California,United States",
13013
+ "sf": "San Francisco,California,United States",
13014
+ "portland": "Portland,Oregon,United States",
13015
+ "las vegas": "Las Vegas,Nevada,United States",
13016
+ "minneapolis": "Minneapolis,Minnesota,United States",
13017
+ "detroit": "Detroit,Michigan,United States",
13018
+ "nashville": "Nashville,Tennessee,United States",
13019
+ "charlotte": "Charlotte,North Carolina,United States",
13020
+ "orlando": "Orlando,Florida,United States",
13021
+ "san diego": "San Diego,California,United States",
13022
+ "baltimore": "Baltimore,Maryland,United States",
13023
+ "sacramento": "Sacramento,California,United States",
13024
+ "columbus": "Columbus,Ohio,United States",
13025
+ "indianapolis": "Indianapolis,Indiana,United States",
13026
+ "san jose": "San Jose,California,United States",
13027
+ "fort worth": "Fort Worth,Texas,United States",
13028
+ "jacksonville": "Jacksonville,Florida,United States",
13029
+ "memphis": "Memphis,Tennessee,United States",
13030
+ "louisville": "Louisville,Kentucky,United States",
13031
+ "raleigh": "Raleigh,North Carolina,United States",
13032
+ "richmond": "Richmond,Virginia,United States",
13033
+ "salt lake city": "Salt Lake City,Utah,United States",
13034
+ "toronto": "Toronto,Ontario,Canada",
13035
+ "vancouver": "Vancouver,British Columbia,Canada",
13036
+ "montreal": "Montreal,Quebec,Canada",
13037
+ "calgary": "Calgary,Alberta,Canada",
13038
+ "ottawa": "Ottawa,Ontario,Canada",
13039
+ "london": "London,England,United Kingdom",
13040
+ "manchester": "Manchester,England,United Kingdom",
13041
+ "birmingham": "Birmingham,England,United Kingdom",
13042
+ "edinburgh": "Edinburgh,Scotland,United Kingdom",
13043
+ "glasgow": "Glasgow,Scotland,United Kingdom",
13044
+ "leeds": "Leeds,England,United Kingdom",
13045
+ "sydney": "Sydney,New South Wales,Australia",
13046
+ "melbourne": "Melbourne,Victoria,Australia",
13047
+ "brisbane": "Brisbane,Queensland,Australia",
13048
+ "perth": "Perth,Western Australia,Australia",
13049
+ "adelaide": "Adelaide,South Australia,Australia",
13050
+ "dublin": "Dublin,Leinster,Ireland"
13051
+ };
13052
+ }
13053
+ });
13054
+
13055
+ // src/uule.ts
13056
+ function encodeVarint(value) {
13057
+ const bytes = [];
13058
+ let remaining = value;
13059
+ do {
13060
+ let byte = remaining & 127;
13061
+ remaining >>>= 7;
13062
+ if (remaining > 0) byte |= 128;
13063
+ bytes.push(byte);
13064
+ } while (remaining > 0);
13065
+ return bytes;
13066
+ }
13067
+ function encodeUule(name) {
13068
+ const locationBytes = Buffer.from(name, "utf8");
13069
+ const payload = Buffer.concat([
13070
+ Buffer.from([8, 2, 16, 32, 34]),
13071
+ Buffer.from(encodeVarint(locationBytes.length)),
13072
+ locationBytes
13073
+ ]);
13074
+ return `w+${payload.toString("base64")}`;
13075
+ }
13076
+ function normalizeLocation(input) {
13077
+ const raw = input.toLowerCase().trim();
13078
+ if (LOCATIONS[raw]) return LOCATIONS[raw];
13079
+ const beforeComma = raw.split(",")[0].trim();
13080
+ if (beforeComma !== raw && LOCATIONS[beforeComma]) return LOCATIONS[beforeComma];
13081
+ const withoutState = raw.replace(/\s+[a-z]{2}$/, "").trim();
13082
+ if (withoutState !== raw && LOCATIONS[withoutState]) return LOCATIONS[withoutState];
13083
+ return input;
13084
+ }
13085
+ var init_uule = __esm({
13086
+ "src/uule.ts"() {
13087
+ "use strict";
13088
+ init_locations();
13089
+ }
13090
+ });
13091
+
13092
+ // src/kernel-proxy-resolver.ts
13093
+ function proxyIdSuffix2(proxyId) {
13094
+ return proxyId ? proxyId.slice(-6) : null;
13095
+ }
13096
+ function resolution(source, proxyMode, proxyId, target, error, disposable = false) {
13097
+ const proxyType = source === "disabled" ? "none" : source === "configured_fallback" ? "configured" : source === "isp_created" ? "isp" : source === "location_created" || source === "location_reused" ? "residential" : "unknown";
13098
+ return {
13099
+ kernelProxyId: proxyId,
13100
+ ...disposable && proxyId ? { disposableProxyId: proxyId } : {},
13101
+ resolution: {
13102
+ source,
13103
+ proxyType,
13104
+ proxyMode,
13105
+ proxyIdPresent: Boolean(proxyId),
13106
+ proxyIdSuffix: proxyIdSuffix2(proxyId),
13107
+ target,
13108
+ error,
13109
+ disposable
13110
+ }
13111
+ };
13112
+ }
13113
+ function normalizeStateName(value) {
13114
+ return value.trim().toLowerCase().replace(/\s+/g, " ");
13115
+ }
13116
+ function normalizeCountryName(value) {
13117
+ return value.trim().toLowerCase().replace(/\./g, "").replace(/\s+/g, " ");
13118
+ }
13119
+ function isUnitedStates(country) {
13120
+ if (!country) return true;
13121
+ const normalized = normalizeCountryName(country);
13122
+ return normalized === "united states" || normalized === "united states of america" || normalized === "usa" || normalized === "us";
13123
+ }
13124
+ function stateCodeFor(region) {
13125
+ const trimmed = region.trim();
13126
+ if (/^[A-Za-z]{2}$/.test(trimmed)) return trimmed.toUpperCase();
13127
+ return US_STATE_CODES[normalizeStateName(trimmed)] ?? null;
13128
+ }
13129
+ function kernelCityIdentifierCandidates(city) {
13130
+ const ascii = city.normalize("NFKD").replace(/[^\x00-\x7F]/g, "").toLowerCase();
13131
+ const words = ascii.split(/[^a-z0-9]+/).filter(Boolean);
13132
+ const underscored = words.join("_");
13133
+ const compact = words.join("");
13134
+ return Array.from(new Set([underscored, compact].filter(Boolean)));
13135
+ }
13136
+ function proxyName(country, state, city) {
13137
+ return city ? `mcp-serp-residential-${country.toLowerCase()}-${state.toLowerCase()}-${city}` : `mcp-serp-residential-${country.toLowerCase()}-${state.toLowerCase()}`;
13138
+ }
13139
+ function freshProxyName(baseName, attemptIndex) {
13140
+ const stamp = `${Date.now()}-${attemptIndex ?? 0}-${Math.random().toString(36).slice(2, 8)}`;
13141
+ return `${baseName}-fresh-${stamp}`;
13142
+ }
13143
+ function zipProxyName(zip) {
13144
+ return `mcp-serp-residential-us-zip-${zip}`;
13145
+ }
13146
+ function parseKernelLocationProxyTarget(location2, gl) {
13147
+ if (!location2 || gl.toLowerCase() !== "us") return null;
13148
+ const canonicalLocation = normalizeLocation(location2);
13149
+ let parts = canonicalLocation.split(",").map((part) => part.trim()).filter(Boolean);
13150
+ if (parts.length > 1 && isUnitedStates(parts[parts.length - 1])) {
13151
+ parts = parts.slice(0, -1);
13152
+ }
13153
+ if (parts.length === 1) {
13154
+ const stateOnly = stateCodeFor(parts[0]);
13155
+ if (!stateOnly) return null;
13156
+ return {
13157
+ canonicalLocation,
13158
+ level: "state",
13159
+ country: "US",
13160
+ state: stateOnly,
13161
+ city: "",
13162
+ cityCandidates: [],
13163
+ proxyName: proxyName("US", stateOnly),
13164
+ config: {
13165
+ country: "US",
13166
+ state: stateOnly
13167
+ }
13168
+ };
13169
+ }
13170
+ const [city = "", region = ""] = parts;
13171
+ if (!city || !region) return null;
13172
+ const state = stateCodeFor(region);
13173
+ if (!state) return null;
13174
+ const cityCandidates = kernelCityIdentifierCandidates(city);
13175
+ const primaryCity = cityCandidates[0];
13176
+ if (!primaryCity) return null;
13177
+ return {
13178
+ canonicalLocation,
13179
+ level: "city",
13180
+ country: "US",
13181
+ state,
13182
+ city: primaryCity,
13183
+ cityCandidates,
13184
+ proxyName: proxyName("US", state, primaryCity),
13185
+ config: {
13186
+ country: "US",
13187
+ state,
13188
+ city: primaryCity
13189
+ }
13190
+ };
13191
+ }
13192
+ function cityZipKey(target) {
13193
+ return `${target.city}|${target.state}`;
13194
+ }
13195
+ function knownZipFor(target, explicitZip) {
13196
+ if (explicitZip && /^\d{5}$/.test(explicitZip)) return explicitZip;
13197
+ return US_CITY_CENTER_ZIPS[cityZipKey(target)] ?? null;
13198
+ }
13199
+ function zipTarget(target, zip) {
13200
+ return {
13201
+ ...target,
13202
+ level: "zip",
13203
+ zip,
13204
+ proxyName: zipProxyName(zip),
13205
+ config: {
13206
+ country: target.country,
13207
+ state: target.state,
13208
+ zip
13209
+ }
13210
+ };
13211
+ }
13212
+ function withProxyName(target, name) {
13213
+ return {
13214
+ ...target,
13215
+ proxyName: name
13216
+ };
13217
+ }
13218
+ function configMatches(config, target, city) {
13219
+ if (target.level === "zip") {
13220
+ return config?.country?.toUpperCase() === target.country && config?.zip === target.zip;
13221
+ }
13222
+ return config?.country?.toUpperCase() === target.country && config?.state?.toUpperCase() === target.state && (city ? config?.city === city : !config?.city);
13223
+ }
13224
+ function findExistingTargetProxy(proxies, target) {
13225
+ return proxies.find((proxy) => proxy.type === "residential" && proxy.status !== "unavailable" && Boolean(proxy.id) && (proxy.name === target.proxyName || configMatches(proxy.config, target, target.level === "city" ? target.city : void 0))) ?? null;
13226
+ }
13227
+ function findExistingProxy(proxies, target) {
13228
+ for (const city of target.cityCandidates) {
13229
+ const name = proxyName(target.country, target.state, city);
13230
+ const found = proxies.find((proxy) => proxy.type === "residential" && proxy.status !== "unavailable" && Boolean(proxy.id) && (proxy.name === name || configMatches(proxy.config, target, city)));
13231
+ if (found) return found;
13232
+ }
13233
+ return null;
13234
+ }
13235
+ function stateTarget(target) {
13236
+ return {
13237
+ ...target,
13238
+ level: "state",
13239
+ proxyName: proxyName(target.country, target.state),
13240
+ config: {
13241
+ country: target.country,
13242
+ state: target.state
13243
+ }
13244
+ };
13245
+ }
13246
+ function findExistingStateProxy(proxies, target) {
13247
+ const name = proxyName(target.country, target.state);
13248
+ return proxies.find((proxy) => proxy.type === "residential" && proxy.status !== "unavailable" && Boolean(proxy.id) && (proxy.name === name || configMatches(proxy.config, target))) ?? null;
13249
+ }
13250
+ function escalatedTargetLevel(target, attemptIndex) {
13251
+ return stateTarget(target);
13252
+ }
13253
+ function errorText2(err) {
13254
+ return err instanceof Error ? err.message : String(err);
13255
+ }
13256
+ function freshTargetCandidates(target, explicitZip, attemptIndex) {
13257
+ const out = [];
13258
+ const zip = knownZipFor(target, explicitZip);
13259
+ if (zip) {
13260
+ const targetZip = zipTarget(target, zip);
13261
+ out.push(withProxyName(targetZip, freshProxyName(targetZip.proxyName, attemptIndex)));
13262
+ }
13263
+ for (const city of target.cityCandidates) {
13264
+ const cityTarget = {
13265
+ ...target,
13266
+ level: "city",
13267
+ city,
13268
+ proxyName: proxyName(target.country, target.state, city),
13269
+ config: {
13270
+ country: target.country,
13271
+ state: target.state,
13272
+ city
13273
+ }
13274
+ };
13275
+ out.push(withProxyName(cityTarget, freshProxyName(cityTarget.proxyName, attemptIndex)));
13276
+ }
13277
+ const fallbackTarget = stateTarget(target);
13278
+ out.push(withProxyName(fallbackTarget, freshProxyName(fallbackTarget.proxyName, attemptIndex)));
13279
+ return out;
13280
+ }
13281
+ async function createFreshLocationProxy(kernel, options, target) {
13282
+ const createErrors = [];
13283
+ for (const candidate of freshTargetCandidates(target, options.proxyZip, options.attemptIndex)) {
13284
+ try {
13285
+ const created = await kernel.proxies.create({
13286
+ type: "residential",
13287
+ name: candidate.proxyName,
13288
+ config: candidate.level === "zip" ? { country: candidate.country, zip: candidate.zip } : candidate.config
13289
+ });
13290
+ if (created.id) {
13291
+ return resolution("location_created", options.proxyMode, created.id, candidate, null, true);
13292
+ }
13293
+ createErrors.push(`${candidate.proxyName}: Kernel did not return a proxy id`);
13294
+ } catch (err) {
13295
+ createErrors.push(`${candidate.proxyName}: ${errorText2(err)}`);
13296
+ }
13297
+ }
13298
+ if (process.env.KERNEL_PROXY_TYPE !== "residential") {
13299
+ const ispTarget = stateTarget(target);
13300
+ try {
13301
+ const created = await kernel.proxies.create({
13302
+ type: "isp",
13303
+ name: freshProxyName(`mcp-serp-isp-${ispTarget.country.toLowerCase()}-${(ispTarget.state ?? "").toLowerCase()}`, options.attemptIndex),
13304
+ config: { country: ispTarget.country, ...ispTarget.state ? { state: ispTarget.state } : {} }
13305
+ });
13306
+ if (created.id) {
13307
+ return resolution("isp_created", options.proxyMode, created.id, ispTarget, null, true);
13308
+ }
13309
+ createErrors.push("isp: Kernel did not return a proxy id");
13310
+ } catch (err) {
13311
+ createErrors.push(`isp: ${errorText2(err)}`);
13312
+ }
13313
+ }
13314
+ return resolution("configured_fallback", options.proxyMode, options.configuredKernelProxyId, target, createErrors.join(" | "));
13315
+ }
13316
+ async function deleteKernelProxyId(kernelApiKey, proxyId) {
13317
+ if (!kernelApiKey || !proxyId) return;
13318
+ const kernel = new import_sdk6.default({ apiKey: kernelApiKey });
13319
+ await kernel.proxies.delete(proxyId);
13320
+ }
13321
+ async function resolveKernelProxyId(options) {
13322
+ if (options.proxyMode === "none") {
13323
+ return resolution("disabled", options.proxyMode, void 0, null, null);
13324
+ }
13325
+ if (options.proxyMode === "configured") {
13326
+ return resolution("configured_fallback", options.proxyMode, options.configuredKernelProxyId, null, null);
13327
+ }
13328
+ const target = parseKernelLocationProxyTarget(options.location, options.gl);
13329
+ if (!target || !options.kernelApiKey) {
13330
+ return resolution("configured_fallback", options.proxyMode, options.configuredKernelProxyId, target, target ? null : "location could not be normalized to a US city/state proxy target");
13331
+ }
13332
+ const kernel = new import_sdk6.default({ apiKey: options.kernelApiKey });
13333
+ try {
13334
+ const attemptIndex = options.attemptIndex ?? 0;
13335
+ if (options.fresh) {
13336
+ return await createFreshLocationProxy(kernel, options, target);
13337
+ }
13338
+ if (attemptIndex >= 1) {
13339
+ const escalatedTarget = escalatedTargetLevel(target, attemptIndex);
13340
+ const createErrors2 = [];
13341
+ try {
13342
+ const created = await kernel.proxies.create({
13343
+ type: "residential",
13344
+ name: escalatedTarget.proxyName,
13345
+ config: escalatedTarget.config
13346
+ });
13347
+ if (created.id) {
13348
+ return resolution("location_created", options.proxyMode, created.id, escalatedTarget, null);
13349
+ }
13350
+ createErrors2.push(`${escalatedTarget.state}: Kernel did not return a proxy id`);
13351
+ } catch (err) {
13352
+ createErrors2.push(`${escalatedTarget.state}: ${errorText2(err)}`);
13353
+ }
13354
+ return resolution("configured_fallback", options.proxyMode, options.configuredKernelProxyId, escalatedTarget, createErrors2.join(" | "));
13355
+ }
13356
+ const proxies = await kernel.proxies.list();
13357
+ const zip = knownZipFor(target, options.proxyZip);
13358
+ const createErrors = [];
13359
+ if (zip) {
13360
+ const targetZip = zipTarget(target, zip);
13361
+ const existingZip = findExistingTargetProxy(proxies, targetZip);
13362
+ if (existingZip?.id) {
13363
+ return resolution("location_reused", options.proxyMode, existingZip.id, targetZip, null);
13364
+ }
13365
+ try {
13366
+ const created = await kernel.proxies.create({
13367
+ type: "residential",
13368
+ name: targetZip.proxyName,
13369
+ config: {
13370
+ country: targetZip.country,
13371
+ zip
13372
+ }
13373
+ });
13374
+ if (created.id) {
13375
+ return resolution("location_created", options.proxyMode, created.id, targetZip, null);
13376
+ }
13377
+ createErrors.push(`${zip}: Kernel did not return a proxy id`);
13378
+ } catch (err) {
13379
+ createErrors.push(`${zip}: ${errorText2(err)}`);
13380
+ }
13381
+ }
13382
+ const existing = findExistingProxy(proxies, target);
13383
+ if (existing?.id) {
13384
+ return resolution("location_reused", options.proxyMode, existing.id, target, createErrors.join(" | ") || null);
13385
+ }
13386
+ for (const city of target.cityCandidates) {
13387
+ try {
13388
+ const created = await kernel.proxies.create({
13389
+ type: "residential",
13390
+ name: proxyName(target.country, target.state, city),
13391
+ config: {
13392
+ country: target.country,
13393
+ state: target.state,
13394
+ city
13395
+ }
13396
+ });
13397
+ if (created.id) {
13398
+ return resolution("location_created", options.proxyMode, created.id, {
13399
+ ...target,
13400
+ level: "city",
13401
+ city,
13402
+ proxyName: proxyName(target.country, target.state, city),
13403
+ config: {
13404
+ country: target.country,
13405
+ state: target.state,
13406
+ city
13407
+ }
13408
+ }, null);
13409
+ }
13410
+ createErrors.push(`${city}: Kernel did not return a proxy id`);
13411
+ } catch (err) {
13412
+ createErrors.push(`${city}: ${errorText2(err)}`);
13413
+ }
13414
+ }
13415
+ const fallbackTarget = stateTarget(target);
13416
+ const existingState = findExistingStateProxy(proxies, fallbackTarget);
13417
+ if (existingState?.id) {
13418
+ return resolution("location_reused", options.proxyMode, existingState.id, fallbackTarget, createErrors.join(" | "));
13419
+ }
13420
+ try {
13421
+ const created = await kernel.proxies.create({
13422
+ type: "residential",
13423
+ name: fallbackTarget.proxyName,
13424
+ config: fallbackTarget.config
13425
+ });
13426
+ if (created.id) {
13427
+ return resolution("location_created", options.proxyMode, created.id, fallbackTarget, createErrors.join(" | "));
13428
+ }
13429
+ createErrors.push(`${fallbackTarget.state}: Kernel did not return a proxy id`);
13430
+ } catch (err) {
13431
+ createErrors.push(`${fallbackTarget.state}: ${errorText2(err)}`);
13432
+ }
13433
+ return resolution("configured_fallback", options.proxyMode, options.configuredKernelProxyId, target, createErrors.join(" | "));
13434
+ } catch (err) {
13435
+ return resolution("configured_fallback", options.proxyMode, options.configuredKernelProxyId, target, errorText2(err));
13436
+ }
13437
+ }
13438
+ var import_sdk6, US_STATE_CODES, US_CITY_CENTER_ZIPS;
13439
+ var init_kernel_proxy_resolver = __esm({
13440
+ "src/kernel-proxy-resolver.ts"() {
13441
+ "use strict";
13442
+ import_sdk6 = __toESM(require("@onkernel/sdk"), 1);
13443
+ init_uule();
13444
+ US_STATE_CODES = {
13445
+ alabama: "AL",
13446
+ alaska: "AK",
13447
+ arizona: "AZ",
13448
+ arkansas: "AR",
13449
+ california: "CA",
13450
+ colorado: "CO",
13451
+ connecticut: "CT",
13452
+ delaware: "DE",
13453
+ florida: "FL",
13454
+ georgia: "GA",
13455
+ hawaii: "HI",
13456
+ idaho: "ID",
13457
+ illinois: "IL",
13458
+ indiana: "IN",
13459
+ iowa: "IA",
13460
+ kansas: "KS",
13461
+ kentucky: "KY",
13462
+ louisiana: "LA",
13463
+ maine: "ME",
13464
+ maryland: "MD",
13465
+ massachusetts: "MA",
13466
+ michigan: "MI",
13467
+ minnesota: "MN",
13468
+ mississippi: "MS",
13469
+ missouri: "MO",
13470
+ montana: "MT",
13471
+ nebraska: "NE",
13472
+ nevada: "NV",
13473
+ "new hampshire": "NH",
13474
+ "new jersey": "NJ",
13475
+ "new mexico": "NM",
13476
+ "new york": "NY",
13477
+ "north carolina": "NC",
13478
+ "north dakota": "ND",
13479
+ ohio: "OH",
13480
+ oklahoma: "OK",
13481
+ oregon: "OR",
13482
+ pennsylvania: "PA",
13483
+ "rhode island": "RI",
13484
+ "south carolina": "SC",
13485
+ "south dakota": "SD",
13486
+ tennessee: "TN",
13487
+ texas: "TX",
13488
+ utah: "UT",
13489
+ vermont: "VT",
13490
+ virginia: "VA",
13491
+ washington: "WA",
13492
+ "west virginia": "WV",
13493
+ wisconsin: "WI",
13494
+ wyoming: "WY"
13495
+ };
13496
+ US_CITY_CENTER_ZIPS = {
13497
+ "atlanta|GA": "30303",
13498
+ "austin|TX": "78701",
13499
+ "baltimore|MD": "21201",
13500
+ "boston|MA": "02108",
13501
+ "boulder|CO": "80302",
13502
+ "charlotte|NC": "28202",
13503
+ "chicago|IL": "60601",
13504
+ "colorado_springs|CO": "80903",
13505
+ "columbus|OH": "43215",
13506
+ "dallas|TX": "75201",
13507
+ "denver|CO": "80202",
13508
+ "detroit|MI": "48226",
13509
+ "fort_collins|CO": "80524",
13510
+ "fort_worth|TX": "76102",
13511
+ "houston|TX": "77002",
13512
+ "indianapolis|IN": "46204",
13513
+ "jacksonville|FL": "32202",
13514
+ "las_vegas|NV": "89101",
13515
+ "los_angeles|CA": "90012",
13516
+ "louisville|KY": "40202",
13517
+ "loveland|CO": "80537",
13518
+ "memphis|TN": "38103",
13519
+ "miami|FL": "33131",
13520
+ "minneapolis|MN": "55401",
13521
+ "nashville|TN": "37203",
13522
+ "new_york|NY": "10001",
13523
+ "orlando|FL": "32801",
13524
+ "philadelphia|PA": "19103",
13525
+ "phoenix|AZ": "85004",
13526
+ "portland|OR": "97205",
13527
+ "raleigh|NC": "27601",
13528
+ "richmond|VA": "23219",
13529
+ "sacramento|CA": "95814",
13530
+ "salt_lake_city|UT": "84101",
13531
+ "san_antonio|TX": "78205",
13532
+ "san_diego|CA": "92101",
13533
+ "san_francisco|CA": "94103",
13534
+ "san_jose|CA": "95113",
13535
+ "seattle|WA": "98101"
13536
+ };
13537
+ }
13538
+ });
13539
+
12896
13540
  // src/youtube/CaptionFetcher.ts
12897
13541
  async function fetchViaYoutubeTranscript(videoId) {
12898
13542
  try {
@@ -12905,6 +13549,11 @@ async function fetchViaYoutubeTranscript(videoId) {
12905
13549
  }));
12906
13550
  const text = chunks.map((c) => c.text).join(" ");
12907
13551
  const last = chunks[chunks.length - 1];
13552
+ const now = Date.now();
13553
+ await runWithCostContext(
13554
+ { ...currentCostContext(), op: "yt_transcription", subOp: "youtube-transcript" },
13555
+ () => recordKernelSession({ kernelSessionId: null, source: "yt_caption_tier", headlessSent: null, openedAtMs: now, closedAtMs: now })
13556
+ );
12908
13557
  return { videoId, text, chunks, durationMs: last ? last.timestamp[1] * 1e3 : 0, method: "youtube-transcript" };
12909
13558
  } catch {
12910
13559
  return null;
@@ -12956,162 +13605,214 @@ function parseTimedtextXml(xml) {
12956
13605
  }
12957
13606
  return results;
12958
13607
  }
12959
- async function fetchViaKernelInnertube(videoId) {
12960
- const kernelApiKey = browserServiceApiKey();
12961
- if (!kernelApiKey) return null;
12962
- const kernelProxyId = browserServiceProxyId();
12963
- const driver = new BrowserDriver();
12964
- const start = Date.now();
12965
- try {
12966
- await driver.launch({ kernelApiKey, kernelProxyId, headless: true, viewport: { width: 1280, height: 800 }, locale: "en-US" });
12967
- const page = driver.getPage();
12968
- await driver.navigateTo(`https://www.youtube.com/watch?v=${videoId}`);
12969
- await page.waitForFunction(
12970
- () => !!window.ytcfg,
12971
- { timeout: 2e4 }
12972
- );
12973
- const xml = await page.evaluate(async (vid) => {
12974
- const ANDROID_VERSION = "20.10.38";
12975
- const resp = await fetch("/youtubei/v1/player?prettyPrint=false", {
12976
- method: "POST",
12977
- headers: {
12978
- "Content-Type": "application/json",
12979
- "User-Agent": `com.google.android.youtube/${ANDROID_VERSION} (Linux; U; Android 14)`
12980
- },
12981
- body: JSON.stringify({
12982
- context: { client: { clientName: "ANDROID", clientVersion: ANDROID_VERSION } },
12983
- videoId: vid
12984
- })
13608
+ function isBotBlockStatus(status) {
13609
+ return status === "LOGIN_REQUIRED" || status === "AGE_VERIFICATION_REQUIRED";
13610
+ }
13611
+ async function probePlayer(page, videoId) {
13612
+ return page.evaluate(
13613
+ async ({ vid, clients }) => {
13614
+ let lastStatus = null;
13615
+ for (const client2 of clients) {
13616
+ const resp = await fetch("/youtubei/v1/player?prettyPrint=false", {
13617
+ method: "POST",
13618
+ headers: { "Content-Type": "application/json", "User-Agent": client2.userAgent },
13619
+ body: JSON.stringify({
13620
+ context: { client: { clientName: client2.name, clientVersion: client2.version } },
13621
+ videoId: vid
13622
+ })
13623
+ }).catch(() => null);
13624
+ if (!resp?.ok) {
13625
+ lastStatus = resp ? `player_http_${resp.status}` : "player_fetch_failed";
13626
+ continue;
13627
+ }
13628
+ const data = await resp.json();
13629
+ lastStatus = data?.playabilityStatus?.status ?? null;
13630
+ const audioFormats = (data?.streamingData?.adaptiveFormats ?? []).filter((f) => typeof f.url === "string" && f.url && typeof f.mimeType === "string" && f.mimeType.startsWith("audio/")).sort((a, b) => (Number(a.contentLength) || Infinity) - (Number(b.contentLength) || Infinity));
13631
+ const pick = audioFormats[0] ?? null;
13632
+ let captions = null;
13633
+ const tracks = data?.captions?.playerCaptionsTracklistRenderer?.captionTracks ?? [];
13634
+ if (tracks.length > 0) {
13635
+ const track = tracks.find((t) => t.languageCode === "en") ?? tracks[0];
13636
+ const sep = track.baseUrl.includes("?") ? "&" : "?";
13637
+ const xmlResp = await fetch(`${track.baseUrl}${sep}fmt=json3`).catch(() => null);
13638
+ if (xmlResp?.ok) captions = await xmlResp.text();
13639
+ }
13640
+ if (!captions && !pick) continue;
13641
+ return {
13642
+ captions,
13643
+ status: lastStatus,
13644
+ audio: pick ? {
13645
+ url: pick.url,
13646
+ mimeType: pick.mimeType,
13647
+ bitrate: pick.bitrate ?? null,
13648
+ contentLength: pick.contentLength ? Number(pick.contentLength) || null : null,
13649
+ approxDurationMs: pick.approxDurationMs ? Number(pick.approxDurationMs) || null : null
13650
+ } : null
13651
+ };
13652
+ }
13653
+ return { captions: null, audio: null, status: lastStatus };
13654
+ },
13655
+ { vid: videoId, clients: INNERTUBE_CLIENTS }
13656
+ ).catch(() => null);
13657
+ }
13658
+ async function downloadWholeAudioStreamInOneRequestBecauseYouTubeAllowsOnlyOnePerIp(page, audio) {
13659
+ if (audio.contentLength && audio.contentLength > MAX_AUDIO_BYTES) {
13660
+ console.warn(`[CaptionFetcher] audio stream ${audio.contentLength} bytes exceeds ${MAX_AUDIO_BYTES} byte cap`);
13661
+ return null;
13662
+ }
13663
+ const sep = audio.url.includes("?") ? "&" : "?";
13664
+ const url = audio.contentLength ? `${audio.url}${sep}range=0-${audio.contentLength - 1}` : audio.url;
13665
+ const result = await page.evaluate(async (u) => {
13666
+ try {
13667
+ const resp = await fetch(u);
13668
+ if (!resp.ok) return { ok: false, error: `http_${resp.status}` };
13669
+ const blob = await resp.blob();
13670
+ const dataUrl = await new Promise((resolve, reject) => {
13671
+ const reader = new FileReader();
13672
+ reader.onload = () => resolve(reader.result);
13673
+ reader.onerror = () => reject(reader.error);
13674
+ reader.readAsDataURL(blob);
12985
13675
  });
12986
- if (!resp.ok) return null;
12987
- const data = await resp.json();
12988
- const tracks = data?.captions?.playerCaptionsTracklistRenderer?.captionTracks ?? [];
12989
- if (tracks.length === 0) return null;
12990
- const track = tracks.find((t) => t.languageCode === "en") ?? tracks[0];
12991
- const sep = track.baseUrl.includes("?") ? "&" : "?";
12992
- const xmlResp = await fetch(`${track.baseUrl}${sep}fmt=json3`);
12993
- return xmlResp.ok ? xmlResp.text() : null;
12994
- }, videoId);
12995
- if (!xml) return null;
12996
- let entries = parseTimedtextJson3(xml);
12997
- if (entries.length === 0) entries = parseTimedtextXml(xml);
12998
- if (entries.length === 0) return null;
12999
- const chunks = entries.map((e) => ({
13000
- timestamp: [e.start / 1e3, (e.start + e.dur) / 1e3],
13001
- text: e.text
13002
- }));
13003
- const text = chunks.map((c) => c.text).join(" ");
13004
- const last = chunks[chunks.length - 1];
13005
- return { videoId, text, chunks, durationMs: Date.now() - start, method: "browser-innertube" };
13006
- } catch {
13676
+ return { ok: true, base64: dataUrl.slice(dataUrl.indexOf(",") + 1) };
13677
+ } catch (e) {
13678
+ return { ok: false, error: String(e) };
13679
+ }
13680
+ }, url).catch((e) => ({ ok: false, error: String(e) }));
13681
+ if (!result.ok || !result.base64) {
13682
+ console.warn(`[CaptionFetcher] audio fetch failed (${result.error ?? "unknown"}) for a ${audio.contentLength ?? 0} byte stream`);
13683
+ return null;
13684
+ }
13685
+ const bytes = Buffer.from(result.base64, "base64");
13686
+ if (audio.contentLength && bytes.length < audio.contentLength) {
13687
+ console.warn(`[CaptionFetcher] partial audio: ${bytes.length} of ${audio.contentLength} bytes`);
13007
13688
  return null;
13008
- } finally {
13009
- await driver.close();
13010
13689
  }
13690
+ return { bytes, mimeType: audio.mimeType.split(";")[0].trim() };
13011
13691
  }
13012
- async function captureAudioViaMediaRecorder(page) {
13013
- const result = await page.evaluate((recordSecs) => {
13014
- return new Promise((resolve) => {
13015
- const video = document.querySelector("video");
13016
- if (!video) {
13017
- resolve({ ok: false, error: "no video element" });
13018
- return;
13019
- }
13020
- video.muted = false;
13021
- video.volume = 1;
13022
- try {
13023
- const ctx = new AudioContext();
13024
- const src = ctx.createMediaElementSource(video);
13025
- const dest = ctx.createMediaStreamDestination();
13026
- src.connect(dest);
13027
- src.connect(ctx.destination);
13028
- const mimeType = MediaRecorder.isTypeSupported("audio/webm;codecs=opus") ? "audio/webm;codecs=opus" : MediaRecorder.isTypeSupported("audio/webm") ? "audio/webm" : "audio/ogg";
13029
- const recorder = new MediaRecorder(dest.stream, { mimeType, audioBitsPerSecond: 48e3 });
13030
- const chunks = [];
13031
- recorder.ondataavailable = (e) => {
13032
- if (e.data.size > 0) chunks.push(e.data);
13033
- };
13034
- recorder.onstop = async () => {
13035
- const blob = new Blob(chunks, { type: mimeType });
13036
- const ab = await blob.arrayBuffer();
13037
- const arr = new Uint8Array(ab);
13038
- let binary = "";
13039
- const chunkSize = 8192;
13040
- for (let i = 0; i < arr.length; i += chunkSize) {
13041
- binary += String.fromCharCode(...arr.slice(i, Math.min(i + chunkSize, arr.length)));
13042
- }
13043
- resolve({ ok: true, base64: btoa(binary), size: ab.byteLength, mimeType });
13044
- };
13045
- recorder.start(1e3);
13046
- video.playbackRate = 2;
13047
- video.play().catch(() => void 0);
13048
- setTimeout(() => {
13049
- recorder.stop();
13050
- ctx.close();
13051
- }, recordSecs * 1e3);
13052
- } catch (e) {
13053
- resolve({ ok: false, error: String(e) });
13054
- }
13055
- });
13056
- }, WHISPER_RECORD_SECONDS);
13057
- if (!result.ok || !result.base64 || !result.mimeType) return null;
13058
- return { bytes: Buffer.from(result.base64, "base64"), mimeType: result.mimeType };
13059
- }
13060
- async function attemptKernelWhisper(videoId, kernelApiKey, kernelProxyId, falKey, start) {
13061
- const driver = new BrowserDriver();
13692
+ async function transcribeWithWizper(videoId, audio, startMs) {
13693
+ const falKey = process.env.FAL_KEY;
13694
+ if (!falKey) return null;
13062
13695
  try {
13063
- await driver.launch({ kernelApiKey, kernelProxyId, headless: true, viewport: { width: 1280, height: 800 }, locale: "en-US" });
13064
- const page = driver.getPage();
13065
- await driver.navigateTo(`https://www.youtube.com/watch?v=${videoId}`);
13066
- await page.waitForFunction(
13067
- () => !!window.ytcfg,
13068
- { timeout: 2e4 }
13069
- );
13070
- const playStatus = await page.evaluate(() => {
13071
- const w = window;
13072
- return w.ytInitialPlayerResponse?.playabilityStatus?.status;
13073
- });
13074
- if (playStatus !== "OK") {
13075
- await driver.close();
13076
- return null;
13077
- }
13078
- const audioInfo = await captureAudioViaMediaRecorder(page);
13079
- await driver.close();
13080
- if (!audioInfo || audioInfo.bytes.length < 1e4) return null;
13081
13696
  import_client3.fal.config({ credentials: falKey });
13082
- const ext = audioInfo.mimeType.includes("ogg") ? "ogg" : "webm";
13083
- const audioFile = new File([new Uint8Array(audioInfo.bytes)], `audio.${ext}`, { type: audioInfo.mimeType });
13697
+ const ext = audio.mimeType.includes("webm") ? "webm" : audio.mimeType.includes("mp4") ? "m4a" : "audio";
13698
+ const audioFile = new File([new Uint8Array(audio.bytes)], `audio.${ext}`, { type: audio.mimeType });
13084
13699
  const uploadedUrl = await import_client3.fal.storage.upload(audioFile);
13700
+ const subscribeStartMs = Date.now();
13085
13701
  const result = await import_client3.fal.subscribe("fal-ai/wizper", {
13086
- input: { audio_url: uploadedUrl, task: "transcribe", language: "en" },
13702
+ input: { audio_url: uploadedUrl, task: "transcribe", language: "en", chunk_level: "segment" },
13087
13703
  logs: false,
13088
13704
  pollInterval: 3e3
13089
13705
  });
13706
+ const subscribeDurationMs = Date.now() - subscribeStartMs;
13090
13707
  const data = result.data;
13091
13708
  const chunks = (data.chunks ?? []).map((c) => ({
13092
13709
  timestamp: c.timestamp,
13093
13710
  text: c.text.trim()
13094
13711
  }));
13095
- const audioSec = chunks.length ? chunks[chunks.length - 1]?.timestamp?.[1] ?? 0 : 0;
13096
- void recordVendorUsage({ vendor: "fal_wizper", model: "fal-ai/wizper", units: audioSec / 60, unitType: "audio_min" });
13712
+ void recordVendorUsage({ vendor: "fal_wizper", model: "fal-ai/wizper", units: subscribeDurationMs / 1e3, unitType: "compute_sec" });
13097
13713
  const text = data.text ?? chunks.map((c) => c.text).join(" ");
13098
13714
  if (!text) return null;
13099
- return { videoId, text, chunks, durationMs: Date.now() - start, method: "browser-whisper" };
13100
- } catch {
13101
- await driver.close();
13715
+ return { videoId, text, chunks, durationMs: Date.now() - startMs, method: "browser-whisper" };
13716
+ } catch (err) {
13717
+ console.warn(`[CaptionFetcher] wizper transcription failed for ${videoId}: ${err instanceof Error ? err.message : String(err)}`);
13102
13718
  return null;
13103
13719
  }
13104
13720
  }
13105
- async function fetchViaKernelWhisper(videoId) {
13721
+ async function resolveResidentialProxy(kernelApiKey) {
13722
+ try {
13723
+ const resolved = await resolveKernelProxyId({
13724
+ kernelApiKey,
13725
+ proxyMode: "location",
13726
+ location: YT_RESIDENTIAL_LOCATION,
13727
+ gl: "us",
13728
+ fresh: true
13729
+ });
13730
+ return { kernelProxyId: resolved.kernelProxyId, disposableProxyId: resolved.disposableProxyId };
13731
+ } catch (err) {
13732
+ console.warn(`[CaptionFetcher] residential proxy resolution failed: ${err instanceof Error ? err.message : String(err)}`);
13733
+ return {};
13734
+ }
13735
+ }
13736
+ async function probeAndExtract(videoId, opts) {
13106
13737
  const kernelApiKey = browserServiceApiKey();
13107
- const kernelProxyId = browserServiceProxyId();
13108
- const falKey = process.env.FAL_KEY;
13109
- if (!kernelApiKey || !falKey) return null;
13738
+ if (!kernelApiKey) return null;
13739
+ const kernelProxyId = opts?.kernelProxyId ?? browserServiceProxyId();
13740
+ const driver = new BrowserDriver();
13741
+ return runWithCostContext({ ...currentCostContext(), op: "yt_transcription", subOp: "browser-innertube" }, async () => {
13742
+ let entries = [];
13743
+ let audio = null;
13744
+ let retryWithFreshIp = false;
13745
+ try {
13746
+ await driver.launch({ kernelApiKey, kernelProxyId, headless: true, headlessMode: "headless", viewport: { width: 1280, height: 800 }, locale: "en-US" });
13747
+ const page = driver.getPage();
13748
+ await driver.navigateTo(`https://www.youtube.com/watch?v=${videoId}`);
13749
+ await page.waitForFunction(
13750
+ () => !!window.ytcfg,
13751
+ { timeout: 2e4 }
13752
+ );
13753
+ const probe = await probePlayer(page, videoId);
13754
+ if (!probe) return null;
13755
+ if (probe.captions && !opts?.skipCaptions) {
13756
+ entries = parseTimedtextJson3(probe.captions);
13757
+ if (entries.length === 0) entries = parseTimedtextXml(probe.captions);
13758
+ }
13759
+ if (entries.length === 0) {
13760
+ if (!probe.audio) {
13761
+ retryWithFreshIp = isBotBlockStatus(probe.status);
13762
+ console.warn(`[CaptionFetcher] no captions and no audio stream for ${videoId} (playability=${probe.status ?? "unknown"}, proxied=${Boolean(kernelProxyId)})`);
13763
+ return { entries, audio: null, retryWithFreshIp };
13764
+ }
13765
+ audio = await downloadWholeAudioStreamInOneRequestBecauseYouTubeAllowsOnlyOnePerIp(page, probe.audio);
13766
+ if (!audio) return { entries, audio: null, retryWithFreshIp: true };
13767
+ }
13768
+ } catch (err) {
13769
+ console.warn(`[CaptionFetcher] kernel session failed for ${videoId}: ${err instanceof Error ? err.message : String(err)}`);
13770
+ return null;
13771
+ } finally {
13772
+ const servedSubOp = audio ? "browser-whisper" : "browser-innertube";
13773
+ await runWithCostContext({ ...currentCostContext(), op: "yt_transcription", subOp: servedSubOp }, () => driver.close());
13774
+ }
13775
+ return { entries, audio, retryWithFreshIp };
13776
+ });
13777
+ }
13778
+ async function extractWithProxyEscalation(videoId, opts) {
13779
+ const direct = await probeAndExtract(videoId, opts);
13780
+ if (direct && (direct.entries.length > 0 || direct.audio)) return direct;
13781
+ const shouldEscalate = direct === null || direct.retryWithFreshIp;
13782
+ if (!shouldEscalate) return direct;
13783
+ const kernelApiKey = browserServiceApiKey();
13784
+ if (!kernelApiKey) return direct;
13785
+ const { kernelProxyId, disposableProxyId } = await resolveResidentialProxy(kernelApiKey);
13786
+ if (!kernelProxyId) return direct;
13787
+ try {
13788
+ console.warn(`[CaptionFetcher] ${videoId}: retrying through residential proxy after bot block`);
13789
+ const viaProxy = await probeAndExtract(videoId, { ...opts, kernelProxyId });
13790
+ return viaProxy ?? direct;
13791
+ } finally {
13792
+ if (disposableProxyId) {
13793
+ await deleteKernelProxyId(kernelApiKey, disposableProxyId).catch(
13794
+ (err) => console.warn(`[CaptionFetcher] failed to delete disposable proxy: ${err instanceof Error ? err.message : String(err)}`)
13795
+ );
13796
+ }
13797
+ }
13798
+ }
13799
+ async function fetchViaKernelSession(videoId) {
13110
13800
  const start = Date.now();
13111
- for (let attempt = 0; attempt < 4; attempt++) {
13112
- if (attempt > 0) await new Promise((r) => setTimeout(r, 2e3 * attempt));
13113
- const result = await attemptKernelWhisper(videoId, kernelApiKey, kernelProxyId, falKey, start);
13114
- if (result) return result;
13801
+ const outcome = await extractWithProxyEscalation(videoId);
13802
+ if (!outcome) return null;
13803
+ if (outcome.entries.length > 0) {
13804
+ const chunks = outcome.entries.map((e) => ({
13805
+ timestamp: [e.start / 1e3, (e.start + e.dur) / 1e3],
13806
+ text: e.text
13807
+ }));
13808
+ const text = chunks.map((c) => c.text).join(" ");
13809
+ return { videoId, text, chunks, durationMs: Date.now() - start, method: "browser-innertube" };
13810
+ }
13811
+ if (outcome.audio) {
13812
+ return runWithCostContext(
13813
+ { ...currentCostContext(), op: "yt_transcription", subOp: "browser-whisper" },
13814
+ () => transcribeWithWizper(videoId, outcome.audio, start)
13815
+ );
13115
13816
  }
13116
13817
  return null;
13117
13818
  }
@@ -13133,21 +13834,29 @@ function finalizeCaptions(result) {
13133
13834
  async function fetchCaptions(videoId) {
13134
13835
  const primary = await fetchViaYoutubeTranscript(videoId);
13135
13836
  if (primary) return finalizeCaptions(primary);
13136
- const kernel = await fetchViaKernelInnertube(videoId);
13137
- if (kernel) return finalizeCaptions(kernel);
13138
- const whisper = await fetchViaKernelWhisper(videoId);
13139
- if (whisper) return finalizeCaptions(whisper);
13140
- throw new Error(`No captions available for ${videoId} \u2014 all three tiers failed`);
13837
+ const result = await fetchViaKernelSession(videoId);
13838
+ if (result) return finalizeCaptions(result);
13839
+ throw new Error(
13840
+ `Could not transcribe ${videoId}. It has no caption track, and its audio stream could not be downloaded \u2014 YouTube permits a single media request per IP, so streams that do not fit in one response are unreachable. Captioned videos of any length still work.`
13841
+ );
13141
13842
  }
13142
- var import_client3, WHISPER_RECORD_SECONDS;
13843
+ var import_client3, INNERTUBE_CLIENTS, MAX_AUDIO_BYTES, YT_RESIDENTIAL_LOCATION;
13143
13844
  var init_CaptionFetcher = __esm({
13144
13845
  "src/youtube/CaptionFetcher.ts"() {
13145
13846
  "use strict";
13146
13847
  init_BrowserDriver();
13147
13848
  init_browser_service_env();
13849
+ init_kernel_proxy_resolver();
13148
13850
  import_client3 = require("@fal-ai/client");
13149
13851
  init_cost_telemetry();
13150
- WHISPER_RECORD_SECONDS = 90;
13852
+ init_cost_context();
13853
+ INNERTUBE_CLIENTS = [
13854
+ { name: "ANDROID", version: "20.10.38", userAgent: "com.google.android.youtube/20.10.38 (Linux; U; Android 14)" },
13855
+ { name: "IOS", version: "20.10.4", userAgent: "com.google.ios.youtube/20.10.4 (iPhone16,2; U; CPU iOS 18_3_2 like Mac OS X;)" },
13856
+ { name: "ANDROID_VR", version: "1.62.27", userAgent: "com.google.android.apps.youtube.vr.oculus/1.62.27 (Linux; U; Android 12L) gzip" }
13857
+ ];
13858
+ MAX_AUDIO_BYTES = 96 * 1024 * 1024;
13859
+ YT_RESIDENTIAL_LOCATION = process.env.YOUTUBE_PROXY_LOCATION ?? "Austin, Texas";
13151
13860
  }
13152
13861
  });
13153
13862
 
@@ -13483,6 +14192,7 @@ var init_youtube_routes = __esm({
13483
14192
  init_api_auth();
13484
14193
  init_server_schemas();
13485
14194
  init_concurrency_gates();
14195
+ init_cost_context();
13486
14196
  youtubeApp = new import_hono3.Hono();
13487
14197
  youtubeApp.post("/harvest", createApiKeyAuth(), async (c) => {
13488
14198
  const raw = await c.req.json().catch(() => ({}));
@@ -13560,7 +14270,7 @@ var init_youtube_routes = __esm({
13560
14270
  const { ok: trOk, balance_mc: trBal } = await debitMc(user.id, holdMc, LedgerOperation.TRANSCRIPTION_HOLD, id);
13561
14271
  if (!trOk) return c.json(insufficientBalanceResponse(trBal, holdMc), 402);
13562
14272
  debited = true;
13563
- const result = await fetchCaptions(id);
14273
+ const result = await runWithCostContext({ ...currentCostContext(), op: "yt_transcription", userId: user.id }, () => fetchCaptions(id));
13564
14274
  const lastChunk = result.chunks?.[result.chunks.length - 1];
13565
14275
  const chunkSecs = lastChunk && Number.isFinite(lastChunk.timestamp[1]) ? lastChunk.timestamp[1] : 0;
13566
14276
  const durationSecs = Number.isFinite(result.durationMs) ? result.durationMs / 1e3 : 0;
@@ -14409,727 +15119,170 @@ async function collectAdLibraryResults(page, url, maxResults, opts = {}) {
14409
15119
  try {
14410
15120
  await page.goto(url, { waitUntil: "domcontentloaded", timeout: 45e3 });
14411
15121
  const deadline = Date.now() + captureMs;
14412
- let lastCount = -1;
14413
- let stableRounds = 0;
14414
- while (Date.now() < deadline && collected.length < maxResults) {
14415
- await page.evaluate(() => window.scrollTo(0, document.body.scrollHeight)).catch(() => void 0);
14416
- await page.waitForTimeout(2e3);
14417
- if (collected.length === lastCount) {
14418
- stableRounds++;
14419
- if (stableRounds >= 2 && collected.length > 0) break;
14420
- } else {
14421
- stableRounds = 0;
14422
- }
14423
- lastCount = collected.length;
14424
- }
14425
- } finally {
14426
- page.off("response", handler);
14427
- }
14428
- return collected.slice(0, maxResults);
14429
- }
14430
- function advertisersFromResults(results, maxResults) {
14431
- const byPage = /* @__PURE__ */ new Map();
14432
- for (const r of results) {
14433
- if (!r.page_id || !r.page_name) continue;
14434
- const collation = typeof r.collation_count === "number" && r.collation_count > 0 ? r.collation_count : 0;
14435
- const existing = byPage.get(r.page_id);
14436
- if (existing) {
14437
- existing.resultCount++;
14438
- existing.maxCollation = Math.max(existing.maxCollation, collation);
14439
- } else {
14440
- byPage.set(r.page_id, { pageName: r.page_name, pageId: r.page_id, sampleLibraryId: r.ad_archive_id, maxCollation: collation, resultCount: 1 });
14441
- }
14442
- }
14443
- return [...byPage.values()].map((e) => ({ pageName: e.pageName, pageId: e.pageId, sampleLibraryId: e.sampleLibraryId, adCount: Math.max(e.maxCollation, e.resultCount) })).sort((a, b) => b.adCount - a.adCount).slice(0, maxResults);
14444
- }
14445
- var AD_LIBRARY_QUERY;
14446
- var init_FacebookAdGraphql = __esm({
14447
- "src/extractor/FacebookAdGraphql.ts"() {
14448
- "use strict";
14449
- AD_LIBRARY_QUERY = "AdLibrarySearchPaginationQuery";
14450
- }
14451
- });
14452
-
14453
- // src/extractor/FacebookOrganicVideoExtractor.ts
14454
- function htmlDecode(value) {
14455
- return value.replace(/&amp;/g, "&").replace(/&quot;/g, '"').replace(/&#039;/g, "'").replace(/&lt;/g, "<").replace(/&gt;/g, ">");
14456
- }
14457
- function decodeEscapedValue(raw) {
14458
- let out = raw;
14459
- try {
14460
- out = JSON.parse(`"${raw.replace(/"/g, '\\"')}"`);
14461
- } catch {
14462
- out = raw.replace(/\\u0025/g, "%").replace(/\\u0026/g, "&").replace(/\\u003D/g, "=").replace(/\\u003d/g, "=").replace(/\\"/g, '"').replace(/\\\//g, "/");
14463
- }
14464
- return htmlDecode(out);
14465
- }
14466
- function facebookVideoIdFromUrl(url) {
14467
- try {
14468
- const parsed = new URL(url);
14469
- const path6 = parsed.pathname;
14470
- const direct = path6.match(/\/(?:reel|videos|watch)\/(?:[^/]+\/)?(\d{8,})/i);
14471
- if (direct) return direct[1];
14472
- const queryVideoId = parsed.searchParams.get("v") ?? parsed.searchParams.get("video_id");
14473
- if (queryVideoId && /^\d{8,}$/.test(queryVideoId)) return queryVideoId;
14474
- const numericTail = path6.match(/\/(\d{8,})\/?$/);
14475
- if (numericTail) return numericTail[1];
14476
- } catch {
14477
- }
14478
- return null;
14479
- }
14480
- function parseEfg(url) {
14481
- try {
14482
- const efg = new URL(url).searchParams.get("efg");
14483
- if (!efg) return null;
14484
- return JSON.parse(Buffer.from(efg, "base64").toString("utf8"));
14485
- } catch {
14486
- return null;
14487
- }
14488
- }
14489
- function manifestDurationSec(htmlWindow) {
14490
- const decoded = decodeEscapedValue(htmlWindow);
14491
- const match = decoded.match(/mediaPresentationDuration="PT([\d.]+)S"/);
14492
- return match ? Number(match[1]) : null;
14493
- }
14494
- function metadataContent(html, key) {
14495
- const escaped = key.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
14496
- const re = new RegExp(`<meta\\s+[^>]*(?:property|name)=["']${escaped}["'][^>]*content=["']([^"']+)["'][^>]*>`, "i");
14497
- const m = html.match(re);
14498
- return m ? htmlDecode(m[1]).trim() : null;
14499
- }
14500
- function ownerNameFromHtml(html) {
14501
- const ogTitle = metadataContent(html, "og:title");
14502
- const pipe = ogTitle?.split("|").map((s) => s.trim()).filter(Boolean);
14503
- if (pipe && pipe.length > 1) return pipe[pipe.length - 1] ?? null;
14504
- return null;
14505
- }
14506
- function extractFacebookOrganicVideo(html, pageUrl, sourceUrl = pageUrl, quality = "best") {
14507
- const videoId = facebookVideoIdFromUrl(pageUrl) ?? facebookVideoIdFromUrl(sourceUrl);
14508
- const manifestIndex = videoId ? html.indexOf(`dash_mpd_debug.mpd?v=${videoId}`) : -1;
14509
- const searchWindow = manifestIndex >= 0 ? html.slice(Math.max(0, manifestIndex - 5e4), Math.min(html.length, manifestIndex + 5e4)) : html;
14510
- const targetDuration = manifestDurationSec(searchWindow);
14511
- const candidates = [];
14512
- const seen = /* @__PURE__ */ new Set();
14513
- const re = /browser_native_(sd|hd)_url\\?"\s*:\s*\\?"([^"<]+?)\\?"/g;
14514
- let match;
14515
- while ((match = re.exec(searchWindow)) !== null) {
14516
- const url = decodeEscapedValue(match[2]);
14517
- if (!url || seen.has(url)) continue;
14518
- seen.add(url);
14519
- const parsed = parseEfg(url);
14520
- let bitrate = null;
14521
- try {
14522
- const value = new URL(url).searchParams.get("bitrate");
14523
- bitrate = value ? Number(value) : null;
14524
- } catch {
14525
- }
14526
- candidates.push({
14527
- url,
14528
- quality: match[1],
14529
- bitrate: Number.isFinite(bitrate) ? bitrate : null,
14530
- durationSec: typeof parsed?.duration_s === "number" ? parsed.duration_s : null,
14531
- assetId: typeof parsed?.xpv_asset_id === "number" ? parsed.xpv_asset_id : null,
14532
- vencodeTag: typeof parsed?.vencode_tag === "string" ? parsed.vencode_tag : null
14533
- });
14534
- }
14535
- const targetCandidates = candidates.filter((candidate) => {
14536
- if (!targetDuration || candidate.durationSec == null) return true;
14537
- return Math.abs(candidate.durationSec - targetDuration) <= 2;
14538
- });
14539
- const selectable = targetCandidates.length ? targetCandidates : candidates;
14540
- const selected = [...selectable].filter((candidate) => quality === "best" || candidate.quality === quality).sort((a, b) => {
14541
- const aq = a.quality === "hd" ? 1 : 0;
14542
- const bq = b.quality === "hd" ? 1 : 0;
14543
- if (quality === "best" && aq !== bq) return bq - aq;
14544
- return (b.bitrate ?? 0) - (a.bitrate ?? 0);
14545
- })[0];
14546
- if (!selected) {
14547
- throw new Error("No progressive Facebook video URL was found in the rendered page state");
14548
- }
14549
- return {
14550
- sourceUrl,
14551
- pageUrl,
14552
- videoId,
14553
- title: metadataContent(html, "og:title") ?? metadataContent(html, "twitter:title"),
14554
- description: metadataContent(html, "og:description") ?? metadataContent(html, "description") ?? metadataContent(html, "twitter:description"),
14555
- ownerName: ownerNameFromHtml(html),
14556
- videoUrl: selected.url,
14557
- selectedQuality: selected.quality,
14558
- bitrate: selected.bitrate,
14559
- durationSec: targetDuration ?? selected.durationSec,
14560
- assetId: selected.assetId,
14561
- vencodeTag: selected.vencodeTag,
14562
- candidates: selectable,
14563
- extractedAt: (/* @__PURE__ */ new Date()).toISOString()
14564
- };
14565
- }
14566
- async function extractFacebookOrganicVideoFromPage(page, sourceUrl, quality = "best") {
14567
- await page.waitForLoadState("domcontentloaded", { timeout: 15e3 }).catch(() => {
14568
- });
14569
- await page.waitForTimeout(1500);
14570
- const html = await page.content();
14571
- return extractFacebookOrganicVideo(html, page.url(), sourceUrl, quality);
14572
- }
14573
- var init_FacebookOrganicVideoExtractor = __esm({
14574
- "src/extractor/FacebookOrganicVideoExtractor.ts"() {
14575
- "use strict";
14576
- }
14577
- });
14578
-
14579
- // src/locations.ts
14580
- var LOCATIONS;
14581
- var init_locations = __esm({
14582
- "src/locations.ts"() {
14583
- "use strict";
14584
- LOCATIONS = {
14585
- "austin": "Austin,Texas,United States",
14586
- "new york": "New York,New York,United States",
14587
- "new york city": "New York,New York,United States",
14588
- "nyc": "New York,New York,United States",
14589
- "los angeles": "Los Angeles,California,United States",
14590
- "la": "Los Angeles,California,United States",
14591
- "chicago": "Chicago,Illinois,United States",
14592
- "houston": "Houston,Texas,United States",
14593
- "phoenix": "Phoenix,Arizona,United States",
14594
- "philadelphia": "Philadelphia,Pennsylvania,United States",
14595
- "philly": "Philadelphia,Pennsylvania,United States",
14596
- "san antonio": "San Antonio,Texas,United States",
14597
- "dallas": "Dallas,Texas,United States",
14598
- "miami": "Miami,Florida,United States",
14599
- "seattle": "Seattle,Washington,United States",
14600
- "denver": "Denver,Colorado,United States",
14601
- "loveland": "Loveland,Colorado,United States",
14602
- "loveland co": "Loveland,Colorado,United States",
14603
- "fort collins": "Fort Collins,Colorado,United States",
14604
- "boulder": "Boulder,Colorado,United States",
14605
- "colorado springs": "Colorado Springs,Colorado,United States",
14606
- "boston": "Boston,Massachusetts,United States",
14607
- "atlanta": "Atlanta,Georgia,United States",
14608
- "san francisco": "San Francisco,California,United States",
14609
- "sf": "San Francisco,California,United States",
14610
- "portland": "Portland,Oregon,United States",
14611
- "las vegas": "Las Vegas,Nevada,United States",
14612
- "minneapolis": "Minneapolis,Minnesota,United States",
14613
- "detroit": "Detroit,Michigan,United States",
14614
- "nashville": "Nashville,Tennessee,United States",
14615
- "charlotte": "Charlotte,North Carolina,United States",
14616
- "orlando": "Orlando,Florida,United States",
14617
- "san diego": "San Diego,California,United States",
14618
- "baltimore": "Baltimore,Maryland,United States",
14619
- "sacramento": "Sacramento,California,United States",
14620
- "columbus": "Columbus,Ohio,United States",
14621
- "indianapolis": "Indianapolis,Indiana,United States",
14622
- "san jose": "San Jose,California,United States",
14623
- "fort worth": "Fort Worth,Texas,United States",
14624
- "jacksonville": "Jacksonville,Florida,United States",
14625
- "memphis": "Memphis,Tennessee,United States",
14626
- "louisville": "Louisville,Kentucky,United States",
14627
- "raleigh": "Raleigh,North Carolina,United States",
14628
- "richmond": "Richmond,Virginia,United States",
14629
- "salt lake city": "Salt Lake City,Utah,United States",
14630
- "toronto": "Toronto,Ontario,Canada",
14631
- "vancouver": "Vancouver,British Columbia,Canada",
14632
- "montreal": "Montreal,Quebec,Canada",
14633
- "calgary": "Calgary,Alberta,Canada",
14634
- "ottawa": "Ottawa,Ontario,Canada",
14635
- "london": "London,England,United Kingdom",
14636
- "manchester": "Manchester,England,United Kingdom",
14637
- "birmingham": "Birmingham,England,United Kingdom",
14638
- "edinburgh": "Edinburgh,Scotland,United Kingdom",
14639
- "glasgow": "Glasgow,Scotland,United Kingdom",
14640
- "leeds": "Leeds,England,United Kingdom",
14641
- "sydney": "Sydney,New South Wales,Australia",
14642
- "melbourne": "Melbourne,Victoria,Australia",
14643
- "brisbane": "Brisbane,Queensland,Australia",
14644
- "perth": "Perth,Western Australia,Australia",
14645
- "adelaide": "Adelaide,South Australia,Australia",
14646
- "dublin": "Dublin,Leinster,Ireland"
14647
- };
14648
- }
14649
- });
14650
-
14651
- // src/uule.ts
14652
- function encodeVarint(value) {
14653
- const bytes = [];
14654
- let remaining = value;
14655
- do {
14656
- let byte = remaining & 127;
14657
- remaining >>>= 7;
14658
- if (remaining > 0) byte |= 128;
14659
- bytes.push(byte);
14660
- } while (remaining > 0);
14661
- return bytes;
14662
- }
14663
- function encodeUule(name) {
14664
- const locationBytes = Buffer.from(name, "utf8");
14665
- const payload = Buffer.concat([
14666
- Buffer.from([8, 2, 16, 32, 34]),
14667
- Buffer.from(encodeVarint(locationBytes.length)),
14668
- locationBytes
14669
- ]);
14670
- return `w+${payload.toString("base64")}`;
14671
- }
14672
- function normalizeLocation(input) {
14673
- const raw = input.toLowerCase().trim();
14674
- if (LOCATIONS[raw]) return LOCATIONS[raw];
14675
- const beforeComma = raw.split(",")[0].trim();
14676
- if (beforeComma !== raw && LOCATIONS[beforeComma]) return LOCATIONS[beforeComma];
14677
- const withoutState = raw.replace(/\s+[a-z]{2}$/, "").trim();
14678
- if (withoutState !== raw && LOCATIONS[withoutState]) return LOCATIONS[withoutState];
14679
- return input;
14680
- }
14681
- var init_uule = __esm({
14682
- "src/uule.ts"() {
14683
- "use strict";
14684
- init_locations();
14685
- }
14686
- });
14687
-
14688
- // src/kernel-proxy-resolver.ts
14689
- function proxyIdSuffix2(proxyId) {
14690
- return proxyId ? proxyId.slice(-6) : null;
14691
- }
14692
- function resolution(source, proxyMode, proxyId, target, error, disposable = false) {
14693
- const proxyType = source === "disabled" ? "none" : source === "configured_fallback" ? "configured" : source === "isp_created" ? "isp" : source === "location_created" || source === "location_reused" ? "residential" : "unknown";
14694
- return {
14695
- kernelProxyId: proxyId,
14696
- ...disposable && proxyId ? { disposableProxyId: proxyId } : {},
14697
- resolution: {
14698
- source,
14699
- proxyType,
14700
- proxyMode,
14701
- proxyIdPresent: Boolean(proxyId),
14702
- proxyIdSuffix: proxyIdSuffix2(proxyId),
14703
- target,
14704
- error,
14705
- disposable
14706
- }
14707
- };
14708
- }
14709
- function normalizeStateName(value) {
14710
- return value.trim().toLowerCase().replace(/\s+/g, " ");
14711
- }
14712
- function normalizeCountryName(value) {
14713
- return value.trim().toLowerCase().replace(/\./g, "").replace(/\s+/g, " ");
14714
- }
14715
- function isUnitedStates(country) {
14716
- if (!country) return true;
14717
- const normalized = normalizeCountryName(country);
14718
- return normalized === "united states" || normalized === "united states of america" || normalized === "usa" || normalized === "us";
14719
- }
14720
- function stateCodeFor(region) {
14721
- const trimmed = region.trim();
14722
- if (/^[A-Za-z]{2}$/.test(trimmed)) return trimmed.toUpperCase();
14723
- return US_STATE_CODES[normalizeStateName(trimmed)] ?? null;
14724
- }
14725
- function kernelCityIdentifierCandidates(city) {
14726
- const ascii = city.normalize("NFKD").replace(/[^\x00-\x7F]/g, "").toLowerCase();
14727
- const words = ascii.split(/[^a-z0-9]+/).filter(Boolean);
14728
- const underscored = words.join("_");
14729
- const compact = words.join("");
14730
- return Array.from(new Set([underscored, compact].filter(Boolean)));
14731
- }
14732
- function proxyName(country, state, city) {
14733
- return city ? `mcp-serp-residential-${country.toLowerCase()}-${state.toLowerCase()}-${city}` : `mcp-serp-residential-${country.toLowerCase()}-${state.toLowerCase()}`;
14734
- }
14735
- function freshProxyName(baseName, attemptIndex) {
14736
- const stamp = `${Date.now()}-${attemptIndex ?? 0}-${Math.random().toString(36).slice(2, 8)}`;
14737
- return `${baseName}-fresh-${stamp}`;
14738
- }
14739
- function zipProxyName(zip) {
14740
- return `mcp-serp-residential-us-zip-${zip}`;
14741
- }
14742
- function parseKernelLocationProxyTarget(location2, gl) {
14743
- if (!location2 || gl.toLowerCase() !== "us") return null;
14744
- const canonicalLocation = normalizeLocation(location2);
14745
- let parts = canonicalLocation.split(",").map((part) => part.trim()).filter(Boolean);
14746
- if (parts.length > 1 && isUnitedStates(parts[parts.length - 1])) {
14747
- parts = parts.slice(0, -1);
14748
- }
14749
- if (parts.length === 1) {
14750
- const stateOnly = stateCodeFor(parts[0]);
14751
- if (!stateOnly) return null;
14752
- return {
14753
- canonicalLocation,
14754
- level: "state",
14755
- country: "US",
14756
- state: stateOnly,
14757
- city: "",
14758
- cityCandidates: [],
14759
- proxyName: proxyName("US", stateOnly),
14760
- config: {
14761
- country: "US",
14762
- state: stateOnly
15122
+ let lastCount = -1;
15123
+ let stableRounds = 0;
15124
+ while (Date.now() < deadline && collected.length < maxResults) {
15125
+ await page.evaluate(() => window.scrollTo(0, document.body.scrollHeight)).catch(() => void 0);
15126
+ await page.waitForTimeout(2e3);
15127
+ if (collected.length === lastCount) {
15128
+ stableRounds++;
15129
+ if (stableRounds >= 2 && collected.length > 0) break;
15130
+ } else {
15131
+ stableRounds = 0;
14763
15132
  }
14764
- };
14765
- }
14766
- const [city = "", region = ""] = parts;
14767
- if (!city || !region) return null;
14768
- const state = stateCodeFor(region);
14769
- if (!state) return null;
14770
- const cityCandidates = kernelCityIdentifierCandidates(city);
14771
- const primaryCity = cityCandidates[0];
14772
- if (!primaryCity) return null;
14773
- return {
14774
- canonicalLocation,
14775
- level: "city",
14776
- country: "US",
14777
- state,
14778
- city: primaryCity,
14779
- cityCandidates,
14780
- proxyName: proxyName("US", state, primaryCity),
14781
- config: {
14782
- country: "US",
14783
- state,
14784
- city: primaryCity
15133
+ lastCount = collected.length;
14785
15134
  }
14786
- };
14787
- }
14788
- function cityZipKey(target) {
14789
- return `${target.city}|${target.state}`;
14790
- }
14791
- function knownZipFor(target, explicitZip) {
14792
- if (explicitZip && /^\d{5}$/.test(explicitZip)) return explicitZip;
14793
- return US_CITY_CENTER_ZIPS[cityZipKey(target)] ?? null;
15135
+ } finally {
15136
+ page.off("response", handler);
15137
+ }
15138
+ return collected.slice(0, maxResults);
14794
15139
  }
14795
- function zipTarget(target, zip) {
14796
- return {
14797
- ...target,
14798
- level: "zip",
14799
- zip,
14800
- proxyName: zipProxyName(zip),
14801
- config: {
14802
- country: target.country,
14803
- state: target.state,
14804
- zip
15140
+ function advertisersFromResults(results, maxResults) {
15141
+ const byPage = /* @__PURE__ */ new Map();
15142
+ for (const r of results) {
15143
+ if (!r.page_id || !r.page_name) continue;
15144
+ const collation = typeof r.collation_count === "number" && r.collation_count > 0 ? r.collation_count : 0;
15145
+ const existing = byPage.get(r.page_id);
15146
+ if (existing) {
15147
+ existing.resultCount++;
15148
+ existing.maxCollation = Math.max(existing.maxCollation, collation);
15149
+ } else {
15150
+ byPage.set(r.page_id, { pageName: r.page_name, pageId: r.page_id, sampleLibraryId: r.ad_archive_id, maxCollation: collation, resultCount: 1 });
14805
15151
  }
14806
- };
14807
- }
14808
- function withProxyName(target, name) {
14809
- return {
14810
- ...target,
14811
- proxyName: name
14812
- };
15152
+ }
15153
+ return [...byPage.values()].map((e) => ({ pageName: e.pageName, pageId: e.pageId, sampleLibraryId: e.sampleLibraryId, adCount: Math.max(e.maxCollation, e.resultCount) })).sort((a, b) => b.adCount - a.adCount).slice(0, maxResults);
14813
15154
  }
14814
- function configMatches(config, target, city) {
14815
- if (target.level === "zip") {
14816
- return config?.country?.toUpperCase() === target.country && config?.zip === target.zip;
15155
+ var AD_LIBRARY_QUERY;
15156
+ var init_FacebookAdGraphql = __esm({
15157
+ "src/extractor/FacebookAdGraphql.ts"() {
15158
+ "use strict";
15159
+ AD_LIBRARY_QUERY = "AdLibrarySearchPaginationQuery";
14817
15160
  }
14818
- return config?.country?.toUpperCase() === target.country && config?.state?.toUpperCase() === target.state && (city ? config?.city === city : !config?.city);
15161
+ });
15162
+
15163
+ // src/extractor/FacebookOrganicVideoExtractor.ts
15164
+ function htmlDecode(value) {
15165
+ return value.replace(/&amp;/g, "&").replace(/&quot;/g, '"').replace(/&#039;/g, "'").replace(/&lt;/g, "<").replace(/&gt;/g, ">");
14819
15166
  }
14820
- function findExistingTargetProxy(proxies, target) {
14821
- return proxies.find((proxy) => proxy.type === "residential" && proxy.status !== "unavailable" && Boolean(proxy.id) && (proxy.name === target.proxyName || configMatches(proxy.config, target, target.level === "city" ? target.city : void 0))) ?? null;
15167
+ function decodeEscapedValue(raw) {
15168
+ let out = raw;
15169
+ try {
15170
+ out = JSON.parse(`"${raw.replace(/"/g, '\\"')}"`);
15171
+ } catch {
15172
+ out = raw.replace(/\\u0025/g, "%").replace(/\\u0026/g, "&").replace(/\\u003D/g, "=").replace(/\\u003d/g, "=").replace(/\\"/g, '"').replace(/\\\//g, "/");
15173
+ }
15174
+ return htmlDecode(out);
14822
15175
  }
14823
- function findExistingProxy(proxies, target) {
14824
- for (const city of target.cityCandidates) {
14825
- const name = proxyName(target.country, target.state, city);
14826
- const found = proxies.find((proxy) => proxy.type === "residential" && proxy.status !== "unavailable" && Boolean(proxy.id) && (proxy.name === name || configMatches(proxy.config, target, city)));
14827
- if (found) return found;
15176
+ function facebookVideoIdFromUrl(url) {
15177
+ try {
15178
+ const parsed = new URL(url);
15179
+ const path6 = parsed.pathname;
15180
+ const direct = path6.match(/\/(?:reel|videos|watch)\/(?:[^/]+\/)?(\d{8,})/i);
15181
+ if (direct) return direct[1];
15182
+ const queryVideoId = parsed.searchParams.get("v") ?? parsed.searchParams.get("video_id");
15183
+ if (queryVideoId && /^\d{8,}$/.test(queryVideoId)) return queryVideoId;
15184
+ const numericTail = path6.match(/\/(\d{8,})\/?$/);
15185
+ if (numericTail) return numericTail[1];
15186
+ } catch {
14828
15187
  }
14829
15188
  return null;
14830
15189
  }
14831
- function stateTarget(target) {
14832
- return {
14833
- ...target,
14834
- level: "state",
14835
- proxyName: proxyName(target.country, target.state),
14836
- config: {
14837
- country: target.country,
14838
- state: target.state
14839
- }
14840
- };
14841
- }
14842
- function findExistingStateProxy(proxies, target) {
14843
- const name = proxyName(target.country, target.state);
14844
- return proxies.find((proxy) => proxy.type === "residential" && proxy.status !== "unavailable" && Boolean(proxy.id) && (proxy.name === name || configMatches(proxy.config, target))) ?? null;
15190
+ function parseEfg(url) {
15191
+ try {
15192
+ const efg = new URL(url).searchParams.get("efg");
15193
+ if (!efg) return null;
15194
+ return JSON.parse(Buffer.from(efg, "base64").toString("utf8"));
15195
+ } catch {
15196
+ return null;
15197
+ }
14845
15198
  }
14846
- function escalatedTargetLevel(target, attemptIndex) {
14847
- return stateTarget(target);
15199
+ function manifestDurationSec(htmlWindow) {
15200
+ const decoded = decodeEscapedValue(htmlWindow);
15201
+ const match = decoded.match(/mediaPresentationDuration="PT([\d.]+)S"/);
15202
+ return match ? Number(match[1]) : null;
14848
15203
  }
14849
- function errorText2(err) {
14850
- return err instanceof Error ? err.message : String(err);
15204
+ function metadataContent(html, key) {
15205
+ const escaped = key.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
15206
+ const re = new RegExp(`<meta\\s+[^>]*(?:property|name)=["']${escaped}["'][^>]*content=["']([^"']+)["'][^>]*>`, "i");
15207
+ const m = html.match(re);
15208
+ return m ? htmlDecode(m[1]).trim() : null;
14851
15209
  }
14852
- function freshTargetCandidates(target, explicitZip, attemptIndex) {
14853
- const out = [];
14854
- const zip = knownZipFor(target, explicitZip);
14855
- if (zip) {
14856
- const targetZip = zipTarget(target, zip);
14857
- out.push(withProxyName(targetZip, freshProxyName(targetZip.proxyName, attemptIndex)));
14858
- }
14859
- for (const city of target.cityCandidates) {
14860
- const cityTarget = {
14861
- ...target,
14862
- level: "city",
14863
- city,
14864
- proxyName: proxyName(target.country, target.state, city),
14865
- config: {
14866
- country: target.country,
14867
- state: target.state,
14868
- city
14869
- }
14870
- };
14871
- out.push(withProxyName(cityTarget, freshProxyName(cityTarget.proxyName, attemptIndex)));
14872
- }
14873
- const fallbackTarget = stateTarget(target);
14874
- out.push(withProxyName(fallbackTarget, freshProxyName(fallbackTarget.proxyName, attemptIndex)));
14875
- return out;
15210
+ function ownerNameFromHtml(html) {
15211
+ const ogTitle = metadataContent(html, "og:title");
15212
+ const pipe = ogTitle?.split("|").map((s) => s.trim()).filter(Boolean);
15213
+ if (pipe && pipe.length > 1) return pipe[pipe.length - 1] ?? null;
15214
+ return null;
14876
15215
  }
14877
- async function createFreshLocationProxy(kernel, options, target) {
14878
- const createErrors = [];
14879
- for (const candidate of freshTargetCandidates(target, options.proxyZip, options.attemptIndex)) {
15216
+ function extractFacebookOrganicVideo(html, pageUrl, sourceUrl = pageUrl, quality = "best") {
15217
+ const videoId = facebookVideoIdFromUrl(pageUrl) ?? facebookVideoIdFromUrl(sourceUrl);
15218
+ const manifestIndex = videoId ? html.indexOf(`dash_mpd_debug.mpd?v=${videoId}`) : -1;
15219
+ const searchWindow = manifestIndex >= 0 ? html.slice(Math.max(0, manifestIndex - 5e4), Math.min(html.length, manifestIndex + 5e4)) : html;
15220
+ const targetDuration = manifestDurationSec(searchWindow);
15221
+ const candidates = [];
15222
+ const seen = /* @__PURE__ */ new Set();
15223
+ const re = /browser_native_(sd|hd)_url\\?"\s*:\s*\\?"([^"<]+?)\\?"/g;
15224
+ let match;
15225
+ while ((match = re.exec(searchWindow)) !== null) {
15226
+ const url = decodeEscapedValue(match[2]);
15227
+ if (!url || seen.has(url)) continue;
15228
+ seen.add(url);
15229
+ const parsed = parseEfg(url);
15230
+ let bitrate = null;
14880
15231
  try {
14881
- const created = await kernel.proxies.create({
14882
- type: "residential",
14883
- name: candidate.proxyName,
14884
- config: candidate.level === "zip" ? { country: candidate.country, zip: candidate.zip } : candidate.config
14885
- });
14886
- if (created.id) {
14887
- return resolution("location_created", options.proxyMode, created.id, candidate, null, true);
14888
- }
14889
- createErrors.push(`${candidate.proxyName}: Kernel did not return a proxy id`);
14890
- } catch (err) {
14891
- createErrors.push(`${candidate.proxyName}: ${errorText2(err)}`);
15232
+ const value = new URL(url).searchParams.get("bitrate");
15233
+ bitrate = value ? Number(value) : null;
15234
+ } catch {
14892
15235
  }
15236
+ candidates.push({
15237
+ url,
15238
+ quality: match[1],
15239
+ bitrate: Number.isFinite(bitrate) ? bitrate : null,
15240
+ durationSec: typeof parsed?.duration_s === "number" ? parsed.duration_s : null,
15241
+ assetId: typeof parsed?.xpv_asset_id === "number" ? parsed.xpv_asset_id : null,
15242
+ vencodeTag: typeof parsed?.vencode_tag === "string" ? parsed.vencode_tag : null
15243
+ });
14893
15244
  }
14894
- if (process.env.KERNEL_PROXY_TYPE !== "residential") {
14895
- const ispTarget = stateTarget(target);
14896
- try {
14897
- const created = await kernel.proxies.create({
14898
- type: "isp",
14899
- name: freshProxyName(`mcp-serp-isp-${ispTarget.country.toLowerCase()}-${(ispTarget.state ?? "").toLowerCase()}`, options.attemptIndex),
14900
- config: { country: ispTarget.country, ...ispTarget.state ? { state: ispTarget.state } : {} }
14901
- });
14902
- if (created.id) {
14903
- return resolution("isp_created", options.proxyMode, created.id, ispTarget, null, true);
14904
- }
14905
- createErrors.push("isp: Kernel did not return a proxy id");
14906
- } catch (err) {
14907
- createErrors.push(`isp: ${errorText2(err)}`);
14908
- }
15245
+ const targetCandidates = candidates.filter((candidate) => {
15246
+ if (!targetDuration || candidate.durationSec == null) return true;
15247
+ return Math.abs(candidate.durationSec - targetDuration) <= 2;
15248
+ });
15249
+ const selectable = targetCandidates.length ? targetCandidates : candidates;
15250
+ const selected = [...selectable].filter((candidate) => quality === "best" || candidate.quality === quality).sort((a, b) => {
15251
+ const aq = a.quality === "hd" ? 1 : 0;
15252
+ const bq = b.quality === "hd" ? 1 : 0;
15253
+ if (quality === "best" && aq !== bq) return bq - aq;
15254
+ return (b.bitrate ?? 0) - (a.bitrate ?? 0);
15255
+ })[0];
15256
+ if (!selected) {
15257
+ throw new Error("No progressive Facebook video URL was found in the rendered page state");
14909
15258
  }
14910
- return resolution("configured_fallback", options.proxyMode, options.configuredKernelProxyId, target, createErrors.join(" | "));
14911
- }
14912
- async function deleteKernelProxyId(kernelApiKey, proxyId) {
14913
- if (!kernelApiKey || !proxyId) return;
14914
- const kernel = new import_sdk6.default({ apiKey: kernelApiKey });
14915
- await kernel.proxies.delete(proxyId);
15259
+ return {
15260
+ sourceUrl,
15261
+ pageUrl,
15262
+ videoId,
15263
+ title: metadataContent(html, "og:title") ?? metadataContent(html, "twitter:title"),
15264
+ description: metadataContent(html, "og:description") ?? metadataContent(html, "description") ?? metadataContent(html, "twitter:description"),
15265
+ ownerName: ownerNameFromHtml(html),
15266
+ videoUrl: selected.url,
15267
+ selectedQuality: selected.quality,
15268
+ bitrate: selected.bitrate,
15269
+ durationSec: targetDuration ?? selected.durationSec,
15270
+ assetId: selected.assetId,
15271
+ vencodeTag: selected.vencodeTag,
15272
+ candidates: selectable,
15273
+ extractedAt: (/* @__PURE__ */ new Date()).toISOString()
15274
+ };
14916
15275
  }
14917
- async function resolveKernelProxyId(options) {
14918
- if (options.proxyMode === "none") {
14919
- return resolution("disabled", options.proxyMode, void 0, null, null);
14920
- }
14921
- if (options.proxyMode === "configured") {
14922
- return resolution("configured_fallback", options.proxyMode, options.configuredKernelProxyId, null, null);
14923
- }
14924
- const target = parseKernelLocationProxyTarget(options.location, options.gl);
14925
- if (!target || !options.kernelApiKey) {
14926
- return resolution("configured_fallback", options.proxyMode, options.configuredKernelProxyId, target, target ? null : "location could not be normalized to a US city/state proxy target");
14927
- }
14928
- const kernel = new import_sdk6.default({ apiKey: options.kernelApiKey });
14929
- try {
14930
- const attemptIndex = options.attemptIndex ?? 0;
14931
- if (options.fresh) {
14932
- return await createFreshLocationProxy(kernel, options, target);
14933
- }
14934
- if (attemptIndex >= 1) {
14935
- const escalatedTarget = escalatedTargetLevel(target, attemptIndex);
14936
- const createErrors2 = [];
14937
- try {
14938
- const created = await kernel.proxies.create({
14939
- type: "residential",
14940
- name: escalatedTarget.proxyName,
14941
- config: escalatedTarget.config
14942
- });
14943
- if (created.id) {
14944
- return resolution("location_created", options.proxyMode, created.id, escalatedTarget, null);
14945
- }
14946
- createErrors2.push(`${escalatedTarget.state}: Kernel did not return a proxy id`);
14947
- } catch (err) {
14948
- createErrors2.push(`${escalatedTarget.state}: ${errorText2(err)}`);
14949
- }
14950
- return resolution("configured_fallback", options.proxyMode, options.configuredKernelProxyId, escalatedTarget, createErrors2.join(" | "));
14951
- }
14952
- const proxies = await kernel.proxies.list();
14953
- const zip = knownZipFor(target, options.proxyZip);
14954
- const createErrors = [];
14955
- if (zip) {
14956
- const targetZip = zipTarget(target, zip);
14957
- const existingZip = findExistingTargetProxy(proxies, targetZip);
14958
- if (existingZip?.id) {
14959
- return resolution("location_reused", options.proxyMode, existingZip.id, targetZip, null);
14960
- }
14961
- try {
14962
- const created = await kernel.proxies.create({
14963
- type: "residential",
14964
- name: targetZip.proxyName,
14965
- config: {
14966
- country: targetZip.country,
14967
- zip
14968
- }
14969
- });
14970
- if (created.id) {
14971
- return resolution("location_created", options.proxyMode, created.id, targetZip, null);
14972
- }
14973
- createErrors.push(`${zip}: Kernel did not return a proxy id`);
14974
- } catch (err) {
14975
- createErrors.push(`${zip}: ${errorText2(err)}`);
14976
- }
14977
- }
14978
- const existing = findExistingProxy(proxies, target);
14979
- if (existing?.id) {
14980
- return resolution("location_reused", options.proxyMode, existing.id, target, createErrors.join(" | ") || null);
14981
- }
14982
- for (const city of target.cityCandidates) {
14983
- try {
14984
- const created = await kernel.proxies.create({
14985
- type: "residential",
14986
- name: proxyName(target.country, target.state, city),
14987
- config: {
14988
- country: target.country,
14989
- state: target.state,
14990
- city
14991
- }
14992
- });
14993
- if (created.id) {
14994
- return resolution("location_created", options.proxyMode, created.id, {
14995
- ...target,
14996
- level: "city",
14997
- city,
14998
- proxyName: proxyName(target.country, target.state, city),
14999
- config: {
15000
- country: target.country,
15001
- state: target.state,
15002
- city
15003
- }
15004
- }, null);
15005
- }
15006
- createErrors.push(`${city}: Kernel did not return a proxy id`);
15007
- } catch (err) {
15008
- createErrors.push(`${city}: ${errorText2(err)}`);
15009
- }
15010
- }
15011
- const fallbackTarget = stateTarget(target);
15012
- const existingState = findExistingStateProxy(proxies, fallbackTarget);
15013
- if (existingState?.id) {
15014
- return resolution("location_reused", options.proxyMode, existingState.id, fallbackTarget, createErrors.join(" | "));
15015
- }
15016
- try {
15017
- const created = await kernel.proxies.create({
15018
- type: "residential",
15019
- name: fallbackTarget.proxyName,
15020
- config: fallbackTarget.config
15021
- });
15022
- if (created.id) {
15023
- return resolution("location_created", options.proxyMode, created.id, fallbackTarget, createErrors.join(" | "));
15024
- }
15025
- createErrors.push(`${fallbackTarget.state}: Kernel did not return a proxy id`);
15026
- } catch (err) {
15027
- createErrors.push(`${fallbackTarget.state}: ${errorText2(err)}`);
15028
- }
15029
- return resolution("configured_fallback", options.proxyMode, options.configuredKernelProxyId, target, createErrors.join(" | "));
15030
- } catch (err) {
15031
- return resolution("configured_fallback", options.proxyMode, options.configuredKernelProxyId, target, errorText2(err));
15032
- }
15276
+ async function extractFacebookOrganicVideoFromPage(page, sourceUrl, quality = "best") {
15277
+ await page.waitForLoadState("domcontentloaded", { timeout: 15e3 }).catch(() => {
15278
+ });
15279
+ await page.waitForTimeout(1500);
15280
+ const html = await page.content();
15281
+ return extractFacebookOrganicVideo(html, page.url(), sourceUrl, quality);
15033
15282
  }
15034
- var import_sdk6, US_STATE_CODES, US_CITY_CENTER_ZIPS;
15035
- var init_kernel_proxy_resolver = __esm({
15036
- "src/kernel-proxy-resolver.ts"() {
15283
+ var init_FacebookOrganicVideoExtractor = __esm({
15284
+ "src/extractor/FacebookOrganicVideoExtractor.ts"() {
15037
15285
  "use strict";
15038
- import_sdk6 = __toESM(require("@onkernel/sdk"), 1);
15039
- init_uule();
15040
- US_STATE_CODES = {
15041
- alabama: "AL",
15042
- alaska: "AK",
15043
- arizona: "AZ",
15044
- arkansas: "AR",
15045
- california: "CA",
15046
- colorado: "CO",
15047
- connecticut: "CT",
15048
- delaware: "DE",
15049
- florida: "FL",
15050
- georgia: "GA",
15051
- hawaii: "HI",
15052
- idaho: "ID",
15053
- illinois: "IL",
15054
- indiana: "IN",
15055
- iowa: "IA",
15056
- kansas: "KS",
15057
- kentucky: "KY",
15058
- louisiana: "LA",
15059
- maine: "ME",
15060
- maryland: "MD",
15061
- massachusetts: "MA",
15062
- michigan: "MI",
15063
- minnesota: "MN",
15064
- mississippi: "MS",
15065
- missouri: "MO",
15066
- montana: "MT",
15067
- nebraska: "NE",
15068
- nevada: "NV",
15069
- "new hampshire": "NH",
15070
- "new jersey": "NJ",
15071
- "new mexico": "NM",
15072
- "new york": "NY",
15073
- "north carolina": "NC",
15074
- "north dakota": "ND",
15075
- ohio: "OH",
15076
- oklahoma: "OK",
15077
- oregon: "OR",
15078
- pennsylvania: "PA",
15079
- "rhode island": "RI",
15080
- "south carolina": "SC",
15081
- "south dakota": "SD",
15082
- tennessee: "TN",
15083
- texas: "TX",
15084
- utah: "UT",
15085
- vermont: "VT",
15086
- virginia: "VA",
15087
- washington: "WA",
15088
- "west virginia": "WV",
15089
- wisconsin: "WI",
15090
- wyoming: "WY"
15091
- };
15092
- US_CITY_CENTER_ZIPS = {
15093
- "atlanta|GA": "30303",
15094
- "austin|TX": "78701",
15095
- "baltimore|MD": "21201",
15096
- "boston|MA": "02108",
15097
- "boulder|CO": "80302",
15098
- "charlotte|NC": "28202",
15099
- "chicago|IL": "60601",
15100
- "colorado_springs|CO": "80903",
15101
- "columbus|OH": "43215",
15102
- "dallas|TX": "75201",
15103
- "denver|CO": "80202",
15104
- "detroit|MI": "48226",
15105
- "fort_collins|CO": "80524",
15106
- "fort_worth|TX": "76102",
15107
- "houston|TX": "77002",
15108
- "indianapolis|IN": "46204",
15109
- "jacksonville|FL": "32202",
15110
- "las_vegas|NV": "89101",
15111
- "los_angeles|CA": "90012",
15112
- "louisville|KY": "40202",
15113
- "loveland|CO": "80537",
15114
- "memphis|TN": "38103",
15115
- "miami|FL": "33131",
15116
- "minneapolis|MN": "55401",
15117
- "nashville|TN": "37203",
15118
- "new_york|NY": "10001",
15119
- "orlando|FL": "32801",
15120
- "philadelphia|PA": "19103",
15121
- "phoenix|AZ": "85004",
15122
- "portland|OR": "97205",
15123
- "raleigh|NC": "27601",
15124
- "richmond|VA": "23219",
15125
- "sacramento|CA": "95814",
15126
- "salt_lake_city|UT": "84101",
15127
- "san_antonio|TX": "78205",
15128
- "san_diego|CA": "92101",
15129
- "san_francisco|CA": "94103",
15130
- "san_jose|CA": "95113",
15131
- "seattle|WA": "98101"
15132
- };
15133
15286
  }
15134
15287
  });
15135
15288
 
@@ -15262,8 +15415,7 @@ async function transcribeMediaUrl(mediaUrl, markdownTitle = "# Media Transcript"
15262
15415
  const text = data.text ?? "";
15263
15416
  const chunks = data.chunks ?? [];
15264
15417
  const durationMs = Date.now() - startMs;
15265
- const audioSec = chunks.length ? chunks[chunks.length - 1]?.timestamp?.[1] ?? 0 : 0;
15266
- void recordVendorUsage({ vendor: "fal_wizper", model: "fal-ai/wizper", units: audioSec / 60, unitType: "audio_min" });
15418
+ void recordVendorUsage({ vendor: "fal_wizper", model: "fal-ai/wizper", units: durationMs / 1e3, unitType: "compute_sec" });
15267
15419
  return {
15268
15420
  text,
15269
15421
  chunks,
@@ -16819,6 +16971,7 @@ var init_reddit_routes = __esm({
16819
16971
  if (!gate.ok) return c.json(concurrencyLimitExceededResponse(gate), 429, { "Retry-After": String(gate.retryAfterSeconds) });
16820
16972
  let debited = false;
16821
16973
  let refunded = false;
16974
+ let commentDebitMc = 0;
16822
16975
  try {
16823
16976
  const { ok, balance_mc } = await debitMc(user.id, MC_COSTS.reddit_thread, LedgerOperation.REDDIT_THREAD, body.url);
16824
16977
  if (!ok) return c.json(insufficientBalanceResponse(balance_mc, MC_COSTS.reddit_thread), 402);
@@ -17080,6 +17233,8 @@ var init_video_routes = __esm({
17080
17233
  init_rates();
17081
17234
  init_api_auth();
17082
17235
  init_memory();
17236
+ init_cost_telemetry();
17237
+ init_cost_context();
17083
17238
  videoApp = new import_hono9.Hono();
17084
17239
  AnalyzeBodySchema = import_zod20.z.object({
17085
17240
  sourceUrl: import_zod20.z.string().trim().url("sourceUrl must be a direct video file URL"),
@@ -17156,6 +17311,21 @@ var init_video_routes = __esm({
17156
17311
  effectiveFrames: st.frameCount ?? null
17157
17312
  };
17158
17313
  }
17314
+ if ((st.status === "done" || st.status === "failed") && typeof st.costUsd === "number" && st.costUsd > 0) {
17315
+ const llmOp = `${LedgerOperation.VIDEO_ANALYSIS_LLM}:${runId}`;
17316
+ if (!await ledgerExistsForOperation(user.id, llmOp)) {
17317
+ await runWithCostContext({ ...currentCostContext(), op: "video_analysis", userId: user.id }, async () => {
17318
+ void recordVendorUsage({ vendor: "mcp_memory_video", model: null, units: st.costUsd, unitType: "usd", error: null });
17319
+ });
17320
+ const surchargeMc = Math.round(st.costUsd * VIDEO_ANALYSIS_LLM_MARGIN_MULTIPLE * MC_PER_USD);
17321
+ if (surchargeMc > 0) {
17322
+ const { ok } = await debitMc(user.id, surchargeMc, llmOp, `real-cost-surcharge:${runId}`);
17323
+ if (!ok) {
17324
+ console.warn(`[video-routes] VIDEO_ANALYSIS_LLM surcharge debit failed for user ${user.id} (run ${runId}): insufficient balance for ${surchargeMc}mc`);
17325
+ }
17326
+ }
17327
+ }
17328
+ }
17159
17329
  return c.json({
17160
17330
  ok: true,
17161
17331
  runId,
@@ -18394,7 +18564,7 @@ var init_trustpilot_routes = __esm({
18394
18564
  let pagesFetched = 0;
18395
18565
  try {
18396
18566
  await driver.launch({
18397
- headless: true,
18567
+ headless: false,
18398
18568
  kernelApiKey: browserServiceApiKey(),
18399
18569
  viewport: { width: 1280, height: 900 },
18400
18570
  locale: "en-US"
@@ -18600,7 +18770,7 @@ var init_g2_routes = __esm({
18600
18770
  const driver = new BrowserDriver();
18601
18771
  try {
18602
18772
  await driver.launch({
18603
- headless: true,
18773
+ headless: false,
18604
18774
  kernelApiKey: browserServiceApiKey(),
18605
18775
  kernelProxyId: proxyId,
18606
18776
  viewport: { width: 1280, height: 900 },
@@ -27117,7 +27287,7 @@ var PACKAGE_VERSION;
27117
27287
  var init_version = __esm({
27118
27288
  "src/version.ts"() {
27119
27289
  "use strict";
27120
- PACKAGE_VERSION = "0.4.6";
27290
+ PACKAGE_VERSION = "0.4.12";
27121
27291
  }
27122
27292
  });
27123
27293
 
@@ -38049,14 +38219,17 @@ async function processJob(job) {
38049
38219
  running++;
38050
38220
  try {
38051
38221
  const opts = typeof job.options === "string" ? JSON.parse(job.options) : job.options;
38052
- const result = await harvest({
38053
- ...opts,
38054
- kernelApiKey: browserServiceApiKey(),
38055
- headless: true,
38056
- format: "json",
38057
- outputDir: "/tmp/paa-output-api",
38058
- onAttemptEvent: createHarvestAttemptRecorder(job.id, job.user_id)
38059
- });
38222
+ const result = await runWithCostContext(
38223
+ { op: opts.serpOnly ? "serp" : "paa", userId: Number(job.user_id) },
38224
+ () => harvest({
38225
+ ...opts,
38226
+ kernelApiKey: browserServiceApiKey(),
38227
+ headless: true,
38228
+ format: "json",
38229
+ outputDir: "/tmp/paa-output-api",
38230
+ onAttemptEvent: createHarvestAttemptRecorder(job.id, job.user_id)
38231
+ })
38232
+ );
38060
38233
  await completeJob(job.id, result);
38061
38234
  const attempts = await listHarvestAttempts(job.id, job.user_id);
38062
38235
  if (!opts.serpOnly && typeof opts.billingHoldMc === "number") {
@@ -38122,6 +38295,7 @@ var init_worker = __esm({
38122
38295
  init_rates();
38123
38296
  init_harvest_problems();
38124
38297
  init_harvest_attempt_events();
38298
+ init_cost_context();
38125
38299
  MAX_CONCURRENT = 2;
38126
38300
  running = 0;
38127
38301
  }
@@ -38200,9 +38374,12 @@ function opForCostPath(p) {
38200
38374
  }
38201
38375
  if (p.startsWith("/instagram/")) return "instagram";
38202
38376
  if (p === "/reddit/thread") return "reddit_thread";
38377
+ if (p.startsWith("/trustpilot/")) return "trustpilot_reviews";
38378
+ if (p.startsWith("/g2/")) return "g2_reviews";
38203
38379
  if (p.startsWith("/api/internal/site-architecture-auditor")) return "audit_site";
38204
38380
  if (p.startsWith("/api/internal/memory")) return "memory_ai";
38205
38381
  if (p.startsWith("/agent/")) return "browser_session";
38382
+ if (p.startsWith("/video/")) return "video_analysis";
38206
38383
  return null;
38207
38384
  }
38208
38385
  function maybeProvisionInBackground(user, vaultCount) {
@@ -39035,16 +39212,20 @@ var init_server = __esm({
39035
39212
  AbortSignal.timeout(syncTimeoutMs)
39036
39213
  ]);
39037
39214
  try {
39038
- const result = await harvest({
39039
- ...options,
39040
- kernelApiKey: browserServiceApiKey(),
39041
- headless: true,
39042
- format: "json",
39043
- outputDir: "/tmp/paa-output-api",
39044
- signal: syncSignal,
39045
- softDeadlineMs: Date.now() + Math.floor(syncTimeoutMs * 0.8),
39046
- onAttemptEvent: recordAttempt
39047
- });
39215
+ const harvestCtx = currentCostContext();
39216
+ const result = await runWithCostContext(
39217
+ { ...harvestCtx, op: options.serpOnly ? "serp" : "paa", userId: user.id },
39218
+ () => harvest({
39219
+ ...options,
39220
+ kernelApiKey: browserServiceApiKey(),
39221
+ headless: true,
39222
+ format: "json",
39223
+ outputDir: "/tmp/paa-output-api",
39224
+ signal: syncSignal,
39225
+ softDeadlineMs: Date.now() + Math.floor(syncTimeoutMs * 0.8),
39226
+ onAttemptEvent: recordAttempt
39227
+ })
39228
+ );
39048
39229
  await completeJob(jobId, result);
39049
39230
  const attempts = await listHarvestAttempts(jobId, user.id);
39050
39231
  if (!options.serpOnly) {