mcp-scraper 0.4.6 → 0.4.12
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bin/api-server.cjs +1532 -1351
- package/dist/bin/api-server.cjs.map +1 -1
- package/dist/bin/api-server.js +2 -2
- package/dist/bin/mcp-scraper-cli.cjs +1 -1
- package/dist/bin/mcp-scraper-cli.cjs.map +1 -1
- package/dist/bin/mcp-scraper-cli.js +1 -1
- package/dist/bin/mcp-scraper-install.cjs +1 -1
- package/dist/bin/mcp-scraper-install.cjs.map +1 -1
- package/dist/bin/mcp-scraper-install.js +1 -1
- package/dist/bin/mcp-stdio-server.cjs +1 -1
- package/dist/bin/mcp-stdio-server.cjs.map +1 -1
- package/dist/bin/mcp-stdio-server.js +2 -2
- package/dist/bin/paa-harvest.cjs +13 -4
- package/dist/bin/paa-harvest.cjs.map +1 -1
- package/dist/bin/paa-harvest.js +1 -1
- package/dist/{chunk-5GAD2AN2.js → chunk-2YY46QYT.js} +27 -10
- package/dist/chunk-2YY46QYT.js.map +1 -0
- package/dist/{chunk-IFXACD4K.js → chunk-6EXP6DQG.js} +2 -2
- package/dist/{chunk-IVQYNY45.js → chunk-EMY7ELRU.js} +23 -6
- package/dist/chunk-EMY7ELRU.js.map +1 -0
- package/dist/{chunk-NYAWN7CZ.js → chunk-ONIOF5XW.js} +2 -2
- package/dist/chunk-SHXJQQOH.js +7 -0
- package/dist/chunk-SHXJQQOH.js.map +1 -0
- package/dist/index.cjs +59 -4
- package/dist/index.cjs.map +1 -1
- package/dist/index.js +8 -2
- package/dist/index.js.map +1 -1
- package/dist/{server-R3KVQNOT.js → server-UNID3SJU.js} +409 -272
- package/dist/server-UNID3SJU.js.map +1 -0
- package/dist/{site-extract-repository-LI2S3S6E.js → site-extract-repository-HTIY52MW.js} +3 -3
- package/dist/{worker-VASAJ7FZ.js → worker-HTYZAYGB.js} +16 -12
- package/dist/worker-HTYZAYGB.js.map +1 -0
- package/package.json +1 -2
- package/dist/chunk-5BB6Y3BB.js +0 -7
- package/dist/chunk-5BB6Y3BB.js.map +0 -1
- package/dist/chunk-5GAD2AN2.js.map +0 -1
- package/dist/chunk-IVQYNY45.js.map +0 -1
- package/dist/server-R3KVQNOT.js.map +0 -1
- package/dist/worker-VASAJ7FZ.js.map +0 -1
- /package/dist/{chunk-IFXACD4K.js.map → chunk-6EXP6DQG.js.map} +0 -0
- /package/dist/{chunk-NYAWN7CZ.js.map → chunk-ONIOF5XW.js.map} +0 -0
- /package/dist/{site-extract-repository-LI2S3S6E.js.map → site-extract-repository-HTIY52MW.js.map} +0 -0
package/dist/bin/api-server.cjs
CHANGED
|
@@ -5341,18 +5341,25 @@ function kernelCostUsd(ms, headful) {
|
|
|
5341
5341
|
return sec * (headful ? KERNEL_HEADFUL_USD_PER_SEC : KERNEL_HEADLESS_USD_PER_SEC);
|
|
5342
5342
|
}
|
|
5343
5343
|
function vendorCostUsd(vendor, units) {
|
|
5344
|
-
if (vendor === "fal_wizper")
|
|
5344
|
+
if (vendor === "fal_wizper") {
|
|
5345
|
+
const estimatedComputeSec = Math.max(0, units) / FAL_WIZPER_WALLCLOCK_TO_COMPUTE_SEC_FACTOR;
|
|
5346
|
+
return estimatedComputeSec * FAL_WIZPER_USD_PER_COMPUTE_SEC;
|
|
5347
|
+
}
|
|
5345
5348
|
if (vendor === "deepinfra_qwen") return Math.max(0, units) / 1e3 * DEEPINFRA_QWEN_USD_PER_1K_OUT_TOKENS;
|
|
5349
|
+
if (vendor === "openrouter") return Math.max(0, units);
|
|
5350
|
+
if (vendor === "mcp_memory_video") return Math.max(0, units);
|
|
5351
|
+
if (vendor === "mcp_memory_ai") return Math.max(0, units);
|
|
5346
5352
|
return 0;
|
|
5347
5353
|
}
|
|
5348
|
-
var KERNEL_HEADLESS_USD_PER_SEC, KERNEL_HEADFUL_USD_PER_SEC, HEADLESS_OPS,
|
|
5354
|
+
var KERNEL_HEADLESS_USD_PER_SEC, KERNEL_HEADFUL_USD_PER_SEC, HEADLESS_OPS, FAL_WIZPER_USD_PER_COMPUTE_SEC, FAL_WIZPER_WALLCLOCK_TO_COMPUTE_SEC_FACTOR, DEEPINFRA_QWEN_USD_PER_1K_OUT_TOKENS;
|
|
5349
5355
|
var init_cost_rates = __esm({
|
|
5350
5356
|
"src/api/cost-rates.ts"() {
|
|
5351
5357
|
"use strict";
|
|
5352
5358
|
KERNEL_HEADLESS_USD_PER_SEC = 166667e-10;
|
|
5353
5359
|
KERNEL_HEADFUL_USD_PER_SEC = 1333336e-10;
|
|
5354
|
-
HEADLESS_OPS = /* @__PURE__ */ new Set(["
|
|
5355
|
-
|
|
5360
|
+
HEADLESS_OPS = /* @__PURE__ */ new Set(["serp", "maps_search", "yt_channel", "yt_transcription", "fb_search", "fb_ad", "instagram"]);
|
|
5361
|
+
FAL_WIZPER_USD_PER_COMPUTE_SEC = 111e-5;
|
|
5362
|
+
FAL_WIZPER_WALLCLOCK_TO_COMPUTE_SEC_FACTOR = 4;
|
|
5356
5363
|
DEEPINFRA_QWEN_USD_PER_1K_OUT_TOKENS = 4e-4;
|
|
5357
5364
|
}
|
|
5358
5365
|
});
|
|
@@ -5393,6 +5400,10 @@ async function migrateCostTelemetry() {
|
|
|
5393
5400
|
await db.execute(`ALTER TABLE kernel_session_log ADD COLUMN proxy_type TEXT`);
|
|
5394
5401
|
} catch {
|
|
5395
5402
|
}
|
|
5403
|
+
try {
|
|
5404
|
+
await db.execute(`ALTER TABLE kernel_session_log ADD COLUMN method TEXT`);
|
|
5405
|
+
} catch {
|
|
5406
|
+
}
|
|
5396
5407
|
await db.execute(`
|
|
5397
5408
|
CREATE TABLE IF NOT EXISTS vendor_usage_log (
|
|
5398
5409
|
id TEXT PRIMARY KEY,
|
|
@@ -5411,6 +5422,10 @@ async function migrateCostTelemetry() {
|
|
|
5411
5422
|
await db.execute(`CREATE INDEX IF NOT EXISTS vendor_usage_log_op ON vendor_usage_log(op)`);
|
|
5412
5423
|
await db.execute(`CREATE INDEX IF NOT EXISTS vendor_usage_log_probe ON vendor_usage_log(probe_run_id)`);
|
|
5413
5424
|
await db.execute(`CREATE INDEX IF NOT EXISTS vendor_usage_log_created ON vendor_usage_log(created_at)`);
|
|
5425
|
+
try {
|
|
5426
|
+
await db.execute(`ALTER TABLE vendor_usage_log ADD COLUMN method TEXT`);
|
|
5427
|
+
} catch {
|
|
5428
|
+
}
|
|
5414
5429
|
await db.execute(`
|
|
5415
5430
|
CREATE TABLE IF NOT EXISTS cost_probe_runs (
|
|
5416
5431
|
id TEXT PRIMARY KEY,
|
|
@@ -5457,8 +5472,8 @@ async function recordKernelSession(r) {
|
|
|
5457
5472
|
const db = getDb();
|
|
5458
5473
|
await db.execute({
|
|
5459
5474
|
sql: `INSERT INTO kernel_session_log
|
|
5460
|
-
(id, kernel_session_id, op, source, probe_run_id, user_id, stealth, headless_sent, proxy_used, proxy_source, proxy_type, fallback, opened_at, closed_at, duration_ms, est_cost_usd_headless, est_cost_usd_headful, error)
|
|
5461
|
-
VALUES (
|
|
5475
|
+
(id, kernel_session_id, op, source, probe_run_id, user_id, stealth, headless_sent, proxy_used, proxy_source, proxy_type, fallback, opened_at, closed_at, duration_ms, est_cost_usd_headless, est_cost_usd_headful, error, method)
|
|
5476
|
+
VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)`,
|
|
5462
5477
|
args: [
|
|
5463
5478
|
(0, import_node_crypto2.randomUUID)(),
|
|
5464
5479
|
r.kernelSessionId ?? null,
|
|
@@ -5477,7 +5492,8 @@ async function recordKernelSession(r) {
|
|
|
5477
5492
|
durationMs,
|
|
5478
5493
|
kernelCostUsd(durationMs, false),
|
|
5479
5494
|
kernelCostUsd(durationMs, true),
|
|
5480
|
-
r.error ?? null
|
|
5495
|
+
r.error ?? null,
|
|
5496
|
+
ctx?.subOp ?? null
|
|
5481
5497
|
]
|
|
5482
5498
|
});
|
|
5483
5499
|
} catch (err) {
|
|
@@ -5491,8 +5507,8 @@ async function recordVendorUsage(r) {
|
|
|
5491
5507
|
const db = getDb();
|
|
5492
5508
|
await db.execute({
|
|
5493
5509
|
sql: `INSERT INTO vendor_usage_log
|
|
5494
|
-
(id, op, probe_run_id, user_id, vendor, model, units, unit_type, est_cost_usd, error)
|
|
5495
|
-
VALUES (
|
|
5510
|
+
(id, op, probe_run_id, user_id, vendor, model, units, unit_type, est_cost_usd, error, method)
|
|
5511
|
+
VALUES (?,?,?,?,?,?,?,?,?,?,?)`,
|
|
5496
5512
|
args: [
|
|
5497
5513
|
(0, import_node_crypto2.randomUUID)(),
|
|
5498
5514
|
ctx?.op ?? null,
|
|
@@ -5503,7 +5519,8 @@ async function recordVendorUsage(r) {
|
|
|
5503
5519
|
r.units,
|
|
5504
5520
|
r.unitType,
|
|
5505
5521
|
vendorCostUsd(r.vendor, r.units),
|
|
5506
|
-
r.error ?? null
|
|
5522
|
+
r.error ?? null,
|
|
5523
|
+
ctx?.subOp ?? null
|
|
5507
5524
|
]
|
|
5508
5525
|
});
|
|
5509
5526
|
} catch (err) {
|
|
@@ -7405,6 +7422,7 @@ var init_llm_parse_with_retry = __esm({
|
|
|
7405
7422
|
DeepInfraLlmClient = class {
|
|
7406
7423
|
apiKey;
|
|
7407
7424
|
model;
|
|
7425
|
+
totalCostUsd = 0;
|
|
7408
7426
|
constructor(apiKey, model = "Qwen/Qwen3.6-35B-A3B") {
|
|
7409
7427
|
this.apiKey = apiKey;
|
|
7410
7428
|
this.model = model;
|
|
@@ -7437,6 +7455,7 @@ ${raw}`
|
|
|
7437
7455
|
OpenRouterLlmClient = class {
|
|
7438
7456
|
apiKey;
|
|
7439
7457
|
model;
|
|
7458
|
+
totalCostUsd = 0;
|
|
7440
7459
|
constructor(apiKey, model = "qwen/qwen3-235b-a22b") {
|
|
7441
7460
|
this.apiKey = apiKey;
|
|
7442
7461
|
this.model = model;
|
|
@@ -7449,6 +7468,9 @@ ${raw}`
|
|
|
7449
7468
|
});
|
|
7450
7469
|
if (!resp.ok) throw new Error(`OpenRouter error ${resp.status}: ${await resp.text()}`);
|
|
7451
7470
|
const data = await resp.json();
|
|
7471
|
+
const cost = data.usage?.cost ?? 0;
|
|
7472
|
+
this.totalCostUsd += cost;
|
|
7473
|
+
void recordVendorUsage({ vendor: "openrouter", model: this.model, units: cost, unitType: "usd" });
|
|
7452
7474
|
return data.choices[0].message.content;
|
|
7453
7475
|
}
|
|
7454
7476
|
async completeJson(prompt) {
|
|
@@ -7474,6 +7496,9 @@ ${raw}`
|
|
|
7474
7496
|
secondary;
|
|
7475
7497
|
lastServedBy = null;
|
|
7476
7498
|
lastPrimaryError = null;
|
|
7499
|
+
get totalCostUsd() {
|
|
7500
|
+
return this.primary.totalCostUsd + this.secondary.totalCostUsd;
|
|
7501
|
+
}
|
|
7477
7502
|
async complete(prompt) {
|
|
7478
7503
|
try {
|
|
7479
7504
|
const result = await this.primary.complete(prompt);
|
|
@@ -7928,6 +7953,390 @@ var init_schemas = __esm({
|
|
|
7928
7953
|
}
|
|
7929
7954
|
});
|
|
7930
7955
|
|
|
7956
|
+
// src/api/rates.ts
|
|
7957
|
+
function browserActiveCostMc(activeMs) {
|
|
7958
|
+
return Math.round(activeMs * MC_PER_BROWSER_MS);
|
|
7959
|
+
}
|
|
7960
|
+
function concurrencySlotBillingInfo() {
|
|
7961
|
+
const billingUrl = process.env.CONCURRENCY_BILLING_URL ?? "https://mcpscraper.dev/billing";
|
|
7962
|
+
return {
|
|
7963
|
+
product: "Extra concurrency slot",
|
|
7964
|
+
price_label: CONCURRENCY_SLOT_PRICE_LABEL,
|
|
7965
|
+
unit_amount_usd: CONCURRENCY_SLOT_PRICE_USD,
|
|
7966
|
+
currency: CONCURRENCY_SLOT_PRICE_CURRENCY,
|
|
7967
|
+
interval: CONCURRENCY_SLOT_PRICE_INTERVAL,
|
|
7968
|
+
billing_url: billingUrl,
|
|
7969
|
+
terminal_command: CONCURRENCY_SLOT_TERMINAL_COMMAND,
|
|
7970
|
+
terminal_command_with_api_key_env: `MCP_SCRAPER_API_KEY=sk_live_your_key ${CONCURRENCY_SLOT_TERMINAL_COMMAND}`
|
|
7971
|
+
};
|
|
7972
|
+
}
|
|
7973
|
+
function mcToCredits(mc) {
|
|
7974
|
+
return mc / MC_PER_CREDIT;
|
|
7975
|
+
}
|
|
7976
|
+
function insufficientBalanceResponse(balanceMc, requiredMc) {
|
|
7977
|
+
const topupUrl = process.env.TOPUP_URL ?? "https://mcpscraper.dev/billing";
|
|
7978
|
+
const balanceCredits = mcToCredits(balanceMc);
|
|
7979
|
+
const requiredCredits = mcToCredits(requiredMc);
|
|
7980
|
+
return {
|
|
7981
|
+
error: "insufficient_balance",
|
|
7982
|
+
error_code: "insufficient_balance",
|
|
7983
|
+
message: `Insufficient credits. Balance: ${balanceCredits} credits. This call requires ${requiredCredits} credits. Top up at ${topupUrl}`,
|
|
7984
|
+
balance_credits: balanceCredits,
|
|
7985
|
+
required_credits: requiredCredits,
|
|
7986
|
+
topup_url: topupUrl
|
|
7987
|
+
};
|
|
7988
|
+
}
|
|
7989
|
+
var MC_COSTS, MC_PER_BROWSER_MS, BROWSER_OPEN_MIN_BALANCE_MC, MC_PER_CREDIT, CREDIT_COST_CATALOG, CONCURRENCY_PRICE_ID, SUBSCRIPTION_TIERS, SUBSCRIPTION_TIER_BY_KEY, CONCURRENCY_SLOT_PRICE_USD, CONCURRENCY_SLOT_PRICE_CURRENCY, CONCURRENCY_SLOT_PRICE_INTERVAL, CONCURRENCY_SLOT_PRICE_LABEL, CONCURRENCY_SLOT_TERMINAL_COMMAND, FREE_SIGNUP_MC, MEMORY_FREE_QUOTA_BYTES, MEMORY_PLANS, MEMORY_PLAN_QUOTA, MEMORY_MARGIN_MULTIPLE, MEMORY_PRO_COST_BUDGET_USD, SCHEDULING_PLANS, LedgerOperation, MEMORY_AI_MARGIN_MULTIPLE, SITE_AUDIT_LLM_MARGIN_MULTIPLE, VIDEO_ANALYSIS_LLM_MARGIN_MULTIPLE, MC_PER_USD;
|
|
7990
|
+
var init_rates = __esm({
|
|
7991
|
+
"src/api/rates.ts"() {
|
|
7992
|
+
"use strict";
|
|
7993
|
+
MC_COSTS = {
|
|
7994
|
+
serp: 100,
|
|
7995
|
+
paa: 200,
|
|
7996
|
+
paa_base: 1e3,
|
|
7997
|
+
page_scrape: 100,
|
|
7998
|
+
url_map: 500,
|
|
7999
|
+
yt_channel: 200,
|
|
8000
|
+
yt_transcription: 200,
|
|
8001
|
+
fb_ad: 200,
|
|
8002
|
+
maps_search: 500,
|
|
8003
|
+
maps_place: 6e3,
|
|
8004
|
+
maps_review: 100,
|
|
8005
|
+
fb_search: 600,
|
|
8006
|
+
fb_transcribe: 200,
|
|
8007
|
+
google_ads_search: 600,
|
|
8008
|
+
google_ads_intel: 200,
|
|
8009
|
+
google_ads_transcribe: 200,
|
|
8010
|
+
instagram_profile: 400,
|
|
8011
|
+
instagram_media: 400,
|
|
8012
|
+
instagram_transcribe: 50,
|
|
8013
|
+
browser_minute: 12e3,
|
|
8014
|
+
reddit_thread: 3e3,
|
|
8015
|
+
reddit_comment: 200,
|
|
8016
|
+
video_analysis: 666667,
|
|
8017
|
+
trustpilot_reviews: 500,
|
|
8018
|
+
trustpilot_review: 100,
|
|
8019
|
+
g2_reviews: 500,
|
|
8020
|
+
g2_review: 150,
|
|
8021
|
+
diff_page: 100
|
|
8022
|
+
};
|
|
8023
|
+
MC_PER_BROWSER_MS = MC_COSTS.browser_minute / 6e4;
|
|
8024
|
+
BROWSER_OPEN_MIN_BALANCE_MC = 1e3;
|
|
8025
|
+
MC_PER_CREDIT = 1e3;
|
|
8026
|
+
CREDIT_COST_CATALOG = [
|
|
8027
|
+
{
|
|
8028
|
+
key: "serp",
|
|
8029
|
+
label: "SERP search",
|
|
8030
|
+
aliases: ["search_serp", "serp", "google search", "organic results"],
|
|
8031
|
+
credits: mcToCredits(MC_COSTS.serp),
|
|
8032
|
+
unit: "per search",
|
|
8033
|
+
notes: "Returns AI Overview, PAA snippet, videos, forums, and local pack."
|
|
8034
|
+
},
|
|
8035
|
+
{
|
|
8036
|
+
key: "paa",
|
|
8037
|
+
label: "PAA harvest",
|
|
8038
|
+
aliases: ["harvest_paa", "paa", "people also ask", "questions"],
|
|
8039
|
+
credits: mcToCredits(MC_COSTS.paa),
|
|
8040
|
+
unit: `per question (+${mcToCredits(MC_COSTS.paa_base)} credit base)`,
|
|
8041
|
+
notes: `Full SERP feature extraction. Billed ${mcToCredits(MC_COSTS.paa_base)} credit base + ${mcToCredits(MC_COSTS.paa)} per question actually returned (unused estimate refunded).`
|
|
8042
|
+
},
|
|
8043
|
+
{
|
|
8044
|
+
key: "page_scrape",
|
|
8045
|
+
label: "Page crawl / extract",
|
|
8046
|
+
aliases: ["extract_url", "extract_site", "page scrape", "url scrape", "single page", "site crawl"],
|
|
8047
|
+
credits: mcToCredits(MC_COSTS.page_scrape),
|
|
8048
|
+
unit: "per page",
|
|
8049
|
+
notes: "Applies to both single-URL extraction and per-page site crawls."
|
|
8050
|
+
},
|
|
8051
|
+
{
|
|
8052
|
+
key: "url_map",
|
|
8053
|
+
label: "Site URL mapping",
|
|
8054
|
+
aliases: ["map_site_urls", "url map", "site map", "crawl urls"],
|
|
8055
|
+
credits: mcToCredits(MC_COSTS.url_map),
|
|
8056
|
+
unit: "per mapping operation",
|
|
8057
|
+
notes: "Flat rate for the full /map-urls call regardless of URL count discovered."
|
|
8058
|
+
},
|
|
8059
|
+
{
|
|
8060
|
+
key: "yt_channel",
|
|
8061
|
+
label: "YouTube search / channel harvest",
|
|
8062
|
+
aliases: ["youtube_harvest", "youtube search", "youtube channel", "yt_channel"],
|
|
8063
|
+
credits: mcToCredits(MC_COSTS.yt_channel),
|
|
8064
|
+
unit: "per call"
|
|
8065
|
+
},
|
|
8066
|
+
{
|
|
8067
|
+
key: "yt_transcription",
|
|
8068
|
+
label: "YouTube transcription",
|
|
8069
|
+
aliases: ["youtube_transcribe", "youtube transcript", "transcription", "yt_transcription"],
|
|
8070
|
+
credits: mcToCredits(MC_COSTS.yt_transcription),
|
|
8071
|
+
unit: "per minute",
|
|
8072
|
+
notes: "A 5-minute hold is taken, then reconciled to actual video duration."
|
|
8073
|
+
},
|
|
8074
|
+
{
|
|
8075
|
+
key: "fb_ad",
|
|
8076
|
+
label: "Facebook search / ad lookup",
|
|
8077
|
+
aliases: ["facebook_page_intel", "facebook_ad_search", "facebook_ad", "facebook ads", "fb ads"],
|
|
8078
|
+
credits: mcToCredits(MC_COSTS.fb_ad),
|
|
8079
|
+
unit: "per call"
|
|
8080
|
+
},
|
|
8081
|
+
{
|
|
8082
|
+
key: "maps_search",
|
|
8083
|
+
label: "Maps business search",
|
|
8084
|
+
aliases: ["maps_search", "google maps search", "gmb search", "gbp search", "business profiles"],
|
|
8085
|
+
credits: mcToCredits(MC_COSTS.maps_search),
|
|
8086
|
+
unit: "per search",
|
|
8087
|
+
notes: "Returns up to 50 Google Maps business/profile candidates. Use maps_place_intel to hydrate selected businesses."
|
|
8088
|
+
},
|
|
8089
|
+
{
|
|
8090
|
+
key: "maps_place",
|
|
8091
|
+
label: "Maps business lookup",
|
|
8092
|
+
aliases: ["maps_place_intel", "google maps", "maps place", "place intel"],
|
|
8093
|
+
credits: mcToCredits(MC_COSTS.maps_place),
|
|
8094
|
+
unit: "per business",
|
|
8095
|
+
notes: "Base lookup. Reviews billed separately per card at maps_review rate."
|
|
8096
|
+
},
|
|
8097
|
+
{
|
|
8098
|
+
key: "maps_review",
|
|
8099
|
+
label: "Maps review",
|
|
8100
|
+
aliases: ["maps_reviews", "google reviews", "review cards", "reviews"],
|
|
8101
|
+
credits: mcToCredits(MC_COSTS.maps_review),
|
|
8102
|
+
unit: "per review card",
|
|
8103
|
+
notes: "Charged after extraction when includeReviews is true."
|
|
8104
|
+
},
|
|
8105
|
+
{
|
|
8106
|
+
key: "fb_search",
|
|
8107
|
+
label: "Facebook ad library search",
|
|
8108
|
+
aliases: ["facebook_search", "fb_search", "fb ad search"],
|
|
8109
|
+
credits: mcToCredits(MC_COSTS.fb_search),
|
|
8110
|
+
unit: "per search",
|
|
8111
|
+
notes: "Browser automation to search Facebook Ads Library by keyword."
|
|
8112
|
+
},
|
|
8113
|
+
{
|
|
8114
|
+
key: "google_ads_search",
|
|
8115
|
+
label: "Google Ads Transparency search",
|
|
8116
|
+
aliases: ["google_ads_search", "google ads search", "ads transparency search"],
|
|
8117
|
+
credits: mcToCredits(MC_COSTS.google_ads_search),
|
|
8118
|
+
unit: "per search",
|
|
8119
|
+
notes: "Browser automation to find advertisers in Google Ads Transparency Center by domain or name."
|
|
8120
|
+
},
|
|
8121
|
+
{
|
|
8122
|
+
key: "google_ads_intel",
|
|
8123
|
+
label: "Google Ads Transparency advertiser intel",
|
|
8124
|
+
aliases: ["google_ads_page_intel", "google ads intel", "ads transparency intel"],
|
|
8125
|
+
credits: mcToCredits(MC_COSTS.google_ads_intel),
|
|
8126
|
+
unit: "per call",
|
|
8127
|
+
notes: "Lists and hydrates an advertiser's creatives with image URLs and video references."
|
|
8128
|
+
},
|
|
8129
|
+
{
|
|
8130
|
+
key: "google_ads_transcribe",
|
|
8131
|
+
label: "Google ad video transcription",
|
|
8132
|
+
aliases: ["google_ads_transcribe", "google ad transcript"],
|
|
8133
|
+
credits: mcToCredits(MC_COSTS.google_ads_transcribe),
|
|
8134
|
+
unit: "per minute",
|
|
8135
|
+
notes: "A hold is taken, then reconciled to actual video duration."
|
|
8136
|
+
},
|
|
8137
|
+
{
|
|
8138
|
+
key: "fb_transcribe",
|
|
8139
|
+
label: "Facebook video / ad transcription",
|
|
8140
|
+
aliases: ["facebook_transcribe", "facebook_video_transcribe", "fb_transcribe", "fb ad transcript"],
|
|
8141
|
+
credits: mcToCredits(MC_COSTS.fb_transcribe),
|
|
8142
|
+
unit: "per minute",
|
|
8143
|
+
notes: "A hold is taken, then reconciled to actual video duration."
|
|
8144
|
+
},
|
|
8145
|
+
{
|
|
8146
|
+
key: "instagram_profile",
|
|
8147
|
+
label: "Instagram profile content discovery",
|
|
8148
|
+
aliases: ["instagram_profile_content", "instagram profile", "instagram content list", "ig profile"],
|
|
8149
|
+
credits: mcToCredits(MC_COSTS.instagram_profile),
|
|
8150
|
+
unit: "per profile scan",
|
|
8151
|
+
notes: "Browser extraction of public Instagram profile grid links. Complete history may require a logged-in profile."
|
|
8152
|
+
},
|
|
8153
|
+
{
|
|
8154
|
+
key: "instagram_media",
|
|
8155
|
+
label: "Instagram media download",
|
|
8156
|
+
aliases: ["instagram_media_download", "instagram reel download", "instagram post download", "ig media"],
|
|
8157
|
+
credits: mcToCredits(MC_COSTS.instagram_media),
|
|
8158
|
+
unit: "per post or reel",
|
|
8159
|
+
notes: "Extracts post text, image URLs, and reel audio/video tracks, with local downloads when the server can write files."
|
|
8160
|
+
},
|
|
8161
|
+
{
|
|
8162
|
+
key: "instagram_transcribe",
|
|
8163
|
+
label: "Instagram media transcription",
|
|
8164
|
+
aliases: ["instagram transcript", "instagram reel transcript", "ig transcribe"],
|
|
8165
|
+
credits: mcToCredits(MC_COSTS.instagram_transcribe),
|
|
8166
|
+
unit: "per call",
|
|
8167
|
+
notes: "Whisper transcription of selected Instagram audio/video media via fal.ai."
|
|
8168
|
+
},
|
|
8169
|
+
{
|
|
8170
|
+
key: "browser_minute",
|
|
8171
|
+
label: "Interactive browser session",
|
|
8172
|
+
aliases: ["browser_open", "browser agent", "browser_agent", "live browser", "browse", "browser control", "interactive browser"],
|
|
8173
|
+
credits: mcToCredits(MC_COSTS.browser_minute),
|
|
8174
|
+
unit: "per minute of use",
|
|
8175
|
+
notes: "Metered per minute of use for the whole time the browser session is open. Close the session to stop the meter; abandoned sessions are auto-closed after a short idle window."
|
|
8176
|
+
},
|
|
8177
|
+
{
|
|
8178
|
+
key: "reddit_thread",
|
|
8179
|
+
label: "Reddit thread base lookup",
|
|
8180
|
+
aliases: ["reddit", "reddit_thread", "reddit comments", "subreddit", "reddit post"],
|
|
8181
|
+
credits: mcToCredits(MC_COSTS.reddit_thread),
|
|
8182
|
+
unit: "per thread",
|
|
8183
|
+
notes: "Base lookup for the post itself. Comments billed separately per comment at reddit_comment rate. Refunded if the thread cannot be retrieved."
|
|
8184
|
+
},
|
|
8185
|
+
{
|
|
8186
|
+
key: "reddit_comment",
|
|
8187
|
+
label: "Reddit comment",
|
|
8188
|
+
aliases: ["reddit_comment", "reddit comments"],
|
|
8189
|
+
credits: mcToCredits(MC_COSTS.reddit_comment),
|
|
8190
|
+
unit: "per comment",
|
|
8191
|
+
notes: "Charged per comment actually extracted, billed down automatically if the balance runs out mid-thread."
|
|
8192
|
+
},
|
|
8193
|
+
{
|
|
8194
|
+
key: "video_analysis",
|
|
8195
|
+
label: "Video breakdown (frame-by-frame + transcript)",
|
|
8196
|
+
aliases: ["video_frame_analysis", "video breakdown", "video analysis", "analyze video"],
|
|
8197
|
+
credits: mcToCredits(MC_COSTS.video_analysis),
|
|
8198
|
+
unit: "per 120 frames (max 480)",
|
|
8199
|
+
notes: "Full multi-lens video breakdown: samples frames across a video (up to 30 minutes), analyzes each with vision AI, transcribes the audio, then produces summary, pacing/energy, words-per-minute, topic outline, key points, hook analysis, visual style, and a how-to-replicate recipe with a quality-control pass. $1 per 120 frames requested (max 480 = $4); billed down automatically if the video cannot use the requested frames, and refunded fully if the run fails."
|
|
8200
|
+
},
|
|
8201
|
+
{
|
|
8202
|
+
key: "trustpilot_reviews",
|
|
8203
|
+
label: "Trustpilot review harvest",
|
|
8204
|
+
aliases: ["trustpilot_reviews", "trustpilot", "trustpilot reviews"],
|
|
8205
|
+
credits: mcToCredits(MC_COSTS.trustpilot_reviews),
|
|
8206
|
+
unit: "per call",
|
|
8207
|
+
notes: "Base lookup. Reviews billed separately per card at trustpilot_review rate. Refunded if no reviews are found."
|
|
8208
|
+
},
|
|
8209
|
+
{
|
|
8210
|
+
key: "trustpilot_review",
|
|
8211
|
+
label: "Trustpilot review card",
|
|
8212
|
+
aliases: ["trustpilot_review", "trustpilot card"],
|
|
8213
|
+
credits: mcToCredits(MC_COSTS.trustpilot_review),
|
|
8214
|
+
unit: "per review card",
|
|
8215
|
+
notes: "Charged per review actually extracted, billed down automatically if the balance runs out mid-page."
|
|
8216
|
+
},
|
|
8217
|
+
{
|
|
8218
|
+
key: "g2_reviews",
|
|
8219
|
+
label: "G2 review harvest",
|
|
8220
|
+
aliases: ["g2_reviews", "g2", "g2 reviews"],
|
|
8221
|
+
credits: mcToCredits(MC_COSTS.g2_reviews),
|
|
8222
|
+
unit: "per call",
|
|
8223
|
+
notes: "Base lookup. Reviews billed separately per card at g2_review rate. Refunded if no reviews are found."
|
|
8224
|
+
},
|
|
8225
|
+
{
|
|
8226
|
+
key: "g2_review",
|
|
8227
|
+
label: "G2 review card",
|
|
8228
|
+
aliases: ["g2_review", "g2 card"],
|
|
8229
|
+
credits: mcToCredits(MC_COSTS.g2_review),
|
|
8230
|
+
unit: "per review card",
|
|
8231
|
+
notes: "Charged per review actually extracted (each card carries up to 3 Q&A sections), billed down automatically if the balance runs out mid-page."
|
|
8232
|
+
},
|
|
8233
|
+
{
|
|
8234
|
+
key: "diff_page",
|
|
8235
|
+
label: "Page change check",
|
|
8236
|
+
aliases: ["diff_page", "page diff", "change detection"],
|
|
8237
|
+
credits: mcToCredits(MC_COSTS.diff_page),
|
|
8238
|
+
unit: "per check",
|
|
8239
|
+
notes: "Same cost as a single page extract \u2014 one scrape is performed per check, then compared against your last stored snapshot for that URL."
|
|
8240
|
+
}
|
|
8241
|
+
];
|
|
8242
|
+
CONCURRENCY_PRICE_ID = "price_1Ta1NRS8aAcsk3TGwsRnYbix";
|
|
8243
|
+
SUBSCRIPTION_TIERS = {
|
|
8244
|
+
"price_1TmiHRS8aAcsk3TGwmSNfNIa": { tier: "starter", label: "Starter", price_id: "price_1TmiHRS8aAcsk3TGwmSNfNIa", monthly_usd: 12, credits_mc: 8e6, concurrency: 3, intro_coupon: "mcp-starter-1dollar-intro-12" },
|
|
8245
|
+
"price_1Tmg1nS8aAcsk3TGe8zcnGTM": { tier: "growth", label: "Growth", price_id: "price_1Tmg1nS8aAcsk3TGe8zcnGTM", monthly_usd: 100, credits_mc: 8e7, concurrency: 10, intro_coupon: null },
|
|
8246
|
+
"price_1Tmg1nS8aAcsk3TGsZw34iXS": { tier: "scale", label: "Scale", price_id: "price_1Tmg1nS8aAcsk3TGsZw34iXS", monthly_usd: 250, credits_mc: 24e7, concurrency: 20, intro_coupon: null }
|
|
8247
|
+
};
|
|
8248
|
+
SUBSCRIPTION_TIER_BY_KEY = Object.fromEntries(
|
|
8249
|
+
Object.values(SUBSCRIPTION_TIERS).map((t) => [t.tier, t])
|
|
8250
|
+
);
|
|
8251
|
+
CONCURRENCY_SLOT_PRICE_USD = 5;
|
|
8252
|
+
CONCURRENCY_SLOT_PRICE_CURRENCY = "usd";
|
|
8253
|
+
CONCURRENCY_SLOT_PRICE_INTERVAL = "month";
|
|
8254
|
+
CONCURRENCY_SLOT_PRICE_LABEL = "$5/month";
|
|
8255
|
+
CONCURRENCY_SLOT_TERMINAL_COMMAND = "npx -y -p mcp-scraper@latest mcp-scraper-cli billing concurrency checkout";
|
|
8256
|
+
FREE_SIGNUP_MC = 1e5;
|
|
8257
|
+
MEMORY_FREE_QUOTA_BYTES = 1e7 * 1;
|
|
8258
|
+
MEMORY_PLANS = {
|
|
8259
|
+
"price_1TnMSTS8aAcsk3TGBgiwuvqL": { plan: "pro", label: "Pro", price_id: "price_1TnMSTS8aAcsk3TGBgiwuvqL", interval: "month", monthly_usd: 19, quota_bytes: 5e9 },
|
|
8260
|
+
"price_1TnMSTS8aAcsk3TGWBVU2agY": { plan: "pro", label: "Pro", price_id: "price_1TnMSTS8aAcsk3TGWBVU2agY", interval: "year", monthly_usd: 19, quota_bytes: 5e9 }
|
|
8261
|
+
};
|
|
8262
|
+
MEMORY_PLAN_QUOTA = { free: MEMORY_FREE_QUOTA_BYTES, pro: 5e9, team: 5e10 };
|
|
8263
|
+
MEMORY_MARGIN_MULTIPLE = 3;
|
|
8264
|
+
MEMORY_PRO_COST_BUDGET_USD = 19 / MEMORY_MARGIN_MULTIPLE;
|
|
8265
|
+
SCHEDULING_PLANS = {
|
|
8266
|
+
"price_1ToXcHS8aAcsk3TGf3cW7zHx": { price_id: "price_1ToXcHS8aAcsk3TGf3cW7zHx", label: "Scheduled Actions", interval: "month", monthly_usd: 10, quota_per_period: 1e3 }
|
|
8267
|
+
};
|
|
8268
|
+
LedgerOperation = {
|
|
8269
|
+
TOPUP: "topup",
|
|
8270
|
+
SUBSCRIPTION: "subscription",
|
|
8271
|
+
SIGNUP_GRANT: "signup_grant",
|
|
8272
|
+
MONTHLY_REFRESH: "monthly_free_refresh",
|
|
8273
|
+
PAA: "paa",
|
|
8274
|
+
PAA_REFUND: "paa_refund",
|
|
8275
|
+
SERP: "serp",
|
|
8276
|
+
REFUND: "refund",
|
|
8277
|
+
TRANSCRIPTION: "transcription",
|
|
8278
|
+
TRANSCRIPTION_HOLD: "transcription_hold",
|
|
8279
|
+
TRANSCRIPTION_REFUND: "transcription_refund",
|
|
8280
|
+
YT_CHANNEL: "yt_channel",
|
|
8281
|
+
FB_AD: "fb_ad",
|
|
8282
|
+
MAPS_SEARCH: "maps_search",
|
|
8283
|
+
MAPS_PLACE: "maps_place",
|
|
8284
|
+
MAPS_REVIEW: "maps_review",
|
|
8285
|
+
MAPS_REVIEW_REFUND: "maps_review_refund",
|
|
8286
|
+
EXTRACT_SITE: "extract_site",
|
|
8287
|
+
EXTRACT_SITE_REFUND: "extract_site_refund",
|
|
8288
|
+
EXTRACT_URL: "page_scrape",
|
|
8289
|
+
URL_MAP: "url_map",
|
|
8290
|
+
EXTRACT_SITE_HOLD: "extract_site_hold",
|
|
8291
|
+
YT_CHANNEL_REFUND: "yt_channel_refund",
|
|
8292
|
+
FB_AD_REFUND: "fb_ad_refund",
|
|
8293
|
+
URL_MAP_REFUND: "url_map_refund",
|
|
8294
|
+
FB_SEARCH: "fb_search",
|
|
8295
|
+
FB_TRANSCRIBE: "fb_transcribe",
|
|
8296
|
+
FB_SEARCH_REFUND: "fb_search_refund",
|
|
8297
|
+
FB_TRANSCRIBE_REFUND: "fb_transcribe_refund",
|
|
8298
|
+
GOOGLE_ADS_SEARCH: "google_ads_search",
|
|
8299
|
+
GOOGLE_ADS_SEARCH_REFUND: "google_ads_search_refund",
|
|
8300
|
+
GOOGLE_ADS_INTEL: "google_ads_intel",
|
|
8301
|
+
GOOGLE_ADS_INTEL_REFUND: "google_ads_intel_refund",
|
|
8302
|
+
GOOGLE_ADS_TRANSCRIBE: "google_ads_transcribe",
|
|
8303
|
+
GOOGLE_ADS_TRANSCRIBE_REFUND: "google_ads_transcribe_refund",
|
|
8304
|
+
INSTAGRAM_PROFILE: "instagram_profile",
|
|
8305
|
+
INSTAGRAM_PROFILE_REFUND: "instagram_profile_refund",
|
|
8306
|
+
INSTAGRAM_MEDIA: "instagram_media",
|
|
8307
|
+
INSTAGRAM_MEDIA_REFUND: "instagram_media_refund",
|
|
8308
|
+
INSTAGRAM_TRANSCRIBE: "instagram_transcribe",
|
|
8309
|
+
INSTAGRAM_TRANSCRIBE_REFUND: "instagram_transcribe_refund",
|
|
8310
|
+
BROWSER_SESSION: "browser_session",
|
|
8311
|
+
REDDIT_THREAD: "reddit_thread",
|
|
8312
|
+
REDDIT_THREAD_REFUND: "reddit_thread_refund",
|
|
8313
|
+
REDDIT_COMMENT: "reddit_comment",
|
|
8314
|
+
REDDIT_COMMENT_REFUND: "reddit_comment_refund",
|
|
8315
|
+
VIDEO_ANALYSIS: "video_analysis",
|
|
8316
|
+
VIDEO_ANALYSIS_REFUND: "video_analysis_refund",
|
|
8317
|
+
VIDEO_ANALYSIS_ADJUST: "video_analysis_adjust",
|
|
8318
|
+
VIDEO_ANALYSIS_LLM: "video_analysis_llm",
|
|
8319
|
+
MEMORY_AI: "memory_ai",
|
|
8320
|
+
MEMORY_AI_REFUND: "memory_ai_refund",
|
|
8321
|
+
TRUSTPILOT_REVIEWS: "trustpilot_reviews",
|
|
8322
|
+
TRUSTPILOT_REVIEWS_REFUND: "trustpilot_reviews_refund",
|
|
8323
|
+
TRUSTPILOT_REVIEW: "trustpilot_review",
|
|
8324
|
+
TRUSTPILOT_REVIEW_REFUND: "trustpilot_review_refund",
|
|
8325
|
+
G2_REVIEWS: "g2_reviews",
|
|
8326
|
+
G2_REVIEWS_REFUND: "g2_reviews_refund",
|
|
8327
|
+
G2_REVIEW: "g2_review",
|
|
8328
|
+
G2_REVIEW_REFUND: "g2_review_refund",
|
|
8329
|
+
DIFF_PAGE: "diff_page",
|
|
8330
|
+
DIFF_PAGE_REFUND: "diff_page_refund",
|
|
8331
|
+
SITE_AUDIT_LLM: "site_audit_llm"
|
|
8332
|
+
};
|
|
8333
|
+
MEMORY_AI_MARGIN_MULTIPLE = 3;
|
|
8334
|
+
SITE_AUDIT_LLM_MARGIN_MULTIPLE = 1.5;
|
|
8335
|
+
VIDEO_ANALYSIS_LLM_MARGIN_MULTIPLE = 1.5;
|
|
8336
|
+
MC_PER_USD = SUBSCRIPTION_TIERS["price_1TmiHRS8aAcsk3TGwmSNfNIa"].credits_mc / SUBSCRIPTION_TIERS["price_1TmiHRS8aAcsk3TGwmSNfNIa"].monthly_usd;
|
|
8337
|
+
}
|
|
8338
|
+
});
|
|
8339
|
+
|
|
7931
8340
|
// src/services/site-architecture-auditor/prompts/ingest-validate.ts
|
|
7932
8341
|
function buildIngestValidatePrompt(input) {
|
|
7933
8342
|
const sessionPath = `<session-path-from-job>`;
|
|
@@ -8951,6 +9360,9 @@ var init_site_audit_service = __esm({
|
|
|
8951
9360
|
import_p_limit = __toESM(require("p-limit"), 1);
|
|
8952
9361
|
init_llm_parse_with_retry();
|
|
8953
9362
|
init_schemas();
|
|
9363
|
+
init_db();
|
|
9364
|
+
init_rates();
|
|
9365
|
+
init_cost_context();
|
|
8954
9366
|
init_ingest_validate();
|
|
8955
9367
|
init_enrich_competitor();
|
|
8956
9368
|
init_build_graph_per_site();
|
|
@@ -9009,8 +9421,23 @@ var init_site_audit_service = __esm({
|
|
|
9009
9421
|
async getJobsForUser(userId) {
|
|
9010
9422
|
return this.deps.repo.listSiteAuditJobs(userId);
|
|
9011
9423
|
}
|
|
9424
|
+
async runLlmBilled(userId, source, fn) {
|
|
9425
|
+
const before = this.deps.llm.totalCostUsd;
|
|
9426
|
+
const result = await runWithCostContext({ ...currentCostContext(), op: "audit_site", userId }, fn);
|
|
9427
|
+
const deltaCostUsd = this.deps.llm.totalCostUsd - before;
|
|
9428
|
+
if (deltaCostUsd <= 0) return result;
|
|
9429
|
+
const mc = Math.round(deltaCostUsd * SITE_AUDIT_LLM_MARGIN_MULTIPLE * MC_PER_USD);
|
|
9430
|
+
if (mc <= 0) return result;
|
|
9431
|
+
const { ok } = await debitMc(userId, mc, LedgerOperation.SITE_AUDIT_LLM, source);
|
|
9432
|
+
if (!ok) {
|
|
9433
|
+
console.warn(`[site-audit-service] SITE_AUDIT_LLM surcharge debit failed for user ${userId} (${source}): insufficient balance for ${mc}mc`);
|
|
9434
|
+
}
|
|
9435
|
+
return result;
|
|
9436
|
+
}
|
|
9012
9437
|
async runIngestPhase(jobId, payload) {
|
|
9013
9438
|
const { request } = payload;
|
|
9439
|
+
const job = await this.deps.repo.getSiteAuditJob(jobId);
|
|
9440
|
+
if (!job) throw new Error(`[site-audit-service] runIngestPhase: job ${jobId} not found`);
|
|
9014
9441
|
const input = {
|
|
9015
9442
|
clientDomain: request.clientDomain,
|
|
9016
9443
|
sfInternalPath: request.sfInternalPath,
|
|
@@ -9025,14 +9452,18 @@ var init_site_audit_service = __esm({
|
|
|
9025
9452
|
businessContext: request.businessContext,
|
|
9026
9453
|
priorAuditMemoryEnabled: request.priorAuditMemoryEnabled
|
|
9027
9454
|
};
|
|
9028
|
-
const result = await
|
|
9029
|
-
|
|
9030
|
-
|
|
9031
|
-
|
|
9032
|
-
|
|
9033
|
-
|
|
9034
|
-
|
|
9035
|
-
|
|
9455
|
+
const result = await this.runLlmBilled(
|
|
9456
|
+
job.user_id,
|
|
9457
|
+
`${jobId}:phase1-ingest`,
|
|
9458
|
+
() => llmParseWithRetry(
|
|
9459
|
+
ValidatedFileManifestSchema,
|
|
9460
|
+
(attempt, error) => {
|
|
9461
|
+
void attempt;
|
|
9462
|
+
void error;
|
|
9463
|
+
return buildIngestValidatePrompt(input);
|
|
9464
|
+
},
|
|
9465
|
+
this.deps.llm
|
|
9466
|
+
)
|
|
9036
9467
|
);
|
|
9037
9468
|
await this.deps.repo.logSiteAuditPhaseComplete(jobId, "phase1-ingest", result);
|
|
9038
9469
|
}
|
|
@@ -9047,23 +9478,27 @@ var init_site_audit_service = __esm({
|
|
|
9047
9478
|
const competitorDomainsCsvPath = request.competitorDomainsCsvPath;
|
|
9048
9479
|
void competitorDomainsCsvPath;
|
|
9049
9480
|
const competitors = await this._readCompetitorDomains(request);
|
|
9050
|
-
const results = await
|
|
9051
|
-
|
|
9052
|
-
|
|
9053
|
-
|
|
9054
|
-
|
|
9055
|
-
|
|
9056
|
-
|
|
9057
|
-
|
|
9058
|
-
(
|
|
9059
|
-
|
|
9060
|
-
|
|
9061
|
-
|
|
9062
|
-
|
|
9063
|
-
|
|
9064
|
-
|
|
9065
|
-
|
|
9066
|
-
|
|
9481
|
+
const results = await this.runLlmBilled(
|
|
9482
|
+
job.user_id,
|
|
9483
|
+
`${jobId}:phase1-enrich`,
|
|
9484
|
+
() => Promise.all(
|
|
9485
|
+
competitors.map(
|
|
9486
|
+
(domain, idx) => limit(async () => {
|
|
9487
|
+
const competitorIndex = idx + 1;
|
|
9488
|
+
const rawPrompt = buildEnrichCompetitorPrompt({ domain, sessionPath });
|
|
9489
|
+
const prompt = rawPrompt.replace("<competitor-index>", String(competitorIndex));
|
|
9490
|
+
const result = await llmParseWithRetry(
|
|
9491
|
+
EnrichCompetitorOutputSchema,
|
|
9492
|
+
(attempt, error) => {
|
|
9493
|
+
void attempt;
|
|
9494
|
+
void error;
|
|
9495
|
+
return prompt;
|
|
9496
|
+
},
|
|
9497
|
+
this.deps.llm
|
|
9498
|
+
);
|
|
9499
|
+
return result;
|
|
9500
|
+
})
|
|
9501
|
+
)
|
|
9067
9502
|
)
|
|
9068
9503
|
);
|
|
9069
9504
|
await this.deps.repo.logSiteAuditPhaseComplete(jobId, "phase1-ingest", { competitors: results });
|
|
@@ -9077,19 +9512,23 @@ var init_site_audit_service = __esm({
|
|
|
9077
9512
|
const clientDomain = request.clientDomain;
|
|
9078
9513
|
const urlInventory = [];
|
|
9079
9514
|
const directedEdgeCount = 0;
|
|
9080
|
-
const orphanAnnotation = await
|
|
9081
|
-
|
|
9082
|
-
|
|
9083
|
-
|
|
9084
|
-
|
|
9085
|
-
|
|
9086
|
-
|
|
9087
|
-
|
|
9088
|
-
|
|
9089
|
-
|
|
9090
|
-
|
|
9091
|
-
|
|
9092
|
-
|
|
9515
|
+
const orphanAnnotation = await this.runLlmBilled(
|
|
9516
|
+
job.user_id,
|
|
9517
|
+
`${jobId}:phase2-build-graph`,
|
|
9518
|
+
() => llmParseWithRetry(
|
|
9519
|
+
OrphanAnnotationOutputSchema,
|
|
9520
|
+
(attempt, error) => {
|
|
9521
|
+
void attempt;
|
|
9522
|
+
void error;
|
|
9523
|
+
return buildGraphPerSitePrompt({
|
|
9524
|
+
siteId: "client",
|
|
9525
|
+
domain: clientDomain,
|
|
9526
|
+
urlInventory,
|
|
9527
|
+
directedEdgeCount
|
|
9528
|
+
});
|
|
9529
|
+
},
|
|
9530
|
+
this.deps.llm
|
|
9531
|
+
)
|
|
9093
9532
|
);
|
|
9094
9533
|
const graphMetrics = await this.deps.python.runGraphMetrics({
|
|
9095
9534
|
directedLinkGraphPath: `${sessionPath}/phase1/directed_link_graph.json`,
|
|
@@ -9134,50 +9573,52 @@ var init_site_audit_service = __esm({
|
|
|
9134
9573
|
}
|
|
9135
9574
|
if (currentBatch.length > 0) batches.push(currentBatch);
|
|
9136
9575
|
const usZipsCsvPath = process.env["US_ZIPS_CSV_PATH"] ?? "/Users/vilovieta/Downloads/sales-magician-api-leads-magician-01c6cff78e31/tools/analytics/data/uszips.csv";
|
|
9137
|
-
|
|
9138
|
-
|
|
9139
|
-
|
|
9140
|
-
|
|
9141
|
-
|
|
9142
|
-
|
|
9143
|
-
|
|
9144
|
-
|
|
9145
|
-
|
|
9146
|
-
|
|
9147
|
-
|
|
9148
|
-
|
|
9149
|
-
|
|
9150
|
-
|
|
9151
|
-
|
|
9152
|
-
|
|
9153
|
-
|
|
9154
|
-
|
|
9155
|
-
|
|
9156
|
-
|
|
9157
|
-
|
|
9158
|
-
|
|
9159
|
-
|
|
9160
|
-
|
|
9161
|
-
|
|
9162
|
-
|
|
9163
|
-
|
|
9164
|
-
|
|
9165
|
-
|
|
9166
|
-
|
|
9167
|
-
|
|
9168
|
-
|
|
9169
|
-
|
|
9170
|
-
|
|
9171
|
-
|
|
9172
|
-
|
|
9173
|
-
|
|
9174
|
-
|
|
9175
|
-
|
|
9176
|
-
|
|
9177
|
-
|
|
9178
|
-
|
|
9576
|
+
await this.runLlmBilled(job.user_id, `${jobId}:phase3-classify`, async () => {
|
|
9577
|
+
for (const batch of batches) {
|
|
9578
|
+
let classified = [];
|
|
9579
|
+
let unclassifiedUrls = batch.map((u) => u.url);
|
|
9580
|
+
try {
|
|
9581
|
+
const pythonResult = await this.deps.python.runLocationClassifier({
|
|
9582
|
+
urls: batch.map((u) => u.url),
|
|
9583
|
+
usZipsCsvPath
|
|
9584
|
+
});
|
|
9585
|
+
classified = pythonResult.classified;
|
|
9586
|
+
unclassifiedUrls = pythonResult.unclassified;
|
|
9587
|
+
} catch (err) {
|
|
9588
|
+
console.warn(
|
|
9589
|
+
"[site-audit-service] runClassifyPhase: python.runLocationClassifier failed (falling back to LLM):",
|
|
9590
|
+
err instanceof Error ? err.message : String(err)
|
|
9591
|
+
);
|
|
9592
|
+
unclassifiedUrls = batch.map((u) => u.url);
|
|
9593
|
+
}
|
|
9594
|
+
for (const c of classified) {
|
|
9595
|
+
allClassifications.push({
|
|
9596
|
+
url: c.url,
|
|
9597
|
+
page_type: c.pageType,
|
|
9598
|
+
confidence: c.confidence > 0.6 ? "high" : c.confidence > 0.4 ? "medium" : "low",
|
|
9599
|
+
_classification_label: "[COMPUTED:python/location-classifier]",
|
|
9600
|
+
topic_cluster: null,
|
|
9601
|
+
location_signals: c.locationSignals
|
|
9602
|
+
});
|
|
9603
|
+
}
|
|
9604
|
+
if (unclassifiedUrls.length > 0) {
|
|
9605
|
+
const unclassifiedBatch = batch.filter((u) => unclassifiedUrls.includes(u.url));
|
|
9606
|
+
const llmResults = await llmParseWithRetry(
|
|
9607
|
+
import_zod10.z.array(PageTypeClassificationRowSchema),
|
|
9608
|
+
(attempt, error) => {
|
|
9609
|
+
void attempt;
|
|
9610
|
+
void error;
|
|
9611
|
+
return buildMetricsClassifyPrompt({
|
|
9612
|
+
urls: unclassifiedBatch,
|
|
9613
|
+
siteDomain: clientDomain
|
|
9614
|
+
});
|
|
9615
|
+
},
|
|
9616
|
+
this.deps.llm
|
|
9617
|
+
);
|
|
9618
|
+
allClassifications.push(...llmResults);
|
|
9619
|
+
}
|
|
9179
9620
|
}
|
|
9180
|
-
}
|
|
9621
|
+
});
|
|
9181
9622
|
void sessionPath;
|
|
9182
9623
|
await this.deps.repo.logSiteAuditPhaseComplete(jobId, "phase3-classify", {
|
|
9183
9624
|
classifications: allClassifications,
|
|
@@ -9190,20 +9631,24 @@ var init_site_audit_service = __esm({
|
|
|
9190
9631
|
if (!job) throw new Error(`[site-audit-service] runComparePhase: job ${jobId} not found`);
|
|
9191
9632
|
const request = JSON.parse(job.request);
|
|
9192
9633
|
const clientDomain = request.clientDomain;
|
|
9193
|
-
const result = await
|
|
9194
|
-
|
|
9195
|
-
|
|
9196
|
-
|
|
9197
|
-
|
|
9198
|
-
|
|
9199
|
-
|
|
9200
|
-
|
|
9201
|
-
|
|
9202
|
-
|
|
9203
|
-
|
|
9204
|
-
|
|
9205
|
-
|
|
9206
|
-
|
|
9634
|
+
const result = await this.runLlmBilled(
|
|
9635
|
+
job.user_id,
|
|
9636
|
+
`${jobId}:phase4-compare`,
|
|
9637
|
+
() => llmParseWithRetry(
|
|
9638
|
+
CompareRecommendOutputSchema,
|
|
9639
|
+
(attempt, error) => {
|
|
9640
|
+
void attempt;
|
|
9641
|
+
void error;
|
|
9642
|
+
return buildCompareRecommendPrompt({
|
|
9643
|
+
clientDomain,
|
|
9644
|
+
clientMoneyPages: [],
|
|
9645
|
+
competitorSites: [],
|
|
9646
|
+
clientUrlMetrics: [],
|
|
9647
|
+
clientPageTypes: []
|
|
9648
|
+
});
|
|
9649
|
+
},
|
|
9650
|
+
this.deps.llm
|
|
9651
|
+
)
|
|
9207
9652
|
);
|
|
9208
9653
|
await this.deps.repo.logSiteAuditPhaseComplete(jobId, "phase4-compare", result);
|
|
9209
9654
|
return result;
|
|
@@ -9233,20 +9678,24 @@ var init_site_audit_service = __esm({
|
|
|
9233
9678
|
total: ilesTotal,
|
|
9234
9679
|
components: ilesComponents
|
|
9235
9680
|
};
|
|
9236
|
-
const reportMarkdown = await
|
|
9237
|
-
|
|
9238
|
-
|
|
9239
|
-
|
|
9240
|
-
|
|
9241
|
-
|
|
9242
|
-
|
|
9243
|
-
|
|
9244
|
-
|
|
9245
|
-
|
|
9246
|
-
|
|
9247
|
-
|
|
9248
|
-
|
|
9249
|
-
|
|
9681
|
+
const reportMarkdown = await this.runLlmBilled(
|
|
9682
|
+
job.user_id,
|
|
9683
|
+
`${jobId}:phase5-synthesize`,
|
|
9684
|
+
() => llmParseWithRetry(
|
|
9685
|
+
import_zod10.z.string(),
|
|
9686
|
+
(attempt, error) => {
|
|
9687
|
+
void attempt;
|
|
9688
|
+
void error;
|
|
9689
|
+
return buildScoreSynthesizePrompt({
|
|
9690
|
+
preComputedArchitectureScore,
|
|
9691
|
+
preComputedIlesScore,
|
|
9692
|
+
rankedBacklog: [],
|
|
9693
|
+
clientDomain,
|
|
9694
|
+
sessionPath
|
|
9695
|
+
});
|
|
9696
|
+
},
|
|
9697
|
+
this.deps.llm
|
|
9698
|
+
)
|
|
9250
9699
|
);
|
|
9251
9700
|
const auditResult = SiteAuditResultSchema.parse({
|
|
9252
9701
|
jobId,
|
|
@@ -10031,375 +10480,6 @@ var init_image_audit = __esm({
|
|
|
10031
10480
|
}
|
|
10032
10481
|
});
|
|
10033
10482
|
|
|
10034
|
-
// src/api/rates.ts
|
|
10035
|
-
function browserActiveCostMc(activeMs) {
|
|
10036
|
-
return Math.round(activeMs * MC_PER_BROWSER_MS);
|
|
10037
|
-
}
|
|
10038
|
-
function concurrencySlotBillingInfo() {
|
|
10039
|
-
const billingUrl = process.env.CONCURRENCY_BILLING_URL ?? "https://mcpscraper.dev/billing";
|
|
10040
|
-
return {
|
|
10041
|
-
product: "Extra concurrency slot",
|
|
10042
|
-
price_label: CONCURRENCY_SLOT_PRICE_LABEL,
|
|
10043
|
-
unit_amount_usd: CONCURRENCY_SLOT_PRICE_USD,
|
|
10044
|
-
currency: CONCURRENCY_SLOT_PRICE_CURRENCY,
|
|
10045
|
-
interval: CONCURRENCY_SLOT_PRICE_INTERVAL,
|
|
10046
|
-
billing_url: billingUrl,
|
|
10047
|
-
terminal_command: CONCURRENCY_SLOT_TERMINAL_COMMAND,
|
|
10048
|
-
terminal_command_with_api_key_env: `MCP_SCRAPER_API_KEY=sk_live_your_key ${CONCURRENCY_SLOT_TERMINAL_COMMAND}`
|
|
10049
|
-
};
|
|
10050
|
-
}
|
|
10051
|
-
function mcToCredits(mc) {
|
|
10052
|
-
return mc / MC_PER_CREDIT;
|
|
10053
|
-
}
|
|
10054
|
-
function insufficientBalanceResponse(balanceMc, requiredMc) {
|
|
10055
|
-
const topupUrl = process.env.TOPUP_URL ?? "https://mcpscraper.dev/billing";
|
|
10056
|
-
const balanceCredits = mcToCredits(balanceMc);
|
|
10057
|
-
const requiredCredits = mcToCredits(requiredMc);
|
|
10058
|
-
return {
|
|
10059
|
-
error: "insufficient_balance",
|
|
10060
|
-
error_code: "insufficient_balance",
|
|
10061
|
-
message: `Insufficient credits. Balance: ${balanceCredits} credits. This call requires ${requiredCredits} credits. Top up at ${topupUrl}`,
|
|
10062
|
-
balance_credits: balanceCredits,
|
|
10063
|
-
required_credits: requiredCredits,
|
|
10064
|
-
topup_url: topupUrl
|
|
10065
|
-
};
|
|
10066
|
-
}
|
|
10067
|
-
var MC_COSTS, MC_PER_BROWSER_MS, BROWSER_OPEN_MIN_BALANCE_MC, MC_PER_CREDIT, CREDIT_COST_CATALOG, CONCURRENCY_PRICE_ID, SUBSCRIPTION_TIERS, SUBSCRIPTION_TIER_BY_KEY, CONCURRENCY_SLOT_PRICE_USD, CONCURRENCY_SLOT_PRICE_CURRENCY, CONCURRENCY_SLOT_PRICE_INTERVAL, CONCURRENCY_SLOT_PRICE_LABEL, CONCURRENCY_SLOT_TERMINAL_COMMAND, FREE_SIGNUP_MC, MEMORY_FREE_QUOTA_BYTES, MEMORY_PLANS, MEMORY_PLAN_QUOTA, MEMORY_MARGIN_MULTIPLE, MEMORY_PRO_COST_BUDGET_USD, SCHEDULING_PLANS, LedgerOperation, MEMORY_AI_MARGIN_MULTIPLE, MC_PER_USD;
|
|
10068
|
-
var init_rates = __esm({
|
|
10069
|
-
"src/api/rates.ts"() {
|
|
10070
|
-
"use strict";
|
|
10071
|
-
MC_COSTS = {
|
|
10072
|
-
serp: 100,
|
|
10073
|
-
paa: 200,
|
|
10074
|
-
paa_base: 1e3,
|
|
10075
|
-
page_scrape: 100,
|
|
10076
|
-
url_map: 500,
|
|
10077
|
-
yt_channel: 200,
|
|
10078
|
-
yt_transcription: 200,
|
|
10079
|
-
fb_ad: 200,
|
|
10080
|
-
maps_search: 500,
|
|
10081
|
-
maps_place: 3e3,
|
|
10082
|
-
maps_review: 100,
|
|
10083
|
-
fb_search: 600,
|
|
10084
|
-
fb_transcribe: 200,
|
|
10085
|
-
google_ads_search: 600,
|
|
10086
|
-
google_ads_intel: 200,
|
|
10087
|
-
google_ads_transcribe: 200,
|
|
10088
|
-
instagram_profile: 400,
|
|
10089
|
-
instagram_media: 400,
|
|
10090
|
-
instagram_transcribe: 50,
|
|
10091
|
-
browser_minute: 12e3,
|
|
10092
|
-
reddit_thread: 2e3,
|
|
10093
|
-
video_analysis: 666667,
|
|
10094
|
-
trustpilot_reviews: 500,
|
|
10095
|
-
trustpilot_review: 100,
|
|
10096
|
-
g2_reviews: 500,
|
|
10097
|
-
g2_review: 150,
|
|
10098
|
-
diff_page: 100
|
|
10099
|
-
};
|
|
10100
|
-
MC_PER_BROWSER_MS = MC_COSTS.browser_minute / 6e4;
|
|
10101
|
-
BROWSER_OPEN_MIN_BALANCE_MC = 1e3;
|
|
10102
|
-
MC_PER_CREDIT = 1e3;
|
|
10103
|
-
CREDIT_COST_CATALOG = [
|
|
10104
|
-
{
|
|
10105
|
-
key: "serp",
|
|
10106
|
-
label: "SERP search",
|
|
10107
|
-
aliases: ["search_serp", "serp", "google search", "organic results"],
|
|
10108
|
-
credits: mcToCredits(MC_COSTS.serp),
|
|
10109
|
-
unit: "per search",
|
|
10110
|
-
notes: "Returns AI Overview, PAA snippet, videos, forums, and local pack."
|
|
10111
|
-
},
|
|
10112
|
-
{
|
|
10113
|
-
key: "paa",
|
|
10114
|
-
label: "PAA harvest",
|
|
10115
|
-
aliases: ["harvest_paa", "paa", "people also ask", "questions"],
|
|
10116
|
-
credits: mcToCredits(MC_COSTS.paa),
|
|
10117
|
-
unit: `per question (+${mcToCredits(MC_COSTS.paa_base)} credit base)`,
|
|
10118
|
-
notes: `Full SERP feature extraction. Billed ${mcToCredits(MC_COSTS.paa_base)} credit base + ${mcToCredits(MC_COSTS.paa)} per question actually returned (unused estimate refunded).`
|
|
10119
|
-
},
|
|
10120
|
-
{
|
|
10121
|
-
key: "page_scrape",
|
|
10122
|
-
label: "Page crawl / extract",
|
|
10123
|
-
aliases: ["extract_url", "extract_site", "page scrape", "url scrape", "single page", "site crawl"],
|
|
10124
|
-
credits: mcToCredits(MC_COSTS.page_scrape),
|
|
10125
|
-
unit: "per page",
|
|
10126
|
-
notes: "Applies to both single-URL extraction and per-page site crawls."
|
|
10127
|
-
},
|
|
10128
|
-
{
|
|
10129
|
-
key: "url_map",
|
|
10130
|
-
label: "Site URL mapping",
|
|
10131
|
-
aliases: ["map_site_urls", "url map", "site map", "crawl urls"],
|
|
10132
|
-
credits: mcToCredits(MC_COSTS.url_map),
|
|
10133
|
-
unit: "per mapping operation",
|
|
10134
|
-
notes: "Flat rate for the full /map-urls call regardless of URL count discovered."
|
|
10135
|
-
},
|
|
10136
|
-
{
|
|
10137
|
-
key: "yt_channel",
|
|
10138
|
-
label: "YouTube search / channel harvest",
|
|
10139
|
-
aliases: ["youtube_harvest", "youtube search", "youtube channel", "yt_channel"],
|
|
10140
|
-
credits: mcToCredits(MC_COSTS.yt_channel),
|
|
10141
|
-
unit: "per call"
|
|
10142
|
-
},
|
|
10143
|
-
{
|
|
10144
|
-
key: "yt_transcription",
|
|
10145
|
-
label: "YouTube transcription",
|
|
10146
|
-
aliases: ["youtube_transcribe", "youtube transcript", "transcription", "yt_transcription"],
|
|
10147
|
-
credits: mcToCredits(MC_COSTS.yt_transcription),
|
|
10148
|
-
unit: "per minute",
|
|
10149
|
-
notes: "A 5-minute hold is taken, then reconciled to actual video duration."
|
|
10150
|
-
},
|
|
10151
|
-
{
|
|
10152
|
-
key: "fb_ad",
|
|
10153
|
-
label: "Facebook search / ad lookup",
|
|
10154
|
-
aliases: ["facebook_page_intel", "facebook_ad_search", "facebook_ad", "facebook ads", "fb ads"],
|
|
10155
|
-
credits: mcToCredits(MC_COSTS.fb_ad),
|
|
10156
|
-
unit: "per call"
|
|
10157
|
-
},
|
|
10158
|
-
{
|
|
10159
|
-
key: "maps_search",
|
|
10160
|
-
label: "Maps business search",
|
|
10161
|
-
aliases: ["maps_search", "google maps search", "gmb search", "gbp search", "business profiles"],
|
|
10162
|
-
credits: mcToCredits(MC_COSTS.maps_search),
|
|
10163
|
-
unit: "per search",
|
|
10164
|
-
notes: "Returns up to 50 Google Maps business/profile candidates. Use maps_place_intel to hydrate selected businesses."
|
|
10165
|
-
},
|
|
10166
|
-
{
|
|
10167
|
-
key: "maps_place",
|
|
10168
|
-
label: "Maps business lookup",
|
|
10169
|
-
aliases: ["maps_place_intel", "google maps", "maps place", "place intel"],
|
|
10170
|
-
credits: mcToCredits(MC_COSTS.maps_place),
|
|
10171
|
-
unit: "per business",
|
|
10172
|
-
notes: "Base lookup. Reviews billed separately per card at maps_review rate."
|
|
10173
|
-
},
|
|
10174
|
-
{
|
|
10175
|
-
key: "maps_review",
|
|
10176
|
-
label: "Maps review",
|
|
10177
|
-
aliases: ["maps_reviews", "google reviews", "review cards", "reviews"],
|
|
10178
|
-
credits: mcToCredits(MC_COSTS.maps_review),
|
|
10179
|
-
unit: "per review card",
|
|
10180
|
-
notes: "Charged after extraction when includeReviews is true."
|
|
10181
|
-
},
|
|
10182
|
-
{
|
|
10183
|
-
key: "fb_search",
|
|
10184
|
-
label: "Facebook ad library search",
|
|
10185
|
-
aliases: ["facebook_search", "fb_search", "fb ad search"],
|
|
10186
|
-
credits: mcToCredits(MC_COSTS.fb_search),
|
|
10187
|
-
unit: "per search",
|
|
10188
|
-
notes: "Browser automation to search Facebook Ads Library by keyword."
|
|
10189
|
-
},
|
|
10190
|
-
{
|
|
10191
|
-
key: "google_ads_search",
|
|
10192
|
-
label: "Google Ads Transparency search",
|
|
10193
|
-
aliases: ["google_ads_search", "google ads search", "ads transparency search"],
|
|
10194
|
-
credits: mcToCredits(MC_COSTS.google_ads_search),
|
|
10195
|
-
unit: "per search",
|
|
10196
|
-
notes: "Browser automation to find advertisers in Google Ads Transparency Center by domain or name."
|
|
10197
|
-
},
|
|
10198
|
-
{
|
|
10199
|
-
key: "google_ads_intel",
|
|
10200
|
-
label: "Google Ads Transparency advertiser intel",
|
|
10201
|
-
aliases: ["google_ads_page_intel", "google ads intel", "ads transparency intel"],
|
|
10202
|
-
credits: mcToCredits(MC_COSTS.google_ads_intel),
|
|
10203
|
-
unit: "per call",
|
|
10204
|
-
notes: "Lists and hydrates an advertiser's creatives with image URLs and video references."
|
|
10205
|
-
},
|
|
10206
|
-
{
|
|
10207
|
-
key: "google_ads_transcribe",
|
|
10208
|
-
label: "Google ad video transcription",
|
|
10209
|
-
aliases: ["google_ads_transcribe", "google ad transcript"],
|
|
10210
|
-
credits: mcToCredits(MC_COSTS.google_ads_transcribe),
|
|
10211
|
-
unit: "per minute",
|
|
10212
|
-
notes: "A hold is taken, then reconciled to actual video duration."
|
|
10213
|
-
},
|
|
10214
|
-
{
|
|
10215
|
-
key: "fb_transcribe",
|
|
10216
|
-
label: "Facebook video / ad transcription",
|
|
10217
|
-
aliases: ["facebook_transcribe", "facebook_video_transcribe", "fb_transcribe", "fb ad transcript"],
|
|
10218
|
-
credits: mcToCredits(MC_COSTS.fb_transcribe),
|
|
10219
|
-
unit: "per minute",
|
|
10220
|
-
notes: "A hold is taken, then reconciled to actual video duration."
|
|
10221
|
-
},
|
|
10222
|
-
{
|
|
10223
|
-
key: "instagram_profile",
|
|
10224
|
-
label: "Instagram profile content discovery",
|
|
10225
|
-
aliases: ["instagram_profile_content", "instagram profile", "instagram content list", "ig profile"],
|
|
10226
|
-
credits: mcToCredits(MC_COSTS.instagram_profile),
|
|
10227
|
-
unit: "per profile scan",
|
|
10228
|
-
notes: "Browser extraction of public Instagram profile grid links. Complete history may require a logged-in profile."
|
|
10229
|
-
},
|
|
10230
|
-
{
|
|
10231
|
-
key: "instagram_media",
|
|
10232
|
-
label: "Instagram media download",
|
|
10233
|
-
aliases: ["instagram_media_download", "instagram reel download", "instagram post download", "ig media"],
|
|
10234
|
-
credits: mcToCredits(MC_COSTS.instagram_media),
|
|
10235
|
-
unit: "per post or reel",
|
|
10236
|
-
notes: "Extracts post text, image URLs, and reel audio/video tracks, with local downloads when the server can write files."
|
|
10237
|
-
},
|
|
10238
|
-
{
|
|
10239
|
-
key: "instagram_transcribe",
|
|
10240
|
-
label: "Instagram media transcription",
|
|
10241
|
-
aliases: ["instagram transcript", "instagram reel transcript", "ig transcribe"],
|
|
10242
|
-
credits: mcToCredits(MC_COSTS.instagram_transcribe),
|
|
10243
|
-
unit: "per call",
|
|
10244
|
-
notes: "Whisper transcription of selected Instagram audio/video media via fal.ai."
|
|
10245
|
-
},
|
|
10246
|
-
{
|
|
10247
|
-
key: "browser_minute",
|
|
10248
|
-
label: "Interactive browser session",
|
|
10249
|
-
aliases: ["browser_open", "browser agent", "browser_agent", "live browser", "browse", "browser control", "interactive browser"],
|
|
10250
|
-
credits: mcToCredits(MC_COSTS.browser_minute),
|
|
10251
|
-
unit: "per minute of use",
|
|
10252
|
-
notes: "Metered per minute of use for the whole time the browser session is open. Close the session to stop the meter; abandoned sessions are auto-closed after a short idle window."
|
|
10253
|
-
},
|
|
10254
|
-
{
|
|
10255
|
-
key: "reddit_thread",
|
|
10256
|
-
label: "Reddit thread + comments",
|
|
10257
|
-
aliases: ["reddit", "reddit_thread", "reddit comments", "subreddit", "reddit post"],
|
|
10258
|
-
credits: mcToCredits(MC_COSTS.reddit_thread),
|
|
10259
|
-
unit: "per thread",
|
|
10260
|
-
notes: "Captures a Reddit post and its comment tree. Charged on success; refunded if the thread cannot be retrieved."
|
|
10261
|
-
},
|
|
10262
|
-
{
|
|
10263
|
-
key: "video_analysis",
|
|
10264
|
-
label: "Video breakdown (frame-by-frame + transcript)",
|
|
10265
|
-
aliases: ["video_frame_analysis", "video breakdown", "video analysis", "analyze video"],
|
|
10266
|
-
credits: mcToCredits(MC_COSTS.video_analysis),
|
|
10267
|
-
unit: "per 120 frames (max 480)",
|
|
10268
|
-
notes: "Full multi-lens video breakdown: samples frames across a video (up to 30 minutes), analyzes each with vision AI, transcribes the audio, then produces summary, pacing/energy, words-per-minute, topic outline, key points, hook analysis, visual style, and a how-to-replicate recipe with a quality-control pass. $1 per 120 frames requested (max 480 = $4); billed down automatically if the video cannot use the requested frames, and refunded fully if the run fails."
|
|
10269
|
-
},
|
|
10270
|
-
{
|
|
10271
|
-
key: "trustpilot_reviews",
|
|
10272
|
-
label: "Trustpilot review harvest",
|
|
10273
|
-
aliases: ["trustpilot_reviews", "trustpilot", "trustpilot reviews"],
|
|
10274
|
-
credits: mcToCredits(MC_COSTS.trustpilot_reviews),
|
|
10275
|
-
unit: "per call",
|
|
10276
|
-
notes: "Base lookup. Reviews billed separately per card at trustpilot_review rate. Refunded if no reviews are found."
|
|
10277
|
-
},
|
|
10278
|
-
{
|
|
10279
|
-
key: "trustpilot_review",
|
|
10280
|
-
label: "Trustpilot review card",
|
|
10281
|
-
aliases: ["trustpilot_review", "trustpilot card"],
|
|
10282
|
-
credits: mcToCredits(MC_COSTS.trustpilot_review),
|
|
10283
|
-
unit: "per review card",
|
|
10284
|
-
notes: "Charged per review actually extracted, billed down automatically if the balance runs out mid-page."
|
|
10285
|
-
},
|
|
10286
|
-
{
|
|
10287
|
-
key: "g2_reviews",
|
|
10288
|
-
label: "G2 review harvest",
|
|
10289
|
-
aliases: ["g2_reviews", "g2", "g2 reviews"],
|
|
10290
|
-
credits: mcToCredits(MC_COSTS.g2_reviews),
|
|
10291
|
-
unit: "per call",
|
|
10292
|
-
notes: "Base lookup. Reviews billed separately per card at g2_review rate. Refunded if no reviews are found."
|
|
10293
|
-
},
|
|
10294
|
-
{
|
|
10295
|
-
key: "g2_review",
|
|
10296
|
-
label: "G2 review card",
|
|
10297
|
-
aliases: ["g2_review", "g2 card"],
|
|
10298
|
-
credits: mcToCredits(MC_COSTS.g2_review),
|
|
10299
|
-
unit: "per review card",
|
|
10300
|
-
notes: "Charged per review actually extracted (each card carries up to 3 Q&A sections), billed down automatically if the balance runs out mid-page."
|
|
10301
|
-
},
|
|
10302
|
-
{
|
|
10303
|
-
key: "diff_page",
|
|
10304
|
-
label: "Page change check",
|
|
10305
|
-
aliases: ["diff_page", "page diff", "change detection"],
|
|
10306
|
-
credits: mcToCredits(MC_COSTS.diff_page),
|
|
10307
|
-
unit: "per check",
|
|
10308
|
-
notes: "Same cost as a single page extract \u2014 one scrape is performed per check, then compared against your last stored snapshot for that URL."
|
|
10309
|
-
}
|
|
10310
|
-
];
|
|
10311
|
-
CONCURRENCY_PRICE_ID = "price_1Ta1NRS8aAcsk3TGwsRnYbix";
|
|
10312
|
-
SUBSCRIPTION_TIERS = {
|
|
10313
|
-
"price_1TmiHRS8aAcsk3TGwmSNfNIa": { tier: "starter", label: "Starter", price_id: "price_1TmiHRS8aAcsk3TGwmSNfNIa", monthly_usd: 12, credits_mc: 8e6, concurrency: 3, intro_coupon: "mcp-starter-1dollar-intro-12" },
|
|
10314
|
-
"price_1Tmg1nS8aAcsk3TGe8zcnGTM": { tier: "growth", label: "Growth", price_id: "price_1Tmg1nS8aAcsk3TGe8zcnGTM", monthly_usd: 100, credits_mc: 8e7, concurrency: 10, intro_coupon: null },
|
|
10315
|
-
"price_1Tmg1nS8aAcsk3TGsZw34iXS": { tier: "scale", label: "Scale", price_id: "price_1Tmg1nS8aAcsk3TGsZw34iXS", monthly_usd: 250, credits_mc: 24e7, concurrency: 20, intro_coupon: null }
|
|
10316
|
-
};
|
|
10317
|
-
SUBSCRIPTION_TIER_BY_KEY = Object.fromEntries(
|
|
10318
|
-
Object.values(SUBSCRIPTION_TIERS).map((t) => [t.tier, t])
|
|
10319
|
-
);
|
|
10320
|
-
CONCURRENCY_SLOT_PRICE_USD = 5;
|
|
10321
|
-
CONCURRENCY_SLOT_PRICE_CURRENCY = "usd";
|
|
10322
|
-
CONCURRENCY_SLOT_PRICE_INTERVAL = "month";
|
|
10323
|
-
CONCURRENCY_SLOT_PRICE_LABEL = "$5/month";
|
|
10324
|
-
CONCURRENCY_SLOT_TERMINAL_COMMAND = "npx -y -p mcp-scraper@latest mcp-scraper-cli billing concurrency checkout";
|
|
10325
|
-
FREE_SIGNUP_MC = 1e5;
|
|
10326
|
-
MEMORY_FREE_QUOTA_BYTES = 1e7 * 1;
|
|
10327
|
-
MEMORY_PLANS = {
|
|
10328
|
-
"price_1TnMSTS8aAcsk3TGBgiwuvqL": { plan: "pro", label: "Pro", price_id: "price_1TnMSTS8aAcsk3TGBgiwuvqL", interval: "month", monthly_usd: 19, quota_bytes: 5e9 },
|
|
10329
|
-
"price_1TnMSTS8aAcsk3TGWBVU2agY": { plan: "pro", label: "Pro", price_id: "price_1TnMSTS8aAcsk3TGWBVU2agY", interval: "year", monthly_usd: 19, quota_bytes: 5e9 }
|
|
10330
|
-
};
|
|
10331
|
-
MEMORY_PLAN_QUOTA = { free: MEMORY_FREE_QUOTA_BYTES, pro: 5e9, team: 5e10 };
|
|
10332
|
-
MEMORY_MARGIN_MULTIPLE = 3;
|
|
10333
|
-
MEMORY_PRO_COST_BUDGET_USD = 19 / MEMORY_MARGIN_MULTIPLE;
|
|
10334
|
-
SCHEDULING_PLANS = {
|
|
10335
|
-
"price_1ToXcHS8aAcsk3TGf3cW7zHx": { price_id: "price_1ToXcHS8aAcsk3TGf3cW7zHx", label: "Scheduled Actions", interval: "month", monthly_usd: 10, quota_per_period: 1e3 }
|
|
10336
|
-
};
|
|
10337
|
-
LedgerOperation = {
|
|
10338
|
-
TOPUP: "topup",
|
|
10339
|
-
SUBSCRIPTION: "subscription",
|
|
10340
|
-
SIGNUP_GRANT: "signup_grant",
|
|
10341
|
-
MONTHLY_REFRESH: "monthly_free_refresh",
|
|
10342
|
-
PAA: "paa",
|
|
10343
|
-
PAA_REFUND: "paa_refund",
|
|
10344
|
-
SERP: "serp",
|
|
10345
|
-
REFUND: "refund",
|
|
10346
|
-
TRANSCRIPTION: "transcription",
|
|
10347
|
-
TRANSCRIPTION_HOLD: "transcription_hold",
|
|
10348
|
-
TRANSCRIPTION_REFUND: "transcription_refund",
|
|
10349
|
-
YT_CHANNEL: "yt_channel",
|
|
10350
|
-
FB_AD: "fb_ad",
|
|
10351
|
-
MAPS_SEARCH: "maps_search",
|
|
10352
|
-
MAPS_PLACE: "maps_place",
|
|
10353
|
-
MAPS_REVIEW: "maps_review",
|
|
10354
|
-
MAPS_REVIEW_REFUND: "maps_review_refund",
|
|
10355
|
-
EXTRACT_SITE: "extract_site",
|
|
10356
|
-
EXTRACT_SITE_REFUND: "extract_site_refund",
|
|
10357
|
-
EXTRACT_URL: "page_scrape",
|
|
10358
|
-
URL_MAP: "url_map",
|
|
10359
|
-
EXTRACT_SITE_HOLD: "extract_site_hold",
|
|
10360
|
-
YT_CHANNEL_REFUND: "yt_channel_refund",
|
|
10361
|
-
FB_AD_REFUND: "fb_ad_refund",
|
|
10362
|
-
URL_MAP_REFUND: "url_map_refund",
|
|
10363
|
-
FB_SEARCH: "fb_search",
|
|
10364
|
-
FB_TRANSCRIBE: "fb_transcribe",
|
|
10365
|
-
FB_SEARCH_REFUND: "fb_search_refund",
|
|
10366
|
-
FB_TRANSCRIBE_REFUND: "fb_transcribe_refund",
|
|
10367
|
-
GOOGLE_ADS_SEARCH: "google_ads_search",
|
|
10368
|
-
GOOGLE_ADS_SEARCH_REFUND: "google_ads_search_refund",
|
|
10369
|
-
GOOGLE_ADS_INTEL: "google_ads_intel",
|
|
10370
|
-
GOOGLE_ADS_INTEL_REFUND: "google_ads_intel_refund",
|
|
10371
|
-
GOOGLE_ADS_TRANSCRIBE: "google_ads_transcribe",
|
|
10372
|
-
GOOGLE_ADS_TRANSCRIBE_REFUND: "google_ads_transcribe_refund",
|
|
10373
|
-
INSTAGRAM_PROFILE: "instagram_profile",
|
|
10374
|
-
INSTAGRAM_PROFILE_REFUND: "instagram_profile_refund",
|
|
10375
|
-
INSTAGRAM_MEDIA: "instagram_media",
|
|
10376
|
-
INSTAGRAM_MEDIA_REFUND: "instagram_media_refund",
|
|
10377
|
-
INSTAGRAM_TRANSCRIBE: "instagram_transcribe",
|
|
10378
|
-
INSTAGRAM_TRANSCRIBE_REFUND: "instagram_transcribe_refund",
|
|
10379
|
-
BROWSER_SESSION: "browser_session",
|
|
10380
|
-
REDDIT_THREAD: "reddit_thread",
|
|
10381
|
-
REDDIT_THREAD_REFUND: "reddit_thread_refund",
|
|
10382
|
-
VIDEO_ANALYSIS: "video_analysis",
|
|
10383
|
-
VIDEO_ANALYSIS_REFUND: "video_analysis_refund",
|
|
10384
|
-
VIDEO_ANALYSIS_ADJUST: "video_analysis_adjust",
|
|
10385
|
-
MEMORY_AI: "memory_ai",
|
|
10386
|
-
MEMORY_AI_REFUND: "memory_ai_refund",
|
|
10387
|
-
TRUSTPILOT_REVIEWS: "trustpilot_reviews",
|
|
10388
|
-
TRUSTPILOT_REVIEWS_REFUND: "trustpilot_reviews_refund",
|
|
10389
|
-
TRUSTPILOT_REVIEW: "trustpilot_review",
|
|
10390
|
-
TRUSTPILOT_REVIEW_REFUND: "trustpilot_review_refund",
|
|
10391
|
-
G2_REVIEWS: "g2_reviews",
|
|
10392
|
-
G2_REVIEWS_REFUND: "g2_reviews_refund",
|
|
10393
|
-
G2_REVIEW: "g2_review",
|
|
10394
|
-
G2_REVIEW_REFUND: "g2_review_refund",
|
|
10395
|
-
DIFF_PAGE: "diff_page",
|
|
10396
|
-
DIFF_PAGE_REFUND: "diff_page_refund"
|
|
10397
|
-
};
|
|
10398
|
-
MEMORY_AI_MARGIN_MULTIPLE = 3;
|
|
10399
|
-
MC_PER_USD = SUBSCRIPTION_TIERS["price_1TmiHRS8aAcsk3TGwmSNfNIa"].credits_mc / SUBSCRIPTION_TIERS["price_1TmiHRS8aAcsk3TGwmSNfNIa"].monthly_usd;
|
|
10400
|
-
}
|
|
10401
|
-
});
|
|
10402
|
-
|
|
10403
10483
|
// src/api/site-extract-repository.ts
|
|
10404
10484
|
var site_extract_repository_exports = {};
|
|
10405
10485
|
__export(site_extract_repository_exports, {
|
|
@@ -11619,6 +11699,8 @@ var init_internal_memory_routes = __esm({
|
|
|
11619
11699
|
import_hono2 = require("hono");
|
|
11620
11700
|
init_db();
|
|
11621
11701
|
init_rates();
|
|
11702
|
+
init_cost_telemetry();
|
|
11703
|
+
init_cost_context();
|
|
11622
11704
|
internalMemoryApp = new import_hono2.Hono();
|
|
11623
11705
|
internalMemoryApp.post("/ai-debit", async (c) => {
|
|
11624
11706
|
const secret2 = c.req.header("x-internal-secret");
|
|
@@ -11631,6 +11713,11 @@ var init_internal_memory_routes = __esm({
|
|
|
11631
11713
|
}
|
|
11632
11714
|
const user = await getUserByEmail(body.identity.trim().toLowerCase());
|
|
11633
11715
|
if (!user) return c.json({ ok: false, reason: "no_user" }, 200);
|
|
11716
|
+
if (body.costUsd > 0) {
|
|
11717
|
+
await runWithCostContext({ ...currentCostContext(), op: "memory_ai", userId: user.id }, async () => {
|
|
11718
|
+
void recordVendorUsage({ vendor: "mcp_memory_ai", model: null, units: body.costUsd, unitType: "usd", error: null });
|
|
11719
|
+
});
|
|
11720
|
+
}
|
|
11634
11721
|
const mc = Math.round(body.costUsd * MEMORY_AI_MARGIN_MULTIPLE * MC_PER_USD);
|
|
11635
11722
|
if (mc <= 0) return c.json({ ok: true, mc: 0, balance_mc: null }, 200);
|
|
11636
11723
|
const { ok, balance_mc } = await debitMc(user.id, mc, LedgerOperation.MEMORY_AI, `${body.source ?? "ai"}:${body.op ?? ""}`);
|
|
@@ -12893,6 +12980,563 @@ var init_youtube_harvest = __esm({
|
|
|
12893
12980
|
}
|
|
12894
12981
|
});
|
|
12895
12982
|
|
|
12983
|
+
// src/locations.ts
|
|
12984
|
+
var LOCATIONS;
|
|
12985
|
+
var init_locations = __esm({
|
|
12986
|
+
"src/locations.ts"() {
|
|
12987
|
+
"use strict";
|
|
12988
|
+
LOCATIONS = {
|
|
12989
|
+
"austin": "Austin,Texas,United States",
|
|
12990
|
+
"new york": "New York,New York,United States",
|
|
12991
|
+
"new york city": "New York,New York,United States",
|
|
12992
|
+
"nyc": "New York,New York,United States",
|
|
12993
|
+
"los angeles": "Los Angeles,California,United States",
|
|
12994
|
+
"la": "Los Angeles,California,United States",
|
|
12995
|
+
"chicago": "Chicago,Illinois,United States",
|
|
12996
|
+
"houston": "Houston,Texas,United States",
|
|
12997
|
+
"phoenix": "Phoenix,Arizona,United States",
|
|
12998
|
+
"philadelphia": "Philadelphia,Pennsylvania,United States",
|
|
12999
|
+
"philly": "Philadelphia,Pennsylvania,United States",
|
|
13000
|
+
"san antonio": "San Antonio,Texas,United States",
|
|
13001
|
+
"dallas": "Dallas,Texas,United States",
|
|
13002
|
+
"miami": "Miami,Florida,United States",
|
|
13003
|
+
"seattle": "Seattle,Washington,United States",
|
|
13004
|
+
"denver": "Denver,Colorado,United States",
|
|
13005
|
+
"loveland": "Loveland,Colorado,United States",
|
|
13006
|
+
"loveland co": "Loveland,Colorado,United States",
|
|
13007
|
+
"fort collins": "Fort Collins,Colorado,United States",
|
|
13008
|
+
"boulder": "Boulder,Colorado,United States",
|
|
13009
|
+
"colorado springs": "Colorado Springs,Colorado,United States",
|
|
13010
|
+
"boston": "Boston,Massachusetts,United States",
|
|
13011
|
+
"atlanta": "Atlanta,Georgia,United States",
|
|
13012
|
+
"san francisco": "San Francisco,California,United States",
|
|
13013
|
+
"sf": "San Francisco,California,United States",
|
|
13014
|
+
"portland": "Portland,Oregon,United States",
|
|
13015
|
+
"las vegas": "Las Vegas,Nevada,United States",
|
|
13016
|
+
"minneapolis": "Minneapolis,Minnesota,United States",
|
|
13017
|
+
"detroit": "Detroit,Michigan,United States",
|
|
13018
|
+
"nashville": "Nashville,Tennessee,United States",
|
|
13019
|
+
"charlotte": "Charlotte,North Carolina,United States",
|
|
13020
|
+
"orlando": "Orlando,Florida,United States",
|
|
13021
|
+
"san diego": "San Diego,California,United States",
|
|
13022
|
+
"baltimore": "Baltimore,Maryland,United States",
|
|
13023
|
+
"sacramento": "Sacramento,California,United States",
|
|
13024
|
+
"columbus": "Columbus,Ohio,United States",
|
|
13025
|
+
"indianapolis": "Indianapolis,Indiana,United States",
|
|
13026
|
+
"san jose": "San Jose,California,United States",
|
|
13027
|
+
"fort worth": "Fort Worth,Texas,United States",
|
|
13028
|
+
"jacksonville": "Jacksonville,Florida,United States",
|
|
13029
|
+
"memphis": "Memphis,Tennessee,United States",
|
|
13030
|
+
"louisville": "Louisville,Kentucky,United States",
|
|
13031
|
+
"raleigh": "Raleigh,North Carolina,United States",
|
|
13032
|
+
"richmond": "Richmond,Virginia,United States",
|
|
13033
|
+
"salt lake city": "Salt Lake City,Utah,United States",
|
|
13034
|
+
"toronto": "Toronto,Ontario,Canada",
|
|
13035
|
+
"vancouver": "Vancouver,British Columbia,Canada",
|
|
13036
|
+
"montreal": "Montreal,Quebec,Canada",
|
|
13037
|
+
"calgary": "Calgary,Alberta,Canada",
|
|
13038
|
+
"ottawa": "Ottawa,Ontario,Canada",
|
|
13039
|
+
"london": "London,England,United Kingdom",
|
|
13040
|
+
"manchester": "Manchester,England,United Kingdom",
|
|
13041
|
+
"birmingham": "Birmingham,England,United Kingdom",
|
|
13042
|
+
"edinburgh": "Edinburgh,Scotland,United Kingdom",
|
|
13043
|
+
"glasgow": "Glasgow,Scotland,United Kingdom",
|
|
13044
|
+
"leeds": "Leeds,England,United Kingdom",
|
|
13045
|
+
"sydney": "Sydney,New South Wales,Australia",
|
|
13046
|
+
"melbourne": "Melbourne,Victoria,Australia",
|
|
13047
|
+
"brisbane": "Brisbane,Queensland,Australia",
|
|
13048
|
+
"perth": "Perth,Western Australia,Australia",
|
|
13049
|
+
"adelaide": "Adelaide,South Australia,Australia",
|
|
13050
|
+
"dublin": "Dublin,Leinster,Ireland"
|
|
13051
|
+
};
|
|
13052
|
+
}
|
|
13053
|
+
});
|
|
13054
|
+
|
|
13055
|
+
// src/uule.ts
|
|
13056
|
+
function encodeVarint(value) {
|
|
13057
|
+
const bytes = [];
|
|
13058
|
+
let remaining = value;
|
|
13059
|
+
do {
|
|
13060
|
+
let byte = remaining & 127;
|
|
13061
|
+
remaining >>>= 7;
|
|
13062
|
+
if (remaining > 0) byte |= 128;
|
|
13063
|
+
bytes.push(byte);
|
|
13064
|
+
} while (remaining > 0);
|
|
13065
|
+
return bytes;
|
|
13066
|
+
}
|
|
13067
|
+
function encodeUule(name) {
|
|
13068
|
+
const locationBytes = Buffer.from(name, "utf8");
|
|
13069
|
+
const payload = Buffer.concat([
|
|
13070
|
+
Buffer.from([8, 2, 16, 32, 34]),
|
|
13071
|
+
Buffer.from(encodeVarint(locationBytes.length)),
|
|
13072
|
+
locationBytes
|
|
13073
|
+
]);
|
|
13074
|
+
return `w+${payload.toString("base64")}`;
|
|
13075
|
+
}
|
|
13076
|
+
function normalizeLocation(input) {
|
|
13077
|
+
const raw = input.toLowerCase().trim();
|
|
13078
|
+
if (LOCATIONS[raw]) return LOCATIONS[raw];
|
|
13079
|
+
const beforeComma = raw.split(",")[0].trim();
|
|
13080
|
+
if (beforeComma !== raw && LOCATIONS[beforeComma]) return LOCATIONS[beforeComma];
|
|
13081
|
+
const withoutState = raw.replace(/\s+[a-z]{2}$/, "").trim();
|
|
13082
|
+
if (withoutState !== raw && LOCATIONS[withoutState]) return LOCATIONS[withoutState];
|
|
13083
|
+
return input;
|
|
13084
|
+
}
|
|
13085
|
+
var init_uule = __esm({
|
|
13086
|
+
"src/uule.ts"() {
|
|
13087
|
+
"use strict";
|
|
13088
|
+
init_locations();
|
|
13089
|
+
}
|
|
13090
|
+
});
|
|
13091
|
+
|
|
13092
|
+
// src/kernel-proxy-resolver.ts
|
|
13093
|
+
function proxyIdSuffix2(proxyId) {
|
|
13094
|
+
return proxyId ? proxyId.slice(-6) : null;
|
|
13095
|
+
}
|
|
13096
|
+
function resolution(source, proxyMode, proxyId, target, error, disposable = false) {
|
|
13097
|
+
const proxyType = source === "disabled" ? "none" : source === "configured_fallback" ? "configured" : source === "isp_created" ? "isp" : source === "location_created" || source === "location_reused" ? "residential" : "unknown";
|
|
13098
|
+
return {
|
|
13099
|
+
kernelProxyId: proxyId,
|
|
13100
|
+
...disposable && proxyId ? { disposableProxyId: proxyId } : {},
|
|
13101
|
+
resolution: {
|
|
13102
|
+
source,
|
|
13103
|
+
proxyType,
|
|
13104
|
+
proxyMode,
|
|
13105
|
+
proxyIdPresent: Boolean(proxyId),
|
|
13106
|
+
proxyIdSuffix: proxyIdSuffix2(proxyId),
|
|
13107
|
+
target,
|
|
13108
|
+
error,
|
|
13109
|
+
disposable
|
|
13110
|
+
}
|
|
13111
|
+
};
|
|
13112
|
+
}
|
|
13113
|
+
function normalizeStateName(value) {
|
|
13114
|
+
return value.trim().toLowerCase().replace(/\s+/g, " ");
|
|
13115
|
+
}
|
|
13116
|
+
function normalizeCountryName(value) {
|
|
13117
|
+
return value.trim().toLowerCase().replace(/\./g, "").replace(/\s+/g, " ");
|
|
13118
|
+
}
|
|
13119
|
+
function isUnitedStates(country) {
|
|
13120
|
+
if (!country) return true;
|
|
13121
|
+
const normalized = normalizeCountryName(country);
|
|
13122
|
+
return normalized === "united states" || normalized === "united states of america" || normalized === "usa" || normalized === "us";
|
|
13123
|
+
}
|
|
13124
|
+
function stateCodeFor(region) {
|
|
13125
|
+
const trimmed = region.trim();
|
|
13126
|
+
if (/^[A-Za-z]{2}$/.test(trimmed)) return trimmed.toUpperCase();
|
|
13127
|
+
return US_STATE_CODES[normalizeStateName(trimmed)] ?? null;
|
|
13128
|
+
}
|
|
13129
|
+
function kernelCityIdentifierCandidates(city) {
|
|
13130
|
+
const ascii = city.normalize("NFKD").replace(/[^\x00-\x7F]/g, "").toLowerCase();
|
|
13131
|
+
const words = ascii.split(/[^a-z0-9]+/).filter(Boolean);
|
|
13132
|
+
const underscored = words.join("_");
|
|
13133
|
+
const compact = words.join("");
|
|
13134
|
+
return Array.from(new Set([underscored, compact].filter(Boolean)));
|
|
13135
|
+
}
|
|
13136
|
+
function proxyName(country, state, city) {
|
|
13137
|
+
return city ? `mcp-serp-residential-${country.toLowerCase()}-${state.toLowerCase()}-${city}` : `mcp-serp-residential-${country.toLowerCase()}-${state.toLowerCase()}`;
|
|
13138
|
+
}
|
|
13139
|
+
function freshProxyName(baseName, attemptIndex) {
|
|
13140
|
+
const stamp = `${Date.now()}-${attemptIndex ?? 0}-${Math.random().toString(36).slice(2, 8)}`;
|
|
13141
|
+
return `${baseName}-fresh-${stamp}`;
|
|
13142
|
+
}
|
|
13143
|
+
function zipProxyName(zip) {
|
|
13144
|
+
return `mcp-serp-residential-us-zip-${zip}`;
|
|
13145
|
+
}
|
|
13146
|
+
function parseKernelLocationProxyTarget(location2, gl) {
|
|
13147
|
+
if (!location2 || gl.toLowerCase() !== "us") return null;
|
|
13148
|
+
const canonicalLocation = normalizeLocation(location2);
|
|
13149
|
+
let parts = canonicalLocation.split(",").map((part) => part.trim()).filter(Boolean);
|
|
13150
|
+
if (parts.length > 1 && isUnitedStates(parts[parts.length - 1])) {
|
|
13151
|
+
parts = parts.slice(0, -1);
|
|
13152
|
+
}
|
|
13153
|
+
if (parts.length === 1) {
|
|
13154
|
+
const stateOnly = stateCodeFor(parts[0]);
|
|
13155
|
+
if (!stateOnly) return null;
|
|
13156
|
+
return {
|
|
13157
|
+
canonicalLocation,
|
|
13158
|
+
level: "state",
|
|
13159
|
+
country: "US",
|
|
13160
|
+
state: stateOnly,
|
|
13161
|
+
city: "",
|
|
13162
|
+
cityCandidates: [],
|
|
13163
|
+
proxyName: proxyName("US", stateOnly),
|
|
13164
|
+
config: {
|
|
13165
|
+
country: "US",
|
|
13166
|
+
state: stateOnly
|
|
13167
|
+
}
|
|
13168
|
+
};
|
|
13169
|
+
}
|
|
13170
|
+
const [city = "", region = ""] = parts;
|
|
13171
|
+
if (!city || !region) return null;
|
|
13172
|
+
const state = stateCodeFor(region);
|
|
13173
|
+
if (!state) return null;
|
|
13174
|
+
const cityCandidates = kernelCityIdentifierCandidates(city);
|
|
13175
|
+
const primaryCity = cityCandidates[0];
|
|
13176
|
+
if (!primaryCity) return null;
|
|
13177
|
+
return {
|
|
13178
|
+
canonicalLocation,
|
|
13179
|
+
level: "city",
|
|
13180
|
+
country: "US",
|
|
13181
|
+
state,
|
|
13182
|
+
city: primaryCity,
|
|
13183
|
+
cityCandidates,
|
|
13184
|
+
proxyName: proxyName("US", state, primaryCity),
|
|
13185
|
+
config: {
|
|
13186
|
+
country: "US",
|
|
13187
|
+
state,
|
|
13188
|
+
city: primaryCity
|
|
13189
|
+
}
|
|
13190
|
+
};
|
|
13191
|
+
}
|
|
13192
|
+
function cityZipKey(target) {
|
|
13193
|
+
return `${target.city}|${target.state}`;
|
|
13194
|
+
}
|
|
13195
|
+
function knownZipFor(target, explicitZip) {
|
|
13196
|
+
if (explicitZip && /^\d{5}$/.test(explicitZip)) return explicitZip;
|
|
13197
|
+
return US_CITY_CENTER_ZIPS[cityZipKey(target)] ?? null;
|
|
13198
|
+
}
|
|
13199
|
+
function zipTarget(target, zip) {
|
|
13200
|
+
return {
|
|
13201
|
+
...target,
|
|
13202
|
+
level: "zip",
|
|
13203
|
+
zip,
|
|
13204
|
+
proxyName: zipProxyName(zip),
|
|
13205
|
+
config: {
|
|
13206
|
+
country: target.country,
|
|
13207
|
+
state: target.state,
|
|
13208
|
+
zip
|
|
13209
|
+
}
|
|
13210
|
+
};
|
|
13211
|
+
}
|
|
13212
|
+
function withProxyName(target, name) {
|
|
13213
|
+
return {
|
|
13214
|
+
...target,
|
|
13215
|
+
proxyName: name
|
|
13216
|
+
};
|
|
13217
|
+
}
|
|
13218
|
+
function configMatches(config, target, city) {
|
|
13219
|
+
if (target.level === "zip") {
|
|
13220
|
+
return config?.country?.toUpperCase() === target.country && config?.zip === target.zip;
|
|
13221
|
+
}
|
|
13222
|
+
return config?.country?.toUpperCase() === target.country && config?.state?.toUpperCase() === target.state && (city ? config?.city === city : !config?.city);
|
|
13223
|
+
}
|
|
13224
|
+
function findExistingTargetProxy(proxies, target) {
|
|
13225
|
+
return proxies.find((proxy) => proxy.type === "residential" && proxy.status !== "unavailable" && Boolean(proxy.id) && (proxy.name === target.proxyName || configMatches(proxy.config, target, target.level === "city" ? target.city : void 0))) ?? null;
|
|
13226
|
+
}
|
|
13227
|
+
function findExistingProxy(proxies, target) {
|
|
13228
|
+
for (const city of target.cityCandidates) {
|
|
13229
|
+
const name = proxyName(target.country, target.state, city);
|
|
13230
|
+
const found = proxies.find((proxy) => proxy.type === "residential" && proxy.status !== "unavailable" && Boolean(proxy.id) && (proxy.name === name || configMatches(proxy.config, target, city)));
|
|
13231
|
+
if (found) return found;
|
|
13232
|
+
}
|
|
13233
|
+
return null;
|
|
13234
|
+
}
|
|
13235
|
+
function stateTarget(target) {
|
|
13236
|
+
return {
|
|
13237
|
+
...target,
|
|
13238
|
+
level: "state",
|
|
13239
|
+
proxyName: proxyName(target.country, target.state),
|
|
13240
|
+
config: {
|
|
13241
|
+
country: target.country,
|
|
13242
|
+
state: target.state
|
|
13243
|
+
}
|
|
13244
|
+
};
|
|
13245
|
+
}
|
|
13246
|
+
function findExistingStateProxy(proxies, target) {
|
|
13247
|
+
const name = proxyName(target.country, target.state);
|
|
13248
|
+
return proxies.find((proxy) => proxy.type === "residential" && proxy.status !== "unavailable" && Boolean(proxy.id) && (proxy.name === name || configMatches(proxy.config, target))) ?? null;
|
|
13249
|
+
}
|
|
13250
|
+
function escalatedTargetLevel(target, attemptIndex) {
|
|
13251
|
+
return stateTarget(target);
|
|
13252
|
+
}
|
|
13253
|
+
function errorText2(err) {
|
|
13254
|
+
return err instanceof Error ? err.message : String(err);
|
|
13255
|
+
}
|
|
13256
|
+
function freshTargetCandidates(target, explicitZip, attemptIndex) {
|
|
13257
|
+
const out = [];
|
|
13258
|
+
const zip = knownZipFor(target, explicitZip);
|
|
13259
|
+
if (zip) {
|
|
13260
|
+
const targetZip = zipTarget(target, zip);
|
|
13261
|
+
out.push(withProxyName(targetZip, freshProxyName(targetZip.proxyName, attemptIndex)));
|
|
13262
|
+
}
|
|
13263
|
+
for (const city of target.cityCandidates) {
|
|
13264
|
+
const cityTarget = {
|
|
13265
|
+
...target,
|
|
13266
|
+
level: "city",
|
|
13267
|
+
city,
|
|
13268
|
+
proxyName: proxyName(target.country, target.state, city),
|
|
13269
|
+
config: {
|
|
13270
|
+
country: target.country,
|
|
13271
|
+
state: target.state,
|
|
13272
|
+
city
|
|
13273
|
+
}
|
|
13274
|
+
};
|
|
13275
|
+
out.push(withProxyName(cityTarget, freshProxyName(cityTarget.proxyName, attemptIndex)));
|
|
13276
|
+
}
|
|
13277
|
+
const fallbackTarget = stateTarget(target);
|
|
13278
|
+
out.push(withProxyName(fallbackTarget, freshProxyName(fallbackTarget.proxyName, attemptIndex)));
|
|
13279
|
+
return out;
|
|
13280
|
+
}
|
|
13281
|
+
async function createFreshLocationProxy(kernel, options, target) {
|
|
13282
|
+
const createErrors = [];
|
|
13283
|
+
for (const candidate of freshTargetCandidates(target, options.proxyZip, options.attemptIndex)) {
|
|
13284
|
+
try {
|
|
13285
|
+
const created = await kernel.proxies.create({
|
|
13286
|
+
type: "residential",
|
|
13287
|
+
name: candidate.proxyName,
|
|
13288
|
+
config: candidate.level === "zip" ? { country: candidate.country, zip: candidate.zip } : candidate.config
|
|
13289
|
+
});
|
|
13290
|
+
if (created.id) {
|
|
13291
|
+
return resolution("location_created", options.proxyMode, created.id, candidate, null, true);
|
|
13292
|
+
}
|
|
13293
|
+
createErrors.push(`${candidate.proxyName}: Kernel did not return a proxy id`);
|
|
13294
|
+
} catch (err) {
|
|
13295
|
+
createErrors.push(`${candidate.proxyName}: ${errorText2(err)}`);
|
|
13296
|
+
}
|
|
13297
|
+
}
|
|
13298
|
+
if (process.env.KERNEL_PROXY_TYPE !== "residential") {
|
|
13299
|
+
const ispTarget = stateTarget(target);
|
|
13300
|
+
try {
|
|
13301
|
+
const created = await kernel.proxies.create({
|
|
13302
|
+
type: "isp",
|
|
13303
|
+
name: freshProxyName(`mcp-serp-isp-${ispTarget.country.toLowerCase()}-${(ispTarget.state ?? "").toLowerCase()}`, options.attemptIndex),
|
|
13304
|
+
config: { country: ispTarget.country, ...ispTarget.state ? { state: ispTarget.state } : {} }
|
|
13305
|
+
});
|
|
13306
|
+
if (created.id) {
|
|
13307
|
+
return resolution("isp_created", options.proxyMode, created.id, ispTarget, null, true);
|
|
13308
|
+
}
|
|
13309
|
+
createErrors.push("isp: Kernel did not return a proxy id");
|
|
13310
|
+
} catch (err) {
|
|
13311
|
+
createErrors.push(`isp: ${errorText2(err)}`);
|
|
13312
|
+
}
|
|
13313
|
+
}
|
|
13314
|
+
return resolution("configured_fallback", options.proxyMode, options.configuredKernelProxyId, target, createErrors.join(" | "));
|
|
13315
|
+
}
|
|
13316
|
+
async function deleteKernelProxyId(kernelApiKey, proxyId) {
|
|
13317
|
+
if (!kernelApiKey || !proxyId) return;
|
|
13318
|
+
const kernel = new import_sdk6.default({ apiKey: kernelApiKey });
|
|
13319
|
+
await kernel.proxies.delete(proxyId);
|
|
13320
|
+
}
|
|
13321
|
+
async function resolveKernelProxyId(options) {
|
|
13322
|
+
if (options.proxyMode === "none") {
|
|
13323
|
+
return resolution("disabled", options.proxyMode, void 0, null, null);
|
|
13324
|
+
}
|
|
13325
|
+
if (options.proxyMode === "configured") {
|
|
13326
|
+
return resolution("configured_fallback", options.proxyMode, options.configuredKernelProxyId, null, null);
|
|
13327
|
+
}
|
|
13328
|
+
const target = parseKernelLocationProxyTarget(options.location, options.gl);
|
|
13329
|
+
if (!target || !options.kernelApiKey) {
|
|
13330
|
+
return resolution("configured_fallback", options.proxyMode, options.configuredKernelProxyId, target, target ? null : "location could not be normalized to a US city/state proxy target");
|
|
13331
|
+
}
|
|
13332
|
+
const kernel = new import_sdk6.default({ apiKey: options.kernelApiKey });
|
|
13333
|
+
try {
|
|
13334
|
+
const attemptIndex = options.attemptIndex ?? 0;
|
|
13335
|
+
if (options.fresh) {
|
|
13336
|
+
return await createFreshLocationProxy(kernel, options, target);
|
|
13337
|
+
}
|
|
13338
|
+
if (attemptIndex >= 1) {
|
|
13339
|
+
const escalatedTarget = escalatedTargetLevel(target, attemptIndex);
|
|
13340
|
+
const createErrors2 = [];
|
|
13341
|
+
try {
|
|
13342
|
+
const created = await kernel.proxies.create({
|
|
13343
|
+
type: "residential",
|
|
13344
|
+
name: escalatedTarget.proxyName,
|
|
13345
|
+
config: escalatedTarget.config
|
|
13346
|
+
});
|
|
13347
|
+
if (created.id) {
|
|
13348
|
+
return resolution("location_created", options.proxyMode, created.id, escalatedTarget, null);
|
|
13349
|
+
}
|
|
13350
|
+
createErrors2.push(`${escalatedTarget.state}: Kernel did not return a proxy id`);
|
|
13351
|
+
} catch (err) {
|
|
13352
|
+
createErrors2.push(`${escalatedTarget.state}: ${errorText2(err)}`);
|
|
13353
|
+
}
|
|
13354
|
+
return resolution("configured_fallback", options.proxyMode, options.configuredKernelProxyId, escalatedTarget, createErrors2.join(" | "));
|
|
13355
|
+
}
|
|
13356
|
+
const proxies = await kernel.proxies.list();
|
|
13357
|
+
const zip = knownZipFor(target, options.proxyZip);
|
|
13358
|
+
const createErrors = [];
|
|
13359
|
+
if (zip) {
|
|
13360
|
+
const targetZip = zipTarget(target, zip);
|
|
13361
|
+
const existingZip = findExistingTargetProxy(proxies, targetZip);
|
|
13362
|
+
if (existingZip?.id) {
|
|
13363
|
+
return resolution("location_reused", options.proxyMode, existingZip.id, targetZip, null);
|
|
13364
|
+
}
|
|
13365
|
+
try {
|
|
13366
|
+
const created = await kernel.proxies.create({
|
|
13367
|
+
type: "residential",
|
|
13368
|
+
name: targetZip.proxyName,
|
|
13369
|
+
config: {
|
|
13370
|
+
country: targetZip.country,
|
|
13371
|
+
zip
|
|
13372
|
+
}
|
|
13373
|
+
});
|
|
13374
|
+
if (created.id) {
|
|
13375
|
+
return resolution("location_created", options.proxyMode, created.id, targetZip, null);
|
|
13376
|
+
}
|
|
13377
|
+
createErrors.push(`${zip}: Kernel did not return a proxy id`);
|
|
13378
|
+
} catch (err) {
|
|
13379
|
+
createErrors.push(`${zip}: ${errorText2(err)}`);
|
|
13380
|
+
}
|
|
13381
|
+
}
|
|
13382
|
+
const existing = findExistingProxy(proxies, target);
|
|
13383
|
+
if (existing?.id) {
|
|
13384
|
+
return resolution("location_reused", options.proxyMode, existing.id, target, createErrors.join(" | ") || null);
|
|
13385
|
+
}
|
|
13386
|
+
for (const city of target.cityCandidates) {
|
|
13387
|
+
try {
|
|
13388
|
+
const created = await kernel.proxies.create({
|
|
13389
|
+
type: "residential",
|
|
13390
|
+
name: proxyName(target.country, target.state, city),
|
|
13391
|
+
config: {
|
|
13392
|
+
country: target.country,
|
|
13393
|
+
state: target.state,
|
|
13394
|
+
city
|
|
13395
|
+
}
|
|
13396
|
+
});
|
|
13397
|
+
if (created.id) {
|
|
13398
|
+
return resolution("location_created", options.proxyMode, created.id, {
|
|
13399
|
+
...target,
|
|
13400
|
+
level: "city",
|
|
13401
|
+
city,
|
|
13402
|
+
proxyName: proxyName(target.country, target.state, city),
|
|
13403
|
+
config: {
|
|
13404
|
+
country: target.country,
|
|
13405
|
+
state: target.state,
|
|
13406
|
+
city
|
|
13407
|
+
}
|
|
13408
|
+
}, null);
|
|
13409
|
+
}
|
|
13410
|
+
createErrors.push(`${city}: Kernel did not return a proxy id`);
|
|
13411
|
+
} catch (err) {
|
|
13412
|
+
createErrors.push(`${city}: ${errorText2(err)}`);
|
|
13413
|
+
}
|
|
13414
|
+
}
|
|
13415
|
+
const fallbackTarget = stateTarget(target);
|
|
13416
|
+
const existingState = findExistingStateProxy(proxies, fallbackTarget);
|
|
13417
|
+
if (existingState?.id) {
|
|
13418
|
+
return resolution("location_reused", options.proxyMode, existingState.id, fallbackTarget, createErrors.join(" | "));
|
|
13419
|
+
}
|
|
13420
|
+
try {
|
|
13421
|
+
const created = await kernel.proxies.create({
|
|
13422
|
+
type: "residential",
|
|
13423
|
+
name: fallbackTarget.proxyName,
|
|
13424
|
+
config: fallbackTarget.config
|
|
13425
|
+
});
|
|
13426
|
+
if (created.id) {
|
|
13427
|
+
return resolution("location_created", options.proxyMode, created.id, fallbackTarget, createErrors.join(" | "));
|
|
13428
|
+
}
|
|
13429
|
+
createErrors.push(`${fallbackTarget.state}: Kernel did not return a proxy id`);
|
|
13430
|
+
} catch (err) {
|
|
13431
|
+
createErrors.push(`${fallbackTarget.state}: ${errorText2(err)}`);
|
|
13432
|
+
}
|
|
13433
|
+
return resolution("configured_fallback", options.proxyMode, options.configuredKernelProxyId, target, createErrors.join(" | "));
|
|
13434
|
+
} catch (err) {
|
|
13435
|
+
return resolution("configured_fallback", options.proxyMode, options.configuredKernelProxyId, target, errorText2(err));
|
|
13436
|
+
}
|
|
13437
|
+
}
|
|
13438
|
+
var import_sdk6, US_STATE_CODES, US_CITY_CENTER_ZIPS;
|
|
13439
|
+
var init_kernel_proxy_resolver = __esm({
|
|
13440
|
+
"src/kernel-proxy-resolver.ts"() {
|
|
13441
|
+
"use strict";
|
|
13442
|
+
import_sdk6 = __toESM(require("@onkernel/sdk"), 1);
|
|
13443
|
+
init_uule();
|
|
13444
|
+
US_STATE_CODES = {
|
|
13445
|
+
alabama: "AL",
|
|
13446
|
+
alaska: "AK",
|
|
13447
|
+
arizona: "AZ",
|
|
13448
|
+
arkansas: "AR",
|
|
13449
|
+
california: "CA",
|
|
13450
|
+
colorado: "CO",
|
|
13451
|
+
connecticut: "CT",
|
|
13452
|
+
delaware: "DE",
|
|
13453
|
+
florida: "FL",
|
|
13454
|
+
georgia: "GA",
|
|
13455
|
+
hawaii: "HI",
|
|
13456
|
+
idaho: "ID",
|
|
13457
|
+
illinois: "IL",
|
|
13458
|
+
indiana: "IN",
|
|
13459
|
+
iowa: "IA",
|
|
13460
|
+
kansas: "KS",
|
|
13461
|
+
kentucky: "KY",
|
|
13462
|
+
louisiana: "LA",
|
|
13463
|
+
maine: "ME",
|
|
13464
|
+
maryland: "MD",
|
|
13465
|
+
massachusetts: "MA",
|
|
13466
|
+
michigan: "MI",
|
|
13467
|
+
minnesota: "MN",
|
|
13468
|
+
mississippi: "MS",
|
|
13469
|
+
missouri: "MO",
|
|
13470
|
+
montana: "MT",
|
|
13471
|
+
nebraska: "NE",
|
|
13472
|
+
nevada: "NV",
|
|
13473
|
+
"new hampshire": "NH",
|
|
13474
|
+
"new jersey": "NJ",
|
|
13475
|
+
"new mexico": "NM",
|
|
13476
|
+
"new york": "NY",
|
|
13477
|
+
"north carolina": "NC",
|
|
13478
|
+
"north dakota": "ND",
|
|
13479
|
+
ohio: "OH",
|
|
13480
|
+
oklahoma: "OK",
|
|
13481
|
+
oregon: "OR",
|
|
13482
|
+
pennsylvania: "PA",
|
|
13483
|
+
"rhode island": "RI",
|
|
13484
|
+
"south carolina": "SC",
|
|
13485
|
+
"south dakota": "SD",
|
|
13486
|
+
tennessee: "TN",
|
|
13487
|
+
texas: "TX",
|
|
13488
|
+
utah: "UT",
|
|
13489
|
+
vermont: "VT",
|
|
13490
|
+
virginia: "VA",
|
|
13491
|
+
washington: "WA",
|
|
13492
|
+
"west virginia": "WV",
|
|
13493
|
+
wisconsin: "WI",
|
|
13494
|
+
wyoming: "WY"
|
|
13495
|
+
};
|
|
13496
|
+
US_CITY_CENTER_ZIPS = {
|
|
13497
|
+
"atlanta|GA": "30303",
|
|
13498
|
+
"austin|TX": "78701",
|
|
13499
|
+
"baltimore|MD": "21201",
|
|
13500
|
+
"boston|MA": "02108",
|
|
13501
|
+
"boulder|CO": "80302",
|
|
13502
|
+
"charlotte|NC": "28202",
|
|
13503
|
+
"chicago|IL": "60601",
|
|
13504
|
+
"colorado_springs|CO": "80903",
|
|
13505
|
+
"columbus|OH": "43215",
|
|
13506
|
+
"dallas|TX": "75201",
|
|
13507
|
+
"denver|CO": "80202",
|
|
13508
|
+
"detroit|MI": "48226",
|
|
13509
|
+
"fort_collins|CO": "80524",
|
|
13510
|
+
"fort_worth|TX": "76102",
|
|
13511
|
+
"houston|TX": "77002",
|
|
13512
|
+
"indianapolis|IN": "46204",
|
|
13513
|
+
"jacksonville|FL": "32202",
|
|
13514
|
+
"las_vegas|NV": "89101",
|
|
13515
|
+
"los_angeles|CA": "90012",
|
|
13516
|
+
"louisville|KY": "40202",
|
|
13517
|
+
"loveland|CO": "80537",
|
|
13518
|
+
"memphis|TN": "38103",
|
|
13519
|
+
"miami|FL": "33131",
|
|
13520
|
+
"minneapolis|MN": "55401",
|
|
13521
|
+
"nashville|TN": "37203",
|
|
13522
|
+
"new_york|NY": "10001",
|
|
13523
|
+
"orlando|FL": "32801",
|
|
13524
|
+
"philadelphia|PA": "19103",
|
|
13525
|
+
"phoenix|AZ": "85004",
|
|
13526
|
+
"portland|OR": "97205",
|
|
13527
|
+
"raleigh|NC": "27601",
|
|
13528
|
+
"richmond|VA": "23219",
|
|
13529
|
+
"sacramento|CA": "95814",
|
|
13530
|
+
"salt_lake_city|UT": "84101",
|
|
13531
|
+
"san_antonio|TX": "78205",
|
|
13532
|
+
"san_diego|CA": "92101",
|
|
13533
|
+
"san_francisco|CA": "94103",
|
|
13534
|
+
"san_jose|CA": "95113",
|
|
13535
|
+
"seattle|WA": "98101"
|
|
13536
|
+
};
|
|
13537
|
+
}
|
|
13538
|
+
});
|
|
13539
|
+
|
|
12896
13540
|
// src/youtube/CaptionFetcher.ts
|
|
12897
13541
|
async function fetchViaYoutubeTranscript(videoId) {
|
|
12898
13542
|
try {
|
|
@@ -12905,6 +13549,11 @@ async function fetchViaYoutubeTranscript(videoId) {
|
|
|
12905
13549
|
}));
|
|
12906
13550
|
const text = chunks.map((c) => c.text).join(" ");
|
|
12907
13551
|
const last = chunks[chunks.length - 1];
|
|
13552
|
+
const now = Date.now();
|
|
13553
|
+
await runWithCostContext(
|
|
13554
|
+
{ ...currentCostContext(), op: "yt_transcription", subOp: "youtube-transcript" },
|
|
13555
|
+
() => recordKernelSession({ kernelSessionId: null, source: "yt_caption_tier", headlessSent: null, openedAtMs: now, closedAtMs: now })
|
|
13556
|
+
);
|
|
12908
13557
|
return { videoId, text, chunks, durationMs: last ? last.timestamp[1] * 1e3 : 0, method: "youtube-transcript" };
|
|
12909
13558
|
} catch {
|
|
12910
13559
|
return null;
|
|
@@ -12956,162 +13605,214 @@ function parseTimedtextXml(xml) {
|
|
|
12956
13605
|
}
|
|
12957
13606
|
return results;
|
|
12958
13607
|
}
|
|
12959
|
-
|
|
12960
|
-
|
|
12961
|
-
|
|
12962
|
-
|
|
12963
|
-
|
|
12964
|
-
|
|
12965
|
-
|
|
12966
|
-
|
|
12967
|
-
|
|
12968
|
-
|
|
12969
|
-
|
|
12970
|
-
|
|
12971
|
-
|
|
12972
|
-
|
|
12973
|
-
|
|
12974
|
-
|
|
12975
|
-
|
|
12976
|
-
|
|
12977
|
-
|
|
12978
|
-
|
|
12979
|
-
|
|
12980
|
-
|
|
12981
|
-
|
|
12982
|
-
|
|
12983
|
-
|
|
12984
|
-
|
|
13608
|
+
function isBotBlockStatus(status) {
|
|
13609
|
+
return status === "LOGIN_REQUIRED" || status === "AGE_VERIFICATION_REQUIRED";
|
|
13610
|
+
}
|
|
13611
|
+
async function probePlayer(page, videoId) {
|
|
13612
|
+
return page.evaluate(
|
|
13613
|
+
async ({ vid, clients }) => {
|
|
13614
|
+
let lastStatus = null;
|
|
13615
|
+
for (const client2 of clients) {
|
|
13616
|
+
const resp = await fetch("/youtubei/v1/player?prettyPrint=false", {
|
|
13617
|
+
method: "POST",
|
|
13618
|
+
headers: { "Content-Type": "application/json", "User-Agent": client2.userAgent },
|
|
13619
|
+
body: JSON.stringify({
|
|
13620
|
+
context: { client: { clientName: client2.name, clientVersion: client2.version } },
|
|
13621
|
+
videoId: vid
|
|
13622
|
+
})
|
|
13623
|
+
}).catch(() => null);
|
|
13624
|
+
if (!resp?.ok) {
|
|
13625
|
+
lastStatus = resp ? `player_http_${resp.status}` : "player_fetch_failed";
|
|
13626
|
+
continue;
|
|
13627
|
+
}
|
|
13628
|
+
const data = await resp.json();
|
|
13629
|
+
lastStatus = data?.playabilityStatus?.status ?? null;
|
|
13630
|
+
const audioFormats = (data?.streamingData?.adaptiveFormats ?? []).filter((f) => typeof f.url === "string" && f.url && typeof f.mimeType === "string" && f.mimeType.startsWith("audio/")).sort((a, b) => (Number(a.contentLength) || Infinity) - (Number(b.contentLength) || Infinity));
|
|
13631
|
+
const pick = audioFormats[0] ?? null;
|
|
13632
|
+
let captions = null;
|
|
13633
|
+
const tracks = data?.captions?.playerCaptionsTracklistRenderer?.captionTracks ?? [];
|
|
13634
|
+
if (tracks.length > 0) {
|
|
13635
|
+
const track = tracks.find((t) => t.languageCode === "en") ?? tracks[0];
|
|
13636
|
+
const sep = track.baseUrl.includes("?") ? "&" : "?";
|
|
13637
|
+
const xmlResp = await fetch(`${track.baseUrl}${sep}fmt=json3`).catch(() => null);
|
|
13638
|
+
if (xmlResp?.ok) captions = await xmlResp.text();
|
|
13639
|
+
}
|
|
13640
|
+
if (!captions && !pick) continue;
|
|
13641
|
+
return {
|
|
13642
|
+
captions,
|
|
13643
|
+
status: lastStatus,
|
|
13644
|
+
audio: pick ? {
|
|
13645
|
+
url: pick.url,
|
|
13646
|
+
mimeType: pick.mimeType,
|
|
13647
|
+
bitrate: pick.bitrate ?? null,
|
|
13648
|
+
contentLength: pick.contentLength ? Number(pick.contentLength) || null : null,
|
|
13649
|
+
approxDurationMs: pick.approxDurationMs ? Number(pick.approxDurationMs) || null : null
|
|
13650
|
+
} : null
|
|
13651
|
+
};
|
|
13652
|
+
}
|
|
13653
|
+
return { captions: null, audio: null, status: lastStatus };
|
|
13654
|
+
},
|
|
13655
|
+
{ vid: videoId, clients: INNERTUBE_CLIENTS }
|
|
13656
|
+
).catch(() => null);
|
|
13657
|
+
}
|
|
13658
|
+
async function downloadWholeAudioStreamInOneRequestBecauseYouTubeAllowsOnlyOnePerIp(page, audio) {
|
|
13659
|
+
if (audio.contentLength && audio.contentLength > MAX_AUDIO_BYTES) {
|
|
13660
|
+
console.warn(`[CaptionFetcher] audio stream ${audio.contentLength} bytes exceeds ${MAX_AUDIO_BYTES} byte cap`);
|
|
13661
|
+
return null;
|
|
13662
|
+
}
|
|
13663
|
+
const sep = audio.url.includes("?") ? "&" : "?";
|
|
13664
|
+
const url = audio.contentLength ? `${audio.url}${sep}range=0-${audio.contentLength - 1}` : audio.url;
|
|
13665
|
+
const result = await page.evaluate(async (u) => {
|
|
13666
|
+
try {
|
|
13667
|
+
const resp = await fetch(u);
|
|
13668
|
+
if (!resp.ok) return { ok: false, error: `http_${resp.status}` };
|
|
13669
|
+
const blob = await resp.blob();
|
|
13670
|
+
const dataUrl = await new Promise((resolve, reject) => {
|
|
13671
|
+
const reader = new FileReader();
|
|
13672
|
+
reader.onload = () => resolve(reader.result);
|
|
13673
|
+
reader.onerror = () => reject(reader.error);
|
|
13674
|
+
reader.readAsDataURL(blob);
|
|
12985
13675
|
});
|
|
12986
|
-
|
|
12987
|
-
|
|
12988
|
-
|
|
12989
|
-
|
|
12990
|
-
|
|
12991
|
-
|
|
12992
|
-
|
|
12993
|
-
|
|
12994
|
-
|
|
12995
|
-
|
|
12996
|
-
|
|
12997
|
-
|
|
12998
|
-
if (entries.length === 0) return null;
|
|
12999
|
-
const chunks = entries.map((e) => ({
|
|
13000
|
-
timestamp: [e.start / 1e3, (e.start + e.dur) / 1e3],
|
|
13001
|
-
text: e.text
|
|
13002
|
-
}));
|
|
13003
|
-
const text = chunks.map((c) => c.text).join(" ");
|
|
13004
|
-
const last = chunks[chunks.length - 1];
|
|
13005
|
-
return { videoId, text, chunks, durationMs: Date.now() - start, method: "browser-innertube" };
|
|
13006
|
-
} catch {
|
|
13676
|
+
return { ok: true, base64: dataUrl.slice(dataUrl.indexOf(",") + 1) };
|
|
13677
|
+
} catch (e) {
|
|
13678
|
+
return { ok: false, error: String(e) };
|
|
13679
|
+
}
|
|
13680
|
+
}, url).catch((e) => ({ ok: false, error: String(e) }));
|
|
13681
|
+
if (!result.ok || !result.base64) {
|
|
13682
|
+
console.warn(`[CaptionFetcher] audio fetch failed (${result.error ?? "unknown"}) for a ${audio.contentLength ?? 0} byte stream`);
|
|
13683
|
+
return null;
|
|
13684
|
+
}
|
|
13685
|
+
const bytes = Buffer.from(result.base64, "base64");
|
|
13686
|
+
if (audio.contentLength && bytes.length < audio.contentLength) {
|
|
13687
|
+
console.warn(`[CaptionFetcher] partial audio: ${bytes.length} of ${audio.contentLength} bytes`);
|
|
13007
13688
|
return null;
|
|
13008
|
-
} finally {
|
|
13009
|
-
await driver.close();
|
|
13010
13689
|
}
|
|
13690
|
+
return { bytes, mimeType: audio.mimeType.split(";")[0].trim() };
|
|
13011
13691
|
}
|
|
13012
|
-
async function
|
|
13013
|
-
const
|
|
13014
|
-
|
|
13015
|
-
const video = document.querySelector("video");
|
|
13016
|
-
if (!video) {
|
|
13017
|
-
resolve({ ok: false, error: "no video element" });
|
|
13018
|
-
return;
|
|
13019
|
-
}
|
|
13020
|
-
video.muted = false;
|
|
13021
|
-
video.volume = 1;
|
|
13022
|
-
try {
|
|
13023
|
-
const ctx = new AudioContext();
|
|
13024
|
-
const src = ctx.createMediaElementSource(video);
|
|
13025
|
-
const dest = ctx.createMediaStreamDestination();
|
|
13026
|
-
src.connect(dest);
|
|
13027
|
-
src.connect(ctx.destination);
|
|
13028
|
-
const mimeType = MediaRecorder.isTypeSupported("audio/webm;codecs=opus") ? "audio/webm;codecs=opus" : MediaRecorder.isTypeSupported("audio/webm") ? "audio/webm" : "audio/ogg";
|
|
13029
|
-
const recorder = new MediaRecorder(dest.stream, { mimeType, audioBitsPerSecond: 48e3 });
|
|
13030
|
-
const chunks = [];
|
|
13031
|
-
recorder.ondataavailable = (e) => {
|
|
13032
|
-
if (e.data.size > 0) chunks.push(e.data);
|
|
13033
|
-
};
|
|
13034
|
-
recorder.onstop = async () => {
|
|
13035
|
-
const blob = new Blob(chunks, { type: mimeType });
|
|
13036
|
-
const ab = await blob.arrayBuffer();
|
|
13037
|
-
const arr = new Uint8Array(ab);
|
|
13038
|
-
let binary = "";
|
|
13039
|
-
const chunkSize = 8192;
|
|
13040
|
-
for (let i = 0; i < arr.length; i += chunkSize) {
|
|
13041
|
-
binary += String.fromCharCode(...arr.slice(i, Math.min(i + chunkSize, arr.length)));
|
|
13042
|
-
}
|
|
13043
|
-
resolve({ ok: true, base64: btoa(binary), size: ab.byteLength, mimeType });
|
|
13044
|
-
};
|
|
13045
|
-
recorder.start(1e3);
|
|
13046
|
-
video.playbackRate = 2;
|
|
13047
|
-
video.play().catch(() => void 0);
|
|
13048
|
-
setTimeout(() => {
|
|
13049
|
-
recorder.stop();
|
|
13050
|
-
ctx.close();
|
|
13051
|
-
}, recordSecs * 1e3);
|
|
13052
|
-
} catch (e) {
|
|
13053
|
-
resolve({ ok: false, error: String(e) });
|
|
13054
|
-
}
|
|
13055
|
-
});
|
|
13056
|
-
}, WHISPER_RECORD_SECONDS);
|
|
13057
|
-
if (!result.ok || !result.base64 || !result.mimeType) return null;
|
|
13058
|
-
return { bytes: Buffer.from(result.base64, "base64"), mimeType: result.mimeType };
|
|
13059
|
-
}
|
|
13060
|
-
async function attemptKernelWhisper(videoId, kernelApiKey, kernelProxyId, falKey, start) {
|
|
13061
|
-
const driver = new BrowserDriver();
|
|
13692
|
+
async function transcribeWithWizper(videoId, audio, startMs) {
|
|
13693
|
+
const falKey = process.env.FAL_KEY;
|
|
13694
|
+
if (!falKey) return null;
|
|
13062
13695
|
try {
|
|
13063
|
-
await driver.launch({ kernelApiKey, kernelProxyId, headless: true, viewport: { width: 1280, height: 800 }, locale: "en-US" });
|
|
13064
|
-
const page = driver.getPage();
|
|
13065
|
-
await driver.navigateTo(`https://www.youtube.com/watch?v=${videoId}`);
|
|
13066
|
-
await page.waitForFunction(
|
|
13067
|
-
() => !!window.ytcfg,
|
|
13068
|
-
{ timeout: 2e4 }
|
|
13069
|
-
);
|
|
13070
|
-
const playStatus = await page.evaluate(() => {
|
|
13071
|
-
const w = window;
|
|
13072
|
-
return w.ytInitialPlayerResponse?.playabilityStatus?.status;
|
|
13073
|
-
});
|
|
13074
|
-
if (playStatus !== "OK") {
|
|
13075
|
-
await driver.close();
|
|
13076
|
-
return null;
|
|
13077
|
-
}
|
|
13078
|
-
const audioInfo = await captureAudioViaMediaRecorder(page);
|
|
13079
|
-
await driver.close();
|
|
13080
|
-
if (!audioInfo || audioInfo.bytes.length < 1e4) return null;
|
|
13081
13696
|
import_client3.fal.config({ credentials: falKey });
|
|
13082
|
-
const ext =
|
|
13083
|
-
const audioFile = new File([new Uint8Array(
|
|
13697
|
+
const ext = audio.mimeType.includes("webm") ? "webm" : audio.mimeType.includes("mp4") ? "m4a" : "audio";
|
|
13698
|
+
const audioFile = new File([new Uint8Array(audio.bytes)], `audio.${ext}`, { type: audio.mimeType });
|
|
13084
13699
|
const uploadedUrl = await import_client3.fal.storage.upload(audioFile);
|
|
13700
|
+
const subscribeStartMs = Date.now();
|
|
13085
13701
|
const result = await import_client3.fal.subscribe("fal-ai/wizper", {
|
|
13086
|
-
input: { audio_url: uploadedUrl, task: "transcribe", language: "en" },
|
|
13702
|
+
input: { audio_url: uploadedUrl, task: "transcribe", language: "en", chunk_level: "segment" },
|
|
13087
13703
|
logs: false,
|
|
13088
13704
|
pollInterval: 3e3
|
|
13089
13705
|
});
|
|
13706
|
+
const subscribeDurationMs = Date.now() - subscribeStartMs;
|
|
13090
13707
|
const data = result.data;
|
|
13091
13708
|
const chunks = (data.chunks ?? []).map((c) => ({
|
|
13092
13709
|
timestamp: c.timestamp,
|
|
13093
13710
|
text: c.text.trim()
|
|
13094
13711
|
}));
|
|
13095
|
-
|
|
13096
|
-
void recordVendorUsage({ vendor: "fal_wizper", model: "fal-ai/wizper", units: audioSec / 60, unitType: "audio_min" });
|
|
13712
|
+
void recordVendorUsage({ vendor: "fal_wizper", model: "fal-ai/wizper", units: subscribeDurationMs / 1e3, unitType: "compute_sec" });
|
|
13097
13713
|
const text = data.text ?? chunks.map((c) => c.text).join(" ");
|
|
13098
13714
|
if (!text) return null;
|
|
13099
|
-
return { videoId, text, chunks, durationMs: Date.now() -
|
|
13100
|
-
} catch {
|
|
13101
|
-
|
|
13715
|
+
return { videoId, text, chunks, durationMs: Date.now() - startMs, method: "browser-whisper" };
|
|
13716
|
+
} catch (err) {
|
|
13717
|
+
console.warn(`[CaptionFetcher] wizper transcription failed for ${videoId}: ${err instanceof Error ? err.message : String(err)}`);
|
|
13102
13718
|
return null;
|
|
13103
13719
|
}
|
|
13104
13720
|
}
|
|
13105
|
-
async function
|
|
13721
|
+
async function resolveResidentialProxy(kernelApiKey) {
|
|
13722
|
+
try {
|
|
13723
|
+
const resolved = await resolveKernelProxyId({
|
|
13724
|
+
kernelApiKey,
|
|
13725
|
+
proxyMode: "location",
|
|
13726
|
+
location: YT_RESIDENTIAL_LOCATION,
|
|
13727
|
+
gl: "us",
|
|
13728
|
+
fresh: true
|
|
13729
|
+
});
|
|
13730
|
+
return { kernelProxyId: resolved.kernelProxyId, disposableProxyId: resolved.disposableProxyId };
|
|
13731
|
+
} catch (err) {
|
|
13732
|
+
console.warn(`[CaptionFetcher] residential proxy resolution failed: ${err instanceof Error ? err.message : String(err)}`);
|
|
13733
|
+
return {};
|
|
13734
|
+
}
|
|
13735
|
+
}
|
|
13736
|
+
async function probeAndExtract(videoId, opts) {
|
|
13106
13737
|
const kernelApiKey = browserServiceApiKey();
|
|
13107
|
-
|
|
13108
|
-
const
|
|
13109
|
-
|
|
13738
|
+
if (!kernelApiKey) return null;
|
|
13739
|
+
const kernelProxyId = opts?.kernelProxyId ?? browserServiceProxyId();
|
|
13740
|
+
const driver = new BrowserDriver();
|
|
13741
|
+
return runWithCostContext({ ...currentCostContext(), op: "yt_transcription", subOp: "browser-innertube" }, async () => {
|
|
13742
|
+
let entries = [];
|
|
13743
|
+
let audio = null;
|
|
13744
|
+
let retryWithFreshIp = false;
|
|
13745
|
+
try {
|
|
13746
|
+
await driver.launch({ kernelApiKey, kernelProxyId, headless: true, headlessMode: "headless", viewport: { width: 1280, height: 800 }, locale: "en-US" });
|
|
13747
|
+
const page = driver.getPage();
|
|
13748
|
+
await driver.navigateTo(`https://www.youtube.com/watch?v=${videoId}`);
|
|
13749
|
+
await page.waitForFunction(
|
|
13750
|
+
() => !!window.ytcfg,
|
|
13751
|
+
{ timeout: 2e4 }
|
|
13752
|
+
);
|
|
13753
|
+
const probe = await probePlayer(page, videoId);
|
|
13754
|
+
if (!probe) return null;
|
|
13755
|
+
if (probe.captions && !opts?.skipCaptions) {
|
|
13756
|
+
entries = parseTimedtextJson3(probe.captions);
|
|
13757
|
+
if (entries.length === 0) entries = parseTimedtextXml(probe.captions);
|
|
13758
|
+
}
|
|
13759
|
+
if (entries.length === 0) {
|
|
13760
|
+
if (!probe.audio) {
|
|
13761
|
+
retryWithFreshIp = isBotBlockStatus(probe.status);
|
|
13762
|
+
console.warn(`[CaptionFetcher] no captions and no audio stream for ${videoId} (playability=${probe.status ?? "unknown"}, proxied=${Boolean(kernelProxyId)})`);
|
|
13763
|
+
return { entries, audio: null, retryWithFreshIp };
|
|
13764
|
+
}
|
|
13765
|
+
audio = await downloadWholeAudioStreamInOneRequestBecauseYouTubeAllowsOnlyOnePerIp(page, probe.audio);
|
|
13766
|
+
if (!audio) return { entries, audio: null, retryWithFreshIp: true };
|
|
13767
|
+
}
|
|
13768
|
+
} catch (err) {
|
|
13769
|
+
console.warn(`[CaptionFetcher] kernel session failed for ${videoId}: ${err instanceof Error ? err.message : String(err)}`);
|
|
13770
|
+
return null;
|
|
13771
|
+
} finally {
|
|
13772
|
+
const servedSubOp = audio ? "browser-whisper" : "browser-innertube";
|
|
13773
|
+
await runWithCostContext({ ...currentCostContext(), op: "yt_transcription", subOp: servedSubOp }, () => driver.close());
|
|
13774
|
+
}
|
|
13775
|
+
return { entries, audio, retryWithFreshIp };
|
|
13776
|
+
});
|
|
13777
|
+
}
|
|
13778
|
+
async function extractWithProxyEscalation(videoId, opts) {
|
|
13779
|
+
const direct = await probeAndExtract(videoId, opts);
|
|
13780
|
+
if (direct && (direct.entries.length > 0 || direct.audio)) return direct;
|
|
13781
|
+
const shouldEscalate = direct === null || direct.retryWithFreshIp;
|
|
13782
|
+
if (!shouldEscalate) return direct;
|
|
13783
|
+
const kernelApiKey = browserServiceApiKey();
|
|
13784
|
+
if (!kernelApiKey) return direct;
|
|
13785
|
+
const { kernelProxyId, disposableProxyId } = await resolveResidentialProxy(kernelApiKey);
|
|
13786
|
+
if (!kernelProxyId) return direct;
|
|
13787
|
+
try {
|
|
13788
|
+
console.warn(`[CaptionFetcher] ${videoId}: retrying through residential proxy after bot block`);
|
|
13789
|
+
const viaProxy = await probeAndExtract(videoId, { ...opts, kernelProxyId });
|
|
13790
|
+
return viaProxy ?? direct;
|
|
13791
|
+
} finally {
|
|
13792
|
+
if (disposableProxyId) {
|
|
13793
|
+
await deleteKernelProxyId(kernelApiKey, disposableProxyId).catch(
|
|
13794
|
+
(err) => console.warn(`[CaptionFetcher] failed to delete disposable proxy: ${err instanceof Error ? err.message : String(err)}`)
|
|
13795
|
+
);
|
|
13796
|
+
}
|
|
13797
|
+
}
|
|
13798
|
+
}
|
|
13799
|
+
async function fetchViaKernelSession(videoId) {
|
|
13110
13800
|
const start = Date.now();
|
|
13111
|
-
|
|
13112
|
-
|
|
13113
|
-
|
|
13114
|
-
|
|
13801
|
+
const outcome = await extractWithProxyEscalation(videoId);
|
|
13802
|
+
if (!outcome) return null;
|
|
13803
|
+
if (outcome.entries.length > 0) {
|
|
13804
|
+
const chunks = outcome.entries.map((e) => ({
|
|
13805
|
+
timestamp: [e.start / 1e3, (e.start + e.dur) / 1e3],
|
|
13806
|
+
text: e.text
|
|
13807
|
+
}));
|
|
13808
|
+
const text = chunks.map((c) => c.text).join(" ");
|
|
13809
|
+
return { videoId, text, chunks, durationMs: Date.now() - start, method: "browser-innertube" };
|
|
13810
|
+
}
|
|
13811
|
+
if (outcome.audio) {
|
|
13812
|
+
return runWithCostContext(
|
|
13813
|
+
{ ...currentCostContext(), op: "yt_transcription", subOp: "browser-whisper" },
|
|
13814
|
+
() => transcribeWithWizper(videoId, outcome.audio, start)
|
|
13815
|
+
);
|
|
13115
13816
|
}
|
|
13116
13817
|
return null;
|
|
13117
13818
|
}
|
|
@@ -13133,21 +13834,29 @@ function finalizeCaptions(result) {
|
|
|
13133
13834
|
async function fetchCaptions(videoId) {
|
|
13134
13835
|
const primary = await fetchViaYoutubeTranscript(videoId);
|
|
13135
13836
|
if (primary) return finalizeCaptions(primary);
|
|
13136
|
-
const
|
|
13137
|
-
if (
|
|
13138
|
-
|
|
13139
|
-
|
|
13140
|
-
|
|
13837
|
+
const result = await fetchViaKernelSession(videoId);
|
|
13838
|
+
if (result) return finalizeCaptions(result);
|
|
13839
|
+
throw new Error(
|
|
13840
|
+
`Could not transcribe ${videoId}. It has no caption track, and its audio stream could not be downloaded \u2014 YouTube permits a single media request per IP, so streams that do not fit in one response are unreachable. Captioned videos of any length still work.`
|
|
13841
|
+
);
|
|
13141
13842
|
}
|
|
13142
|
-
var import_client3,
|
|
13843
|
+
var import_client3, INNERTUBE_CLIENTS, MAX_AUDIO_BYTES, YT_RESIDENTIAL_LOCATION;
|
|
13143
13844
|
var init_CaptionFetcher = __esm({
|
|
13144
13845
|
"src/youtube/CaptionFetcher.ts"() {
|
|
13145
13846
|
"use strict";
|
|
13146
13847
|
init_BrowserDriver();
|
|
13147
13848
|
init_browser_service_env();
|
|
13849
|
+
init_kernel_proxy_resolver();
|
|
13148
13850
|
import_client3 = require("@fal-ai/client");
|
|
13149
13851
|
init_cost_telemetry();
|
|
13150
|
-
|
|
13852
|
+
init_cost_context();
|
|
13853
|
+
INNERTUBE_CLIENTS = [
|
|
13854
|
+
{ name: "ANDROID", version: "20.10.38", userAgent: "com.google.android.youtube/20.10.38 (Linux; U; Android 14)" },
|
|
13855
|
+
{ name: "IOS", version: "20.10.4", userAgent: "com.google.ios.youtube/20.10.4 (iPhone16,2; U; CPU iOS 18_3_2 like Mac OS X;)" },
|
|
13856
|
+
{ name: "ANDROID_VR", version: "1.62.27", userAgent: "com.google.android.apps.youtube.vr.oculus/1.62.27 (Linux; U; Android 12L) gzip" }
|
|
13857
|
+
];
|
|
13858
|
+
MAX_AUDIO_BYTES = 96 * 1024 * 1024;
|
|
13859
|
+
YT_RESIDENTIAL_LOCATION = process.env.YOUTUBE_PROXY_LOCATION ?? "Austin, Texas";
|
|
13151
13860
|
}
|
|
13152
13861
|
});
|
|
13153
13862
|
|
|
@@ -13483,6 +14192,7 @@ var init_youtube_routes = __esm({
|
|
|
13483
14192
|
init_api_auth();
|
|
13484
14193
|
init_server_schemas();
|
|
13485
14194
|
init_concurrency_gates();
|
|
14195
|
+
init_cost_context();
|
|
13486
14196
|
youtubeApp = new import_hono3.Hono();
|
|
13487
14197
|
youtubeApp.post("/harvest", createApiKeyAuth(), async (c) => {
|
|
13488
14198
|
const raw = await c.req.json().catch(() => ({}));
|
|
@@ -13560,7 +14270,7 @@ var init_youtube_routes = __esm({
|
|
|
13560
14270
|
const { ok: trOk, balance_mc: trBal } = await debitMc(user.id, holdMc, LedgerOperation.TRANSCRIPTION_HOLD, id);
|
|
13561
14271
|
if (!trOk) return c.json(insufficientBalanceResponse(trBal, holdMc), 402);
|
|
13562
14272
|
debited = true;
|
|
13563
|
-
const result = await fetchCaptions(id);
|
|
14273
|
+
const result = await runWithCostContext({ ...currentCostContext(), op: "yt_transcription", userId: user.id }, () => fetchCaptions(id));
|
|
13564
14274
|
const lastChunk = result.chunks?.[result.chunks.length - 1];
|
|
13565
14275
|
const chunkSecs = lastChunk && Number.isFinite(lastChunk.timestamp[1]) ? lastChunk.timestamp[1] : 0;
|
|
13566
14276
|
const durationSecs = Number.isFinite(result.durationMs) ? result.durationMs / 1e3 : 0;
|
|
@@ -14409,727 +15119,170 @@ async function collectAdLibraryResults(page, url, maxResults, opts = {}) {
|
|
|
14409
15119
|
try {
|
|
14410
15120
|
await page.goto(url, { waitUntil: "domcontentloaded", timeout: 45e3 });
|
|
14411
15121
|
const deadline = Date.now() + captureMs;
|
|
14412
|
-
let lastCount = -1;
|
|
14413
|
-
let stableRounds = 0;
|
|
14414
|
-
while (Date.now() < deadline && collected.length < maxResults) {
|
|
14415
|
-
await page.evaluate(() => window.scrollTo(0, document.body.scrollHeight)).catch(() => void 0);
|
|
14416
|
-
await page.waitForTimeout(2e3);
|
|
14417
|
-
if (collected.length === lastCount) {
|
|
14418
|
-
stableRounds++;
|
|
14419
|
-
if (stableRounds >= 2 && collected.length > 0) break;
|
|
14420
|
-
} else {
|
|
14421
|
-
stableRounds = 0;
|
|
14422
|
-
}
|
|
14423
|
-
lastCount = collected.length;
|
|
14424
|
-
}
|
|
14425
|
-
} finally {
|
|
14426
|
-
page.off("response", handler);
|
|
14427
|
-
}
|
|
14428
|
-
return collected.slice(0, maxResults);
|
|
14429
|
-
}
|
|
14430
|
-
function advertisersFromResults(results, maxResults) {
|
|
14431
|
-
const byPage = /* @__PURE__ */ new Map();
|
|
14432
|
-
for (const r of results) {
|
|
14433
|
-
if (!r.page_id || !r.page_name) continue;
|
|
14434
|
-
const collation = typeof r.collation_count === "number" && r.collation_count > 0 ? r.collation_count : 0;
|
|
14435
|
-
const existing = byPage.get(r.page_id);
|
|
14436
|
-
if (existing) {
|
|
14437
|
-
existing.resultCount++;
|
|
14438
|
-
existing.maxCollation = Math.max(existing.maxCollation, collation);
|
|
14439
|
-
} else {
|
|
14440
|
-
byPage.set(r.page_id, { pageName: r.page_name, pageId: r.page_id, sampleLibraryId: r.ad_archive_id, maxCollation: collation, resultCount: 1 });
|
|
14441
|
-
}
|
|
14442
|
-
}
|
|
14443
|
-
return [...byPage.values()].map((e) => ({ pageName: e.pageName, pageId: e.pageId, sampleLibraryId: e.sampleLibraryId, adCount: Math.max(e.maxCollation, e.resultCount) })).sort((a, b) => b.adCount - a.adCount).slice(0, maxResults);
|
|
14444
|
-
}
|
|
14445
|
-
var AD_LIBRARY_QUERY;
|
|
14446
|
-
var init_FacebookAdGraphql = __esm({
|
|
14447
|
-
"src/extractor/FacebookAdGraphql.ts"() {
|
|
14448
|
-
"use strict";
|
|
14449
|
-
AD_LIBRARY_QUERY = "AdLibrarySearchPaginationQuery";
|
|
14450
|
-
}
|
|
14451
|
-
});
|
|
14452
|
-
|
|
14453
|
-
// src/extractor/FacebookOrganicVideoExtractor.ts
|
|
14454
|
-
function htmlDecode(value) {
|
|
14455
|
-
return value.replace(/&/g, "&").replace(/"/g, '"').replace(/'/g, "'").replace(/</g, "<").replace(/>/g, ">");
|
|
14456
|
-
}
|
|
14457
|
-
function decodeEscapedValue(raw) {
|
|
14458
|
-
let out = raw;
|
|
14459
|
-
try {
|
|
14460
|
-
out = JSON.parse(`"${raw.replace(/"/g, '\\"')}"`);
|
|
14461
|
-
} catch {
|
|
14462
|
-
out = raw.replace(/\\u0025/g, "%").replace(/\\u0026/g, "&").replace(/\\u003D/g, "=").replace(/\\u003d/g, "=").replace(/\\"/g, '"').replace(/\\\//g, "/");
|
|
14463
|
-
}
|
|
14464
|
-
return htmlDecode(out);
|
|
14465
|
-
}
|
|
14466
|
-
function facebookVideoIdFromUrl(url) {
|
|
14467
|
-
try {
|
|
14468
|
-
const parsed = new URL(url);
|
|
14469
|
-
const path6 = parsed.pathname;
|
|
14470
|
-
const direct = path6.match(/\/(?:reel|videos|watch)\/(?:[^/]+\/)?(\d{8,})/i);
|
|
14471
|
-
if (direct) return direct[1];
|
|
14472
|
-
const queryVideoId = parsed.searchParams.get("v") ?? parsed.searchParams.get("video_id");
|
|
14473
|
-
if (queryVideoId && /^\d{8,}$/.test(queryVideoId)) return queryVideoId;
|
|
14474
|
-
const numericTail = path6.match(/\/(\d{8,})\/?$/);
|
|
14475
|
-
if (numericTail) return numericTail[1];
|
|
14476
|
-
} catch {
|
|
14477
|
-
}
|
|
14478
|
-
return null;
|
|
14479
|
-
}
|
|
14480
|
-
function parseEfg(url) {
|
|
14481
|
-
try {
|
|
14482
|
-
const efg = new URL(url).searchParams.get("efg");
|
|
14483
|
-
if (!efg) return null;
|
|
14484
|
-
return JSON.parse(Buffer.from(efg, "base64").toString("utf8"));
|
|
14485
|
-
} catch {
|
|
14486
|
-
return null;
|
|
14487
|
-
}
|
|
14488
|
-
}
|
|
14489
|
-
function manifestDurationSec(htmlWindow) {
|
|
14490
|
-
const decoded = decodeEscapedValue(htmlWindow);
|
|
14491
|
-
const match = decoded.match(/mediaPresentationDuration="PT([\d.]+)S"/);
|
|
14492
|
-
return match ? Number(match[1]) : null;
|
|
14493
|
-
}
|
|
14494
|
-
function metadataContent(html, key) {
|
|
14495
|
-
const escaped = key.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
|
14496
|
-
const re = new RegExp(`<meta\\s+[^>]*(?:property|name)=["']${escaped}["'][^>]*content=["']([^"']+)["'][^>]*>`, "i");
|
|
14497
|
-
const m = html.match(re);
|
|
14498
|
-
return m ? htmlDecode(m[1]).trim() : null;
|
|
14499
|
-
}
|
|
14500
|
-
function ownerNameFromHtml(html) {
|
|
14501
|
-
const ogTitle = metadataContent(html, "og:title");
|
|
14502
|
-
const pipe = ogTitle?.split("|").map((s) => s.trim()).filter(Boolean);
|
|
14503
|
-
if (pipe && pipe.length > 1) return pipe[pipe.length - 1] ?? null;
|
|
14504
|
-
return null;
|
|
14505
|
-
}
|
|
14506
|
-
function extractFacebookOrganicVideo(html, pageUrl, sourceUrl = pageUrl, quality = "best") {
|
|
14507
|
-
const videoId = facebookVideoIdFromUrl(pageUrl) ?? facebookVideoIdFromUrl(sourceUrl);
|
|
14508
|
-
const manifestIndex = videoId ? html.indexOf(`dash_mpd_debug.mpd?v=${videoId}`) : -1;
|
|
14509
|
-
const searchWindow = manifestIndex >= 0 ? html.slice(Math.max(0, manifestIndex - 5e4), Math.min(html.length, manifestIndex + 5e4)) : html;
|
|
14510
|
-
const targetDuration = manifestDurationSec(searchWindow);
|
|
14511
|
-
const candidates = [];
|
|
14512
|
-
const seen = /* @__PURE__ */ new Set();
|
|
14513
|
-
const re = /browser_native_(sd|hd)_url\\?"\s*:\s*\\?"([^"<]+?)\\?"/g;
|
|
14514
|
-
let match;
|
|
14515
|
-
while ((match = re.exec(searchWindow)) !== null) {
|
|
14516
|
-
const url = decodeEscapedValue(match[2]);
|
|
14517
|
-
if (!url || seen.has(url)) continue;
|
|
14518
|
-
seen.add(url);
|
|
14519
|
-
const parsed = parseEfg(url);
|
|
14520
|
-
let bitrate = null;
|
|
14521
|
-
try {
|
|
14522
|
-
const value = new URL(url).searchParams.get("bitrate");
|
|
14523
|
-
bitrate = value ? Number(value) : null;
|
|
14524
|
-
} catch {
|
|
14525
|
-
}
|
|
14526
|
-
candidates.push({
|
|
14527
|
-
url,
|
|
14528
|
-
quality: match[1],
|
|
14529
|
-
bitrate: Number.isFinite(bitrate) ? bitrate : null,
|
|
14530
|
-
durationSec: typeof parsed?.duration_s === "number" ? parsed.duration_s : null,
|
|
14531
|
-
assetId: typeof parsed?.xpv_asset_id === "number" ? parsed.xpv_asset_id : null,
|
|
14532
|
-
vencodeTag: typeof parsed?.vencode_tag === "string" ? parsed.vencode_tag : null
|
|
14533
|
-
});
|
|
14534
|
-
}
|
|
14535
|
-
const targetCandidates = candidates.filter((candidate) => {
|
|
14536
|
-
if (!targetDuration || candidate.durationSec == null) return true;
|
|
14537
|
-
return Math.abs(candidate.durationSec - targetDuration) <= 2;
|
|
14538
|
-
});
|
|
14539
|
-
const selectable = targetCandidates.length ? targetCandidates : candidates;
|
|
14540
|
-
const selected = [...selectable].filter((candidate) => quality === "best" || candidate.quality === quality).sort((a, b) => {
|
|
14541
|
-
const aq = a.quality === "hd" ? 1 : 0;
|
|
14542
|
-
const bq = b.quality === "hd" ? 1 : 0;
|
|
14543
|
-
if (quality === "best" && aq !== bq) return bq - aq;
|
|
14544
|
-
return (b.bitrate ?? 0) - (a.bitrate ?? 0);
|
|
14545
|
-
})[0];
|
|
14546
|
-
if (!selected) {
|
|
14547
|
-
throw new Error("No progressive Facebook video URL was found in the rendered page state");
|
|
14548
|
-
}
|
|
14549
|
-
return {
|
|
14550
|
-
sourceUrl,
|
|
14551
|
-
pageUrl,
|
|
14552
|
-
videoId,
|
|
14553
|
-
title: metadataContent(html, "og:title") ?? metadataContent(html, "twitter:title"),
|
|
14554
|
-
description: metadataContent(html, "og:description") ?? metadataContent(html, "description") ?? metadataContent(html, "twitter:description"),
|
|
14555
|
-
ownerName: ownerNameFromHtml(html),
|
|
14556
|
-
videoUrl: selected.url,
|
|
14557
|
-
selectedQuality: selected.quality,
|
|
14558
|
-
bitrate: selected.bitrate,
|
|
14559
|
-
durationSec: targetDuration ?? selected.durationSec,
|
|
14560
|
-
assetId: selected.assetId,
|
|
14561
|
-
vencodeTag: selected.vencodeTag,
|
|
14562
|
-
candidates: selectable,
|
|
14563
|
-
extractedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
14564
|
-
};
|
|
14565
|
-
}
|
|
14566
|
-
async function extractFacebookOrganicVideoFromPage(page, sourceUrl, quality = "best") {
|
|
14567
|
-
await page.waitForLoadState("domcontentloaded", { timeout: 15e3 }).catch(() => {
|
|
14568
|
-
});
|
|
14569
|
-
await page.waitForTimeout(1500);
|
|
14570
|
-
const html = await page.content();
|
|
14571
|
-
return extractFacebookOrganicVideo(html, page.url(), sourceUrl, quality);
|
|
14572
|
-
}
|
|
14573
|
-
var init_FacebookOrganicVideoExtractor = __esm({
|
|
14574
|
-
"src/extractor/FacebookOrganicVideoExtractor.ts"() {
|
|
14575
|
-
"use strict";
|
|
14576
|
-
}
|
|
14577
|
-
});
|
|
14578
|
-
|
|
14579
|
-
// src/locations.ts
|
|
14580
|
-
var LOCATIONS;
|
|
14581
|
-
var init_locations = __esm({
|
|
14582
|
-
"src/locations.ts"() {
|
|
14583
|
-
"use strict";
|
|
14584
|
-
LOCATIONS = {
|
|
14585
|
-
"austin": "Austin,Texas,United States",
|
|
14586
|
-
"new york": "New York,New York,United States",
|
|
14587
|
-
"new york city": "New York,New York,United States",
|
|
14588
|
-
"nyc": "New York,New York,United States",
|
|
14589
|
-
"los angeles": "Los Angeles,California,United States",
|
|
14590
|
-
"la": "Los Angeles,California,United States",
|
|
14591
|
-
"chicago": "Chicago,Illinois,United States",
|
|
14592
|
-
"houston": "Houston,Texas,United States",
|
|
14593
|
-
"phoenix": "Phoenix,Arizona,United States",
|
|
14594
|
-
"philadelphia": "Philadelphia,Pennsylvania,United States",
|
|
14595
|
-
"philly": "Philadelphia,Pennsylvania,United States",
|
|
14596
|
-
"san antonio": "San Antonio,Texas,United States",
|
|
14597
|
-
"dallas": "Dallas,Texas,United States",
|
|
14598
|
-
"miami": "Miami,Florida,United States",
|
|
14599
|
-
"seattle": "Seattle,Washington,United States",
|
|
14600
|
-
"denver": "Denver,Colorado,United States",
|
|
14601
|
-
"loveland": "Loveland,Colorado,United States",
|
|
14602
|
-
"loveland co": "Loveland,Colorado,United States",
|
|
14603
|
-
"fort collins": "Fort Collins,Colorado,United States",
|
|
14604
|
-
"boulder": "Boulder,Colorado,United States",
|
|
14605
|
-
"colorado springs": "Colorado Springs,Colorado,United States",
|
|
14606
|
-
"boston": "Boston,Massachusetts,United States",
|
|
14607
|
-
"atlanta": "Atlanta,Georgia,United States",
|
|
14608
|
-
"san francisco": "San Francisco,California,United States",
|
|
14609
|
-
"sf": "San Francisco,California,United States",
|
|
14610
|
-
"portland": "Portland,Oregon,United States",
|
|
14611
|
-
"las vegas": "Las Vegas,Nevada,United States",
|
|
14612
|
-
"minneapolis": "Minneapolis,Minnesota,United States",
|
|
14613
|
-
"detroit": "Detroit,Michigan,United States",
|
|
14614
|
-
"nashville": "Nashville,Tennessee,United States",
|
|
14615
|
-
"charlotte": "Charlotte,North Carolina,United States",
|
|
14616
|
-
"orlando": "Orlando,Florida,United States",
|
|
14617
|
-
"san diego": "San Diego,California,United States",
|
|
14618
|
-
"baltimore": "Baltimore,Maryland,United States",
|
|
14619
|
-
"sacramento": "Sacramento,California,United States",
|
|
14620
|
-
"columbus": "Columbus,Ohio,United States",
|
|
14621
|
-
"indianapolis": "Indianapolis,Indiana,United States",
|
|
14622
|
-
"san jose": "San Jose,California,United States",
|
|
14623
|
-
"fort worth": "Fort Worth,Texas,United States",
|
|
14624
|
-
"jacksonville": "Jacksonville,Florida,United States",
|
|
14625
|
-
"memphis": "Memphis,Tennessee,United States",
|
|
14626
|
-
"louisville": "Louisville,Kentucky,United States",
|
|
14627
|
-
"raleigh": "Raleigh,North Carolina,United States",
|
|
14628
|
-
"richmond": "Richmond,Virginia,United States",
|
|
14629
|
-
"salt lake city": "Salt Lake City,Utah,United States",
|
|
14630
|
-
"toronto": "Toronto,Ontario,Canada",
|
|
14631
|
-
"vancouver": "Vancouver,British Columbia,Canada",
|
|
14632
|
-
"montreal": "Montreal,Quebec,Canada",
|
|
14633
|
-
"calgary": "Calgary,Alberta,Canada",
|
|
14634
|
-
"ottawa": "Ottawa,Ontario,Canada",
|
|
14635
|
-
"london": "London,England,United Kingdom",
|
|
14636
|
-
"manchester": "Manchester,England,United Kingdom",
|
|
14637
|
-
"birmingham": "Birmingham,England,United Kingdom",
|
|
14638
|
-
"edinburgh": "Edinburgh,Scotland,United Kingdom",
|
|
14639
|
-
"glasgow": "Glasgow,Scotland,United Kingdom",
|
|
14640
|
-
"leeds": "Leeds,England,United Kingdom",
|
|
14641
|
-
"sydney": "Sydney,New South Wales,Australia",
|
|
14642
|
-
"melbourne": "Melbourne,Victoria,Australia",
|
|
14643
|
-
"brisbane": "Brisbane,Queensland,Australia",
|
|
14644
|
-
"perth": "Perth,Western Australia,Australia",
|
|
14645
|
-
"adelaide": "Adelaide,South Australia,Australia",
|
|
14646
|
-
"dublin": "Dublin,Leinster,Ireland"
|
|
14647
|
-
};
|
|
14648
|
-
}
|
|
14649
|
-
});
|
|
14650
|
-
|
|
14651
|
-
// src/uule.ts
|
|
14652
|
-
function encodeVarint(value) {
|
|
14653
|
-
const bytes = [];
|
|
14654
|
-
let remaining = value;
|
|
14655
|
-
do {
|
|
14656
|
-
let byte = remaining & 127;
|
|
14657
|
-
remaining >>>= 7;
|
|
14658
|
-
if (remaining > 0) byte |= 128;
|
|
14659
|
-
bytes.push(byte);
|
|
14660
|
-
} while (remaining > 0);
|
|
14661
|
-
return bytes;
|
|
14662
|
-
}
|
|
14663
|
-
function encodeUule(name) {
|
|
14664
|
-
const locationBytes = Buffer.from(name, "utf8");
|
|
14665
|
-
const payload = Buffer.concat([
|
|
14666
|
-
Buffer.from([8, 2, 16, 32, 34]),
|
|
14667
|
-
Buffer.from(encodeVarint(locationBytes.length)),
|
|
14668
|
-
locationBytes
|
|
14669
|
-
]);
|
|
14670
|
-
return `w+${payload.toString("base64")}`;
|
|
14671
|
-
}
|
|
14672
|
-
function normalizeLocation(input) {
|
|
14673
|
-
const raw = input.toLowerCase().trim();
|
|
14674
|
-
if (LOCATIONS[raw]) return LOCATIONS[raw];
|
|
14675
|
-
const beforeComma = raw.split(",")[0].trim();
|
|
14676
|
-
if (beforeComma !== raw && LOCATIONS[beforeComma]) return LOCATIONS[beforeComma];
|
|
14677
|
-
const withoutState = raw.replace(/\s+[a-z]{2}$/, "").trim();
|
|
14678
|
-
if (withoutState !== raw && LOCATIONS[withoutState]) return LOCATIONS[withoutState];
|
|
14679
|
-
return input;
|
|
14680
|
-
}
|
|
14681
|
-
var init_uule = __esm({
|
|
14682
|
-
"src/uule.ts"() {
|
|
14683
|
-
"use strict";
|
|
14684
|
-
init_locations();
|
|
14685
|
-
}
|
|
14686
|
-
});
|
|
14687
|
-
|
|
14688
|
-
// src/kernel-proxy-resolver.ts
|
|
14689
|
-
function proxyIdSuffix2(proxyId) {
|
|
14690
|
-
return proxyId ? proxyId.slice(-6) : null;
|
|
14691
|
-
}
|
|
14692
|
-
function resolution(source, proxyMode, proxyId, target, error, disposable = false) {
|
|
14693
|
-
const proxyType = source === "disabled" ? "none" : source === "configured_fallback" ? "configured" : source === "isp_created" ? "isp" : source === "location_created" || source === "location_reused" ? "residential" : "unknown";
|
|
14694
|
-
return {
|
|
14695
|
-
kernelProxyId: proxyId,
|
|
14696
|
-
...disposable && proxyId ? { disposableProxyId: proxyId } : {},
|
|
14697
|
-
resolution: {
|
|
14698
|
-
source,
|
|
14699
|
-
proxyType,
|
|
14700
|
-
proxyMode,
|
|
14701
|
-
proxyIdPresent: Boolean(proxyId),
|
|
14702
|
-
proxyIdSuffix: proxyIdSuffix2(proxyId),
|
|
14703
|
-
target,
|
|
14704
|
-
error,
|
|
14705
|
-
disposable
|
|
14706
|
-
}
|
|
14707
|
-
};
|
|
14708
|
-
}
|
|
14709
|
-
function normalizeStateName(value) {
|
|
14710
|
-
return value.trim().toLowerCase().replace(/\s+/g, " ");
|
|
14711
|
-
}
|
|
14712
|
-
function normalizeCountryName(value) {
|
|
14713
|
-
return value.trim().toLowerCase().replace(/\./g, "").replace(/\s+/g, " ");
|
|
14714
|
-
}
|
|
14715
|
-
function isUnitedStates(country) {
|
|
14716
|
-
if (!country) return true;
|
|
14717
|
-
const normalized = normalizeCountryName(country);
|
|
14718
|
-
return normalized === "united states" || normalized === "united states of america" || normalized === "usa" || normalized === "us";
|
|
14719
|
-
}
|
|
14720
|
-
function stateCodeFor(region) {
|
|
14721
|
-
const trimmed = region.trim();
|
|
14722
|
-
if (/^[A-Za-z]{2}$/.test(trimmed)) return trimmed.toUpperCase();
|
|
14723
|
-
return US_STATE_CODES[normalizeStateName(trimmed)] ?? null;
|
|
14724
|
-
}
|
|
14725
|
-
function kernelCityIdentifierCandidates(city) {
|
|
14726
|
-
const ascii = city.normalize("NFKD").replace(/[^\x00-\x7F]/g, "").toLowerCase();
|
|
14727
|
-
const words = ascii.split(/[^a-z0-9]+/).filter(Boolean);
|
|
14728
|
-
const underscored = words.join("_");
|
|
14729
|
-
const compact = words.join("");
|
|
14730
|
-
return Array.from(new Set([underscored, compact].filter(Boolean)));
|
|
14731
|
-
}
|
|
14732
|
-
function proxyName(country, state, city) {
|
|
14733
|
-
return city ? `mcp-serp-residential-${country.toLowerCase()}-${state.toLowerCase()}-${city}` : `mcp-serp-residential-${country.toLowerCase()}-${state.toLowerCase()}`;
|
|
14734
|
-
}
|
|
14735
|
-
function freshProxyName(baseName, attemptIndex) {
|
|
14736
|
-
const stamp = `${Date.now()}-${attemptIndex ?? 0}-${Math.random().toString(36).slice(2, 8)}`;
|
|
14737
|
-
return `${baseName}-fresh-${stamp}`;
|
|
14738
|
-
}
|
|
14739
|
-
function zipProxyName(zip) {
|
|
14740
|
-
return `mcp-serp-residential-us-zip-${zip}`;
|
|
14741
|
-
}
|
|
14742
|
-
function parseKernelLocationProxyTarget(location2, gl) {
|
|
14743
|
-
if (!location2 || gl.toLowerCase() !== "us") return null;
|
|
14744
|
-
const canonicalLocation = normalizeLocation(location2);
|
|
14745
|
-
let parts = canonicalLocation.split(",").map((part) => part.trim()).filter(Boolean);
|
|
14746
|
-
if (parts.length > 1 && isUnitedStates(parts[parts.length - 1])) {
|
|
14747
|
-
parts = parts.slice(0, -1);
|
|
14748
|
-
}
|
|
14749
|
-
if (parts.length === 1) {
|
|
14750
|
-
const stateOnly = stateCodeFor(parts[0]);
|
|
14751
|
-
if (!stateOnly) return null;
|
|
14752
|
-
return {
|
|
14753
|
-
canonicalLocation,
|
|
14754
|
-
level: "state",
|
|
14755
|
-
country: "US",
|
|
14756
|
-
state: stateOnly,
|
|
14757
|
-
city: "",
|
|
14758
|
-
cityCandidates: [],
|
|
14759
|
-
proxyName: proxyName("US", stateOnly),
|
|
14760
|
-
config: {
|
|
14761
|
-
country: "US",
|
|
14762
|
-
state: stateOnly
|
|
15122
|
+
let lastCount = -1;
|
|
15123
|
+
let stableRounds = 0;
|
|
15124
|
+
while (Date.now() < deadline && collected.length < maxResults) {
|
|
15125
|
+
await page.evaluate(() => window.scrollTo(0, document.body.scrollHeight)).catch(() => void 0);
|
|
15126
|
+
await page.waitForTimeout(2e3);
|
|
15127
|
+
if (collected.length === lastCount) {
|
|
15128
|
+
stableRounds++;
|
|
15129
|
+
if (stableRounds >= 2 && collected.length > 0) break;
|
|
15130
|
+
} else {
|
|
15131
|
+
stableRounds = 0;
|
|
14763
15132
|
}
|
|
14764
|
-
|
|
14765
|
-
}
|
|
14766
|
-
const [city = "", region = ""] = parts;
|
|
14767
|
-
if (!city || !region) return null;
|
|
14768
|
-
const state = stateCodeFor(region);
|
|
14769
|
-
if (!state) return null;
|
|
14770
|
-
const cityCandidates = kernelCityIdentifierCandidates(city);
|
|
14771
|
-
const primaryCity = cityCandidates[0];
|
|
14772
|
-
if (!primaryCity) return null;
|
|
14773
|
-
return {
|
|
14774
|
-
canonicalLocation,
|
|
14775
|
-
level: "city",
|
|
14776
|
-
country: "US",
|
|
14777
|
-
state,
|
|
14778
|
-
city: primaryCity,
|
|
14779
|
-
cityCandidates,
|
|
14780
|
-
proxyName: proxyName("US", state, primaryCity),
|
|
14781
|
-
config: {
|
|
14782
|
-
country: "US",
|
|
14783
|
-
state,
|
|
14784
|
-
city: primaryCity
|
|
15133
|
+
lastCount = collected.length;
|
|
14785
15134
|
}
|
|
14786
|
-
}
|
|
14787
|
-
|
|
14788
|
-
|
|
14789
|
-
return
|
|
14790
|
-
}
|
|
14791
|
-
function knownZipFor(target, explicitZip) {
|
|
14792
|
-
if (explicitZip && /^\d{5}$/.test(explicitZip)) return explicitZip;
|
|
14793
|
-
return US_CITY_CENTER_ZIPS[cityZipKey(target)] ?? null;
|
|
15135
|
+
} finally {
|
|
15136
|
+
page.off("response", handler);
|
|
15137
|
+
}
|
|
15138
|
+
return collected.slice(0, maxResults);
|
|
14794
15139
|
}
|
|
14795
|
-
function
|
|
14796
|
-
|
|
14797
|
-
|
|
14798
|
-
|
|
14799
|
-
|
|
14800
|
-
|
|
14801
|
-
|
|
14802
|
-
|
|
14803
|
-
|
|
14804
|
-
|
|
15140
|
+
function advertisersFromResults(results, maxResults) {
|
|
15141
|
+
const byPage = /* @__PURE__ */ new Map();
|
|
15142
|
+
for (const r of results) {
|
|
15143
|
+
if (!r.page_id || !r.page_name) continue;
|
|
15144
|
+
const collation = typeof r.collation_count === "number" && r.collation_count > 0 ? r.collation_count : 0;
|
|
15145
|
+
const existing = byPage.get(r.page_id);
|
|
15146
|
+
if (existing) {
|
|
15147
|
+
existing.resultCount++;
|
|
15148
|
+
existing.maxCollation = Math.max(existing.maxCollation, collation);
|
|
15149
|
+
} else {
|
|
15150
|
+
byPage.set(r.page_id, { pageName: r.page_name, pageId: r.page_id, sampleLibraryId: r.ad_archive_id, maxCollation: collation, resultCount: 1 });
|
|
14805
15151
|
}
|
|
14806
|
-
}
|
|
14807
|
-
}
|
|
14808
|
-
function withProxyName(target, name) {
|
|
14809
|
-
return {
|
|
14810
|
-
...target,
|
|
14811
|
-
proxyName: name
|
|
14812
|
-
};
|
|
15152
|
+
}
|
|
15153
|
+
return [...byPage.values()].map((e) => ({ pageName: e.pageName, pageId: e.pageId, sampleLibraryId: e.sampleLibraryId, adCount: Math.max(e.maxCollation, e.resultCount) })).sort((a, b) => b.adCount - a.adCount).slice(0, maxResults);
|
|
14813
15154
|
}
|
|
14814
|
-
|
|
14815
|
-
|
|
14816
|
-
|
|
15155
|
+
var AD_LIBRARY_QUERY;
|
|
15156
|
+
var init_FacebookAdGraphql = __esm({
|
|
15157
|
+
"src/extractor/FacebookAdGraphql.ts"() {
|
|
15158
|
+
"use strict";
|
|
15159
|
+
AD_LIBRARY_QUERY = "AdLibrarySearchPaginationQuery";
|
|
14817
15160
|
}
|
|
14818
|
-
|
|
15161
|
+
});
|
|
15162
|
+
|
|
15163
|
+
// src/extractor/FacebookOrganicVideoExtractor.ts
|
|
15164
|
+
function htmlDecode(value) {
|
|
15165
|
+
return value.replace(/&/g, "&").replace(/"/g, '"').replace(/'/g, "'").replace(/</g, "<").replace(/>/g, ">");
|
|
14819
15166
|
}
|
|
14820
|
-
function
|
|
14821
|
-
|
|
15167
|
+
function decodeEscapedValue(raw) {
|
|
15168
|
+
let out = raw;
|
|
15169
|
+
try {
|
|
15170
|
+
out = JSON.parse(`"${raw.replace(/"/g, '\\"')}"`);
|
|
15171
|
+
} catch {
|
|
15172
|
+
out = raw.replace(/\\u0025/g, "%").replace(/\\u0026/g, "&").replace(/\\u003D/g, "=").replace(/\\u003d/g, "=").replace(/\\"/g, '"').replace(/\\\//g, "/");
|
|
15173
|
+
}
|
|
15174
|
+
return htmlDecode(out);
|
|
14822
15175
|
}
|
|
14823
|
-
function
|
|
14824
|
-
|
|
14825
|
-
const
|
|
14826
|
-
const
|
|
14827
|
-
|
|
15176
|
+
function facebookVideoIdFromUrl(url) {
|
|
15177
|
+
try {
|
|
15178
|
+
const parsed = new URL(url);
|
|
15179
|
+
const path6 = parsed.pathname;
|
|
15180
|
+
const direct = path6.match(/\/(?:reel|videos|watch)\/(?:[^/]+\/)?(\d{8,})/i);
|
|
15181
|
+
if (direct) return direct[1];
|
|
15182
|
+
const queryVideoId = parsed.searchParams.get("v") ?? parsed.searchParams.get("video_id");
|
|
15183
|
+
if (queryVideoId && /^\d{8,}$/.test(queryVideoId)) return queryVideoId;
|
|
15184
|
+
const numericTail = path6.match(/\/(\d{8,})\/?$/);
|
|
15185
|
+
if (numericTail) return numericTail[1];
|
|
15186
|
+
} catch {
|
|
14828
15187
|
}
|
|
14829
15188
|
return null;
|
|
14830
15189
|
}
|
|
14831
|
-
function
|
|
14832
|
-
|
|
14833
|
-
|
|
14834
|
-
|
|
14835
|
-
|
|
14836
|
-
|
|
14837
|
-
|
|
14838
|
-
|
|
14839
|
-
}
|
|
14840
|
-
};
|
|
14841
|
-
}
|
|
14842
|
-
function findExistingStateProxy(proxies, target) {
|
|
14843
|
-
const name = proxyName(target.country, target.state);
|
|
14844
|
-
return proxies.find((proxy) => proxy.type === "residential" && proxy.status !== "unavailable" && Boolean(proxy.id) && (proxy.name === name || configMatches(proxy.config, target))) ?? null;
|
|
15190
|
+
function parseEfg(url) {
|
|
15191
|
+
try {
|
|
15192
|
+
const efg = new URL(url).searchParams.get("efg");
|
|
15193
|
+
if (!efg) return null;
|
|
15194
|
+
return JSON.parse(Buffer.from(efg, "base64").toString("utf8"));
|
|
15195
|
+
} catch {
|
|
15196
|
+
return null;
|
|
15197
|
+
}
|
|
14845
15198
|
}
|
|
14846
|
-
function
|
|
14847
|
-
|
|
15199
|
+
function manifestDurationSec(htmlWindow) {
|
|
15200
|
+
const decoded = decodeEscapedValue(htmlWindow);
|
|
15201
|
+
const match = decoded.match(/mediaPresentationDuration="PT([\d.]+)S"/);
|
|
15202
|
+
return match ? Number(match[1]) : null;
|
|
14848
15203
|
}
|
|
14849
|
-
function
|
|
14850
|
-
|
|
15204
|
+
function metadataContent(html, key) {
|
|
15205
|
+
const escaped = key.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
|
15206
|
+
const re = new RegExp(`<meta\\s+[^>]*(?:property|name)=["']${escaped}["'][^>]*content=["']([^"']+)["'][^>]*>`, "i");
|
|
15207
|
+
const m = html.match(re);
|
|
15208
|
+
return m ? htmlDecode(m[1]).trim() : null;
|
|
14851
15209
|
}
|
|
14852
|
-
function
|
|
14853
|
-
const
|
|
14854
|
-
const
|
|
14855
|
-
if (
|
|
14856
|
-
|
|
14857
|
-
out.push(withProxyName(targetZip, freshProxyName(targetZip.proxyName, attemptIndex)));
|
|
14858
|
-
}
|
|
14859
|
-
for (const city of target.cityCandidates) {
|
|
14860
|
-
const cityTarget = {
|
|
14861
|
-
...target,
|
|
14862
|
-
level: "city",
|
|
14863
|
-
city,
|
|
14864
|
-
proxyName: proxyName(target.country, target.state, city),
|
|
14865
|
-
config: {
|
|
14866
|
-
country: target.country,
|
|
14867
|
-
state: target.state,
|
|
14868
|
-
city
|
|
14869
|
-
}
|
|
14870
|
-
};
|
|
14871
|
-
out.push(withProxyName(cityTarget, freshProxyName(cityTarget.proxyName, attemptIndex)));
|
|
14872
|
-
}
|
|
14873
|
-
const fallbackTarget = stateTarget(target);
|
|
14874
|
-
out.push(withProxyName(fallbackTarget, freshProxyName(fallbackTarget.proxyName, attemptIndex)));
|
|
14875
|
-
return out;
|
|
15210
|
+
function ownerNameFromHtml(html) {
|
|
15211
|
+
const ogTitle = metadataContent(html, "og:title");
|
|
15212
|
+
const pipe = ogTitle?.split("|").map((s) => s.trim()).filter(Boolean);
|
|
15213
|
+
if (pipe && pipe.length > 1) return pipe[pipe.length - 1] ?? null;
|
|
15214
|
+
return null;
|
|
14876
15215
|
}
|
|
14877
|
-
|
|
14878
|
-
const
|
|
14879
|
-
|
|
15216
|
+
function extractFacebookOrganicVideo(html, pageUrl, sourceUrl = pageUrl, quality = "best") {
|
|
15217
|
+
const videoId = facebookVideoIdFromUrl(pageUrl) ?? facebookVideoIdFromUrl(sourceUrl);
|
|
15218
|
+
const manifestIndex = videoId ? html.indexOf(`dash_mpd_debug.mpd?v=${videoId}`) : -1;
|
|
15219
|
+
const searchWindow = manifestIndex >= 0 ? html.slice(Math.max(0, manifestIndex - 5e4), Math.min(html.length, manifestIndex + 5e4)) : html;
|
|
15220
|
+
const targetDuration = manifestDurationSec(searchWindow);
|
|
15221
|
+
const candidates = [];
|
|
15222
|
+
const seen = /* @__PURE__ */ new Set();
|
|
15223
|
+
const re = /browser_native_(sd|hd)_url\\?"\s*:\s*\\?"([^"<]+?)\\?"/g;
|
|
15224
|
+
let match;
|
|
15225
|
+
while ((match = re.exec(searchWindow)) !== null) {
|
|
15226
|
+
const url = decodeEscapedValue(match[2]);
|
|
15227
|
+
if (!url || seen.has(url)) continue;
|
|
15228
|
+
seen.add(url);
|
|
15229
|
+
const parsed = parseEfg(url);
|
|
15230
|
+
let bitrate = null;
|
|
14880
15231
|
try {
|
|
14881
|
-
const
|
|
14882
|
-
|
|
14883
|
-
|
|
14884
|
-
config: candidate.level === "zip" ? { country: candidate.country, zip: candidate.zip } : candidate.config
|
|
14885
|
-
});
|
|
14886
|
-
if (created.id) {
|
|
14887
|
-
return resolution("location_created", options.proxyMode, created.id, candidate, null, true);
|
|
14888
|
-
}
|
|
14889
|
-
createErrors.push(`${candidate.proxyName}: Kernel did not return a proxy id`);
|
|
14890
|
-
} catch (err) {
|
|
14891
|
-
createErrors.push(`${candidate.proxyName}: ${errorText2(err)}`);
|
|
15232
|
+
const value = new URL(url).searchParams.get("bitrate");
|
|
15233
|
+
bitrate = value ? Number(value) : null;
|
|
15234
|
+
} catch {
|
|
14892
15235
|
}
|
|
15236
|
+
candidates.push({
|
|
15237
|
+
url,
|
|
15238
|
+
quality: match[1],
|
|
15239
|
+
bitrate: Number.isFinite(bitrate) ? bitrate : null,
|
|
15240
|
+
durationSec: typeof parsed?.duration_s === "number" ? parsed.duration_s : null,
|
|
15241
|
+
assetId: typeof parsed?.xpv_asset_id === "number" ? parsed.xpv_asset_id : null,
|
|
15242
|
+
vencodeTag: typeof parsed?.vencode_tag === "string" ? parsed.vencode_tag : null
|
|
15243
|
+
});
|
|
14893
15244
|
}
|
|
14894
|
-
|
|
14895
|
-
|
|
14896
|
-
|
|
14897
|
-
|
|
14898
|
-
|
|
14899
|
-
|
|
14900
|
-
|
|
14901
|
-
|
|
14902
|
-
|
|
14903
|
-
|
|
14904
|
-
|
|
14905
|
-
|
|
14906
|
-
|
|
14907
|
-
createErrors.push(`isp: ${errorText2(err)}`);
|
|
14908
|
-
}
|
|
15245
|
+
const targetCandidates = candidates.filter((candidate) => {
|
|
15246
|
+
if (!targetDuration || candidate.durationSec == null) return true;
|
|
15247
|
+
return Math.abs(candidate.durationSec - targetDuration) <= 2;
|
|
15248
|
+
});
|
|
15249
|
+
const selectable = targetCandidates.length ? targetCandidates : candidates;
|
|
15250
|
+
const selected = [...selectable].filter((candidate) => quality === "best" || candidate.quality === quality).sort((a, b) => {
|
|
15251
|
+
const aq = a.quality === "hd" ? 1 : 0;
|
|
15252
|
+
const bq = b.quality === "hd" ? 1 : 0;
|
|
15253
|
+
if (quality === "best" && aq !== bq) return bq - aq;
|
|
15254
|
+
return (b.bitrate ?? 0) - (a.bitrate ?? 0);
|
|
15255
|
+
})[0];
|
|
15256
|
+
if (!selected) {
|
|
15257
|
+
throw new Error("No progressive Facebook video URL was found in the rendered page state");
|
|
14909
15258
|
}
|
|
14910
|
-
return
|
|
14911
|
-
|
|
14912
|
-
|
|
14913
|
-
|
|
14914
|
-
|
|
14915
|
-
|
|
15259
|
+
return {
|
|
15260
|
+
sourceUrl,
|
|
15261
|
+
pageUrl,
|
|
15262
|
+
videoId,
|
|
15263
|
+
title: metadataContent(html, "og:title") ?? metadataContent(html, "twitter:title"),
|
|
15264
|
+
description: metadataContent(html, "og:description") ?? metadataContent(html, "description") ?? metadataContent(html, "twitter:description"),
|
|
15265
|
+
ownerName: ownerNameFromHtml(html),
|
|
15266
|
+
videoUrl: selected.url,
|
|
15267
|
+
selectedQuality: selected.quality,
|
|
15268
|
+
bitrate: selected.bitrate,
|
|
15269
|
+
durationSec: targetDuration ?? selected.durationSec,
|
|
15270
|
+
assetId: selected.assetId,
|
|
15271
|
+
vencodeTag: selected.vencodeTag,
|
|
15272
|
+
candidates: selectable,
|
|
15273
|
+
extractedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
15274
|
+
};
|
|
14916
15275
|
}
|
|
14917
|
-
async function
|
|
14918
|
-
|
|
14919
|
-
|
|
14920
|
-
|
|
14921
|
-
|
|
14922
|
-
|
|
14923
|
-
}
|
|
14924
|
-
const target = parseKernelLocationProxyTarget(options.location, options.gl);
|
|
14925
|
-
if (!target || !options.kernelApiKey) {
|
|
14926
|
-
return resolution("configured_fallback", options.proxyMode, options.configuredKernelProxyId, target, target ? null : "location could not be normalized to a US city/state proxy target");
|
|
14927
|
-
}
|
|
14928
|
-
const kernel = new import_sdk6.default({ apiKey: options.kernelApiKey });
|
|
14929
|
-
try {
|
|
14930
|
-
const attemptIndex = options.attemptIndex ?? 0;
|
|
14931
|
-
if (options.fresh) {
|
|
14932
|
-
return await createFreshLocationProxy(kernel, options, target);
|
|
14933
|
-
}
|
|
14934
|
-
if (attemptIndex >= 1) {
|
|
14935
|
-
const escalatedTarget = escalatedTargetLevel(target, attemptIndex);
|
|
14936
|
-
const createErrors2 = [];
|
|
14937
|
-
try {
|
|
14938
|
-
const created = await kernel.proxies.create({
|
|
14939
|
-
type: "residential",
|
|
14940
|
-
name: escalatedTarget.proxyName,
|
|
14941
|
-
config: escalatedTarget.config
|
|
14942
|
-
});
|
|
14943
|
-
if (created.id) {
|
|
14944
|
-
return resolution("location_created", options.proxyMode, created.id, escalatedTarget, null);
|
|
14945
|
-
}
|
|
14946
|
-
createErrors2.push(`${escalatedTarget.state}: Kernel did not return a proxy id`);
|
|
14947
|
-
} catch (err) {
|
|
14948
|
-
createErrors2.push(`${escalatedTarget.state}: ${errorText2(err)}`);
|
|
14949
|
-
}
|
|
14950
|
-
return resolution("configured_fallback", options.proxyMode, options.configuredKernelProxyId, escalatedTarget, createErrors2.join(" | "));
|
|
14951
|
-
}
|
|
14952
|
-
const proxies = await kernel.proxies.list();
|
|
14953
|
-
const zip = knownZipFor(target, options.proxyZip);
|
|
14954
|
-
const createErrors = [];
|
|
14955
|
-
if (zip) {
|
|
14956
|
-
const targetZip = zipTarget(target, zip);
|
|
14957
|
-
const existingZip = findExistingTargetProxy(proxies, targetZip);
|
|
14958
|
-
if (existingZip?.id) {
|
|
14959
|
-
return resolution("location_reused", options.proxyMode, existingZip.id, targetZip, null);
|
|
14960
|
-
}
|
|
14961
|
-
try {
|
|
14962
|
-
const created = await kernel.proxies.create({
|
|
14963
|
-
type: "residential",
|
|
14964
|
-
name: targetZip.proxyName,
|
|
14965
|
-
config: {
|
|
14966
|
-
country: targetZip.country,
|
|
14967
|
-
zip
|
|
14968
|
-
}
|
|
14969
|
-
});
|
|
14970
|
-
if (created.id) {
|
|
14971
|
-
return resolution("location_created", options.proxyMode, created.id, targetZip, null);
|
|
14972
|
-
}
|
|
14973
|
-
createErrors.push(`${zip}: Kernel did not return a proxy id`);
|
|
14974
|
-
} catch (err) {
|
|
14975
|
-
createErrors.push(`${zip}: ${errorText2(err)}`);
|
|
14976
|
-
}
|
|
14977
|
-
}
|
|
14978
|
-
const existing = findExistingProxy(proxies, target);
|
|
14979
|
-
if (existing?.id) {
|
|
14980
|
-
return resolution("location_reused", options.proxyMode, existing.id, target, createErrors.join(" | ") || null);
|
|
14981
|
-
}
|
|
14982
|
-
for (const city of target.cityCandidates) {
|
|
14983
|
-
try {
|
|
14984
|
-
const created = await kernel.proxies.create({
|
|
14985
|
-
type: "residential",
|
|
14986
|
-
name: proxyName(target.country, target.state, city),
|
|
14987
|
-
config: {
|
|
14988
|
-
country: target.country,
|
|
14989
|
-
state: target.state,
|
|
14990
|
-
city
|
|
14991
|
-
}
|
|
14992
|
-
});
|
|
14993
|
-
if (created.id) {
|
|
14994
|
-
return resolution("location_created", options.proxyMode, created.id, {
|
|
14995
|
-
...target,
|
|
14996
|
-
level: "city",
|
|
14997
|
-
city,
|
|
14998
|
-
proxyName: proxyName(target.country, target.state, city),
|
|
14999
|
-
config: {
|
|
15000
|
-
country: target.country,
|
|
15001
|
-
state: target.state,
|
|
15002
|
-
city
|
|
15003
|
-
}
|
|
15004
|
-
}, null);
|
|
15005
|
-
}
|
|
15006
|
-
createErrors.push(`${city}: Kernel did not return a proxy id`);
|
|
15007
|
-
} catch (err) {
|
|
15008
|
-
createErrors.push(`${city}: ${errorText2(err)}`);
|
|
15009
|
-
}
|
|
15010
|
-
}
|
|
15011
|
-
const fallbackTarget = stateTarget(target);
|
|
15012
|
-
const existingState = findExistingStateProxy(proxies, fallbackTarget);
|
|
15013
|
-
if (existingState?.id) {
|
|
15014
|
-
return resolution("location_reused", options.proxyMode, existingState.id, fallbackTarget, createErrors.join(" | "));
|
|
15015
|
-
}
|
|
15016
|
-
try {
|
|
15017
|
-
const created = await kernel.proxies.create({
|
|
15018
|
-
type: "residential",
|
|
15019
|
-
name: fallbackTarget.proxyName,
|
|
15020
|
-
config: fallbackTarget.config
|
|
15021
|
-
});
|
|
15022
|
-
if (created.id) {
|
|
15023
|
-
return resolution("location_created", options.proxyMode, created.id, fallbackTarget, createErrors.join(" | "));
|
|
15024
|
-
}
|
|
15025
|
-
createErrors.push(`${fallbackTarget.state}: Kernel did not return a proxy id`);
|
|
15026
|
-
} catch (err) {
|
|
15027
|
-
createErrors.push(`${fallbackTarget.state}: ${errorText2(err)}`);
|
|
15028
|
-
}
|
|
15029
|
-
return resolution("configured_fallback", options.proxyMode, options.configuredKernelProxyId, target, createErrors.join(" | "));
|
|
15030
|
-
} catch (err) {
|
|
15031
|
-
return resolution("configured_fallback", options.proxyMode, options.configuredKernelProxyId, target, errorText2(err));
|
|
15032
|
-
}
|
|
15276
|
+
async function extractFacebookOrganicVideoFromPage(page, sourceUrl, quality = "best") {
|
|
15277
|
+
await page.waitForLoadState("domcontentloaded", { timeout: 15e3 }).catch(() => {
|
|
15278
|
+
});
|
|
15279
|
+
await page.waitForTimeout(1500);
|
|
15280
|
+
const html = await page.content();
|
|
15281
|
+
return extractFacebookOrganicVideo(html, page.url(), sourceUrl, quality);
|
|
15033
15282
|
}
|
|
15034
|
-
var
|
|
15035
|
-
|
|
15036
|
-
"src/kernel-proxy-resolver.ts"() {
|
|
15283
|
+
var init_FacebookOrganicVideoExtractor = __esm({
|
|
15284
|
+
"src/extractor/FacebookOrganicVideoExtractor.ts"() {
|
|
15037
15285
|
"use strict";
|
|
15038
|
-
import_sdk6 = __toESM(require("@onkernel/sdk"), 1);
|
|
15039
|
-
init_uule();
|
|
15040
|
-
US_STATE_CODES = {
|
|
15041
|
-
alabama: "AL",
|
|
15042
|
-
alaska: "AK",
|
|
15043
|
-
arizona: "AZ",
|
|
15044
|
-
arkansas: "AR",
|
|
15045
|
-
california: "CA",
|
|
15046
|
-
colorado: "CO",
|
|
15047
|
-
connecticut: "CT",
|
|
15048
|
-
delaware: "DE",
|
|
15049
|
-
florida: "FL",
|
|
15050
|
-
georgia: "GA",
|
|
15051
|
-
hawaii: "HI",
|
|
15052
|
-
idaho: "ID",
|
|
15053
|
-
illinois: "IL",
|
|
15054
|
-
indiana: "IN",
|
|
15055
|
-
iowa: "IA",
|
|
15056
|
-
kansas: "KS",
|
|
15057
|
-
kentucky: "KY",
|
|
15058
|
-
louisiana: "LA",
|
|
15059
|
-
maine: "ME",
|
|
15060
|
-
maryland: "MD",
|
|
15061
|
-
massachusetts: "MA",
|
|
15062
|
-
michigan: "MI",
|
|
15063
|
-
minnesota: "MN",
|
|
15064
|
-
mississippi: "MS",
|
|
15065
|
-
missouri: "MO",
|
|
15066
|
-
montana: "MT",
|
|
15067
|
-
nebraska: "NE",
|
|
15068
|
-
nevada: "NV",
|
|
15069
|
-
"new hampshire": "NH",
|
|
15070
|
-
"new jersey": "NJ",
|
|
15071
|
-
"new mexico": "NM",
|
|
15072
|
-
"new york": "NY",
|
|
15073
|
-
"north carolina": "NC",
|
|
15074
|
-
"north dakota": "ND",
|
|
15075
|
-
ohio: "OH",
|
|
15076
|
-
oklahoma: "OK",
|
|
15077
|
-
oregon: "OR",
|
|
15078
|
-
pennsylvania: "PA",
|
|
15079
|
-
"rhode island": "RI",
|
|
15080
|
-
"south carolina": "SC",
|
|
15081
|
-
"south dakota": "SD",
|
|
15082
|
-
tennessee: "TN",
|
|
15083
|
-
texas: "TX",
|
|
15084
|
-
utah: "UT",
|
|
15085
|
-
vermont: "VT",
|
|
15086
|
-
virginia: "VA",
|
|
15087
|
-
washington: "WA",
|
|
15088
|
-
"west virginia": "WV",
|
|
15089
|
-
wisconsin: "WI",
|
|
15090
|
-
wyoming: "WY"
|
|
15091
|
-
};
|
|
15092
|
-
US_CITY_CENTER_ZIPS = {
|
|
15093
|
-
"atlanta|GA": "30303",
|
|
15094
|
-
"austin|TX": "78701",
|
|
15095
|
-
"baltimore|MD": "21201",
|
|
15096
|
-
"boston|MA": "02108",
|
|
15097
|
-
"boulder|CO": "80302",
|
|
15098
|
-
"charlotte|NC": "28202",
|
|
15099
|
-
"chicago|IL": "60601",
|
|
15100
|
-
"colorado_springs|CO": "80903",
|
|
15101
|
-
"columbus|OH": "43215",
|
|
15102
|
-
"dallas|TX": "75201",
|
|
15103
|
-
"denver|CO": "80202",
|
|
15104
|
-
"detroit|MI": "48226",
|
|
15105
|
-
"fort_collins|CO": "80524",
|
|
15106
|
-
"fort_worth|TX": "76102",
|
|
15107
|
-
"houston|TX": "77002",
|
|
15108
|
-
"indianapolis|IN": "46204",
|
|
15109
|
-
"jacksonville|FL": "32202",
|
|
15110
|
-
"las_vegas|NV": "89101",
|
|
15111
|
-
"los_angeles|CA": "90012",
|
|
15112
|
-
"louisville|KY": "40202",
|
|
15113
|
-
"loveland|CO": "80537",
|
|
15114
|
-
"memphis|TN": "38103",
|
|
15115
|
-
"miami|FL": "33131",
|
|
15116
|
-
"minneapolis|MN": "55401",
|
|
15117
|
-
"nashville|TN": "37203",
|
|
15118
|
-
"new_york|NY": "10001",
|
|
15119
|
-
"orlando|FL": "32801",
|
|
15120
|
-
"philadelphia|PA": "19103",
|
|
15121
|
-
"phoenix|AZ": "85004",
|
|
15122
|
-
"portland|OR": "97205",
|
|
15123
|
-
"raleigh|NC": "27601",
|
|
15124
|
-
"richmond|VA": "23219",
|
|
15125
|
-
"sacramento|CA": "95814",
|
|
15126
|
-
"salt_lake_city|UT": "84101",
|
|
15127
|
-
"san_antonio|TX": "78205",
|
|
15128
|
-
"san_diego|CA": "92101",
|
|
15129
|
-
"san_francisco|CA": "94103",
|
|
15130
|
-
"san_jose|CA": "95113",
|
|
15131
|
-
"seattle|WA": "98101"
|
|
15132
|
-
};
|
|
15133
15286
|
}
|
|
15134
15287
|
});
|
|
15135
15288
|
|
|
@@ -15262,8 +15415,7 @@ async function transcribeMediaUrl(mediaUrl, markdownTitle = "# Media Transcript"
|
|
|
15262
15415
|
const text = data.text ?? "";
|
|
15263
15416
|
const chunks = data.chunks ?? [];
|
|
15264
15417
|
const durationMs = Date.now() - startMs;
|
|
15265
|
-
|
|
15266
|
-
void recordVendorUsage({ vendor: "fal_wizper", model: "fal-ai/wizper", units: audioSec / 60, unitType: "audio_min" });
|
|
15418
|
+
void recordVendorUsage({ vendor: "fal_wizper", model: "fal-ai/wizper", units: durationMs / 1e3, unitType: "compute_sec" });
|
|
15267
15419
|
return {
|
|
15268
15420
|
text,
|
|
15269
15421
|
chunks,
|
|
@@ -16819,6 +16971,7 @@ var init_reddit_routes = __esm({
|
|
|
16819
16971
|
if (!gate.ok) return c.json(concurrencyLimitExceededResponse(gate), 429, { "Retry-After": String(gate.retryAfterSeconds) });
|
|
16820
16972
|
let debited = false;
|
|
16821
16973
|
let refunded = false;
|
|
16974
|
+
let commentDebitMc = 0;
|
|
16822
16975
|
try {
|
|
16823
16976
|
const { ok, balance_mc } = await debitMc(user.id, MC_COSTS.reddit_thread, LedgerOperation.REDDIT_THREAD, body.url);
|
|
16824
16977
|
if (!ok) return c.json(insufficientBalanceResponse(balance_mc, MC_COSTS.reddit_thread), 402);
|
|
@@ -17080,6 +17233,8 @@ var init_video_routes = __esm({
|
|
|
17080
17233
|
init_rates();
|
|
17081
17234
|
init_api_auth();
|
|
17082
17235
|
init_memory();
|
|
17236
|
+
init_cost_telemetry();
|
|
17237
|
+
init_cost_context();
|
|
17083
17238
|
videoApp = new import_hono9.Hono();
|
|
17084
17239
|
AnalyzeBodySchema = import_zod20.z.object({
|
|
17085
17240
|
sourceUrl: import_zod20.z.string().trim().url("sourceUrl must be a direct video file URL"),
|
|
@@ -17156,6 +17311,21 @@ var init_video_routes = __esm({
|
|
|
17156
17311
|
effectiveFrames: st.frameCount ?? null
|
|
17157
17312
|
};
|
|
17158
17313
|
}
|
|
17314
|
+
if ((st.status === "done" || st.status === "failed") && typeof st.costUsd === "number" && st.costUsd > 0) {
|
|
17315
|
+
const llmOp = `${LedgerOperation.VIDEO_ANALYSIS_LLM}:${runId}`;
|
|
17316
|
+
if (!await ledgerExistsForOperation(user.id, llmOp)) {
|
|
17317
|
+
await runWithCostContext({ ...currentCostContext(), op: "video_analysis", userId: user.id }, async () => {
|
|
17318
|
+
void recordVendorUsage({ vendor: "mcp_memory_video", model: null, units: st.costUsd, unitType: "usd", error: null });
|
|
17319
|
+
});
|
|
17320
|
+
const surchargeMc = Math.round(st.costUsd * VIDEO_ANALYSIS_LLM_MARGIN_MULTIPLE * MC_PER_USD);
|
|
17321
|
+
if (surchargeMc > 0) {
|
|
17322
|
+
const { ok } = await debitMc(user.id, surchargeMc, llmOp, `real-cost-surcharge:${runId}`);
|
|
17323
|
+
if (!ok) {
|
|
17324
|
+
console.warn(`[video-routes] VIDEO_ANALYSIS_LLM surcharge debit failed for user ${user.id} (run ${runId}): insufficient balance for ${surchargeMc}mc`);
|
|
17325
|
+
}
|
|
17326
|
+
}
|
|
17327
|
+
}
|
|
17328
|
+
}
|
|
17159
17329
|
return c.json({
|
|
17160
17330
|
ok: true,
|
|
17161
17331
|
runId,
|
|
@@ -18394,7 +18564,7 @@ var init_trustpilot_routes = __esm({
|
|
|
18394
18564
|
let pagesFetched = 0;
|
|
18395
18565
|
try {
|
|
18396
18566
|
await driver.launch({
|
|
18397
|
-
headless:
|
|
18567
|
+
headless: false,
|
|
18398
18568
|
kernelApiKey: browserServiceApiKey(),
|
|
18399
18569
|
viewport: { width: 1280, height: 900 },
|
|
18400
18570
|
locale: "en-US"
|
|
@@ -18600,7 +18770,7 @@ var init_g2_routes = __esm({
|
|
|
18600
18770
|
const driver = new BrowserDriver();
|
|
18601
18771
|
try {
|
|
18602
18772
|
await driver.launch({
|
|
18603
|
-
headless:
|
|
18773
|
+
headless: false,
|
|
18604
18774
|
kernelApiKey: browserServiceApiKey(),
|
|
18605
18775
|
kernelProxyId: proxyId,
|
|
18606
18776
|
viewport: { width: 1280, height: 900 },
|
|
@@ -27117,7 +27287,7 @@ var PACKAGE_VERSION;
|
|
|
27117
27287
|
var init_version = __esm({
|
|
27118
27288
|
"src/version.ts"() {
|
|
27119
27289
|
"use strict";
|
|
27120
|
-
PACKAGE_VERSION = "0.4.
|
|
27290
|
+
PACKAGE_VERSION = "0.4.12";
|
|
27121
27291
|
}
|
|
27122
27292
|
});
|
|
27123
27293
|
|
|
@@ -38049,14 +38219,17 @@ async function processJob(job) {
|
|
|
38049
38219
|
running++;
|
|
38050
38220
|
try {
|
|
38051
38221
|
const opts = typeof job.options === "string" ? JSON.parse(job.options) : job.options;
|
|
38052
|
-
const result = await
|
|
38053
|
-
|
|
38054
|
-
|
|
38055
|
-
|
|
38056
|
-
|
|
38057
|
-
|
|
38058
|
-
|
|
38059
|
-
|
|
38222
|
+
const result = await runWithCostContext(
|
|
38223
|
+
{ op: opts.serpOnly ? "serp" : "paa", userId: Number(job.user_id) },
|
|
38224
|
+
() => harvest({
|
|
38225
|
+
...opts,
|
|
38226
|
+
kernelApiKey: browserServiceApiKey(),
|
|
38227
|
+
headless: true,
|
|
38228
|
+
format: "json",
|
|
38229
|
+
outputDir: "/tmp/paa-output-api",
|
|
38230
|
+
onAttemptEvent: createHarvestAttemptRecorder(job.id, job.user_id)
|
|
38231
|
+
})
|
|
38232
|
+
);
|
|
38060
38233
|
await completeJob(job.id, result);
|
|
38061
38234
|
const attempts = await listHarvestAttempts(job.id, job.user_id);
|
|
38062
38235
|
if (!opts.serpOnly && typeof opts.billingHoldMc === "number") {
|
|
@@ -38122,6 +38295,7 @@ var init_worker = __esm({
|
|
|
38122
38295
|
init_rates();
|
|
38123
38296
|
init_harvest_problems();
|
|
38124
38297
|
init_harvest_attempt_events();
|
|
38298
|
+
init_cost_context();
|
|
38125
38299
|
MAX_CONCURRENT = 2;
|
|
38126
38300
|
running = 0;
|
|
38127
38301
|
}
|
|
@@ -38200,9 +38374,12 @@ function opForCostPath(p) {
|
|
|
38200
38374
|
}
|
|
38201
38375
|
if (p.startsWith("/instagram/")) return "instagram";
|
|
38202
38376
|
if (p === "/reddit/thread") return "reddit_thread";
|
|
38377
|
+
if (p.startsWith("/trustpilot/")) return "trustpilot_reviews";
|
|
38378
|
+
if (p.startsWith("/g2/")) return "g2_reviews";
|
|
38203
38379
|
if (p.startsWith("/api/internal/site-architecture-auditor")) return "audit_site";
|
|
38204
38380
|
if (p.startsWith("/api/internal/memory")) return "memory_ai";
|
|
38205
38381
|
if (p.startsWith("/agent/")) return "browser_session";
|
|
38382
|
+
if (p.startsWith("/video/")) return "video_analysis";
|
|
38206
38383
|
return null;
|
|
38207
38384
|
}
|
|
38208
38385
|
function maybeProvisionInBackground(user, vaultCount) {
|
|
@@ -39035,16 +39212,20 @@ var init_server = __esm({
|
|
|
39035
39212
|
AbortSignal.timeout(syncTimeoutMs)
|
|
39036
39213
|
]);
|
|
39037
39214
|
try {
|
|
39038
|
-
const
|
|
39039
|
-
|
|
39040
|
-
|
|
39041
|
-
|
|
39042
|
-
|
|
39043
|
-
|
|
39044
|
-
|
|
39045
|
-
|
|
39046
|
-
|
|
39047
|
-
|
|
39215
|
+
const harvestCtx = currentCostContext();
|
|
39216
|
+
const result = await runWithCostContext(
|
|
39217
|
+
{ ...harvestCtx, op: options.serpOnly ? "serp" : "paa", userId: user.id },
|
|
39218
|
+
() => harvest({
|
|
39219
|
+
...options,
|
|
39220
|
+
kernelApiKey: browserServiceApiKey(),
|
|
39221
|
+
headless: true,
|
|
39222
|
+
format: "json",
|
|
39223
|
+
outputDir: "/tmp/paa-output-api",
|
|
39224
|
+
signal: syncSignal,
|
|
39225
|
+
softDeadlineMs: Date.now() + Math.floor(syncTimeoutMs * 0.8),
|
|
39226
|
+
onAttemptEvent: recordAttempt
|
|
39227
|
+
})
|
|
39228
|
+
);
|
|
39048
39229
|
await completeJob(jobId, result);
|
|
39049
39230
|
const attempts = await listHarvestAttempts(jobId, user.id);
|
|
39050
39231
|
if (!options.serpOnly) {
|