mcp-scraper 0.2.13 → 0.2.15
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +12 -2
- package/dist/bin/api-server.cjs +977 -200
- package/dist/bin/api-server.cjs.map +1 -1
- package/dist/bin/api-server.js +3 -3
- package/dist/bin/browser-agent-stdio-server.cjs +1 -1
- package/dist/bin/browser-agent-stdio-server.cjs.map +1 -1
- package/dist/bin/browser-agent-stdio-server.js +2 -2
- package/dist/bin/mcp-scraper-cli.cjs +46 -3
- package/dist/bin/mcp-scraper-cli.cjs.map +1 -1
- package/dist/bin/mcp-scraper-cli.js +42 -2
- package/dist/bin/mcp-scraper-cli.js.map +1 -1
- package/dist/bin/mcp-scraper-combined-stdio-server.cjs +135 -5
- package/dist/bin/mcp-scraper-combined-stdio-server.cjs.map +1 -1
- package/dist/bin/mcp-scraper-combined-stdio-server.js +3 -3
- package/dist/bin/mcp-scraper-install.cjs +4 -3
- package/dist/bin/mcp-scraper-install.cjs.map +1 -1
- package/dist/bin/mcp-scraper-install.js +4 -3
- package/dist/bin/mcp-scraper-install.js.map +1 -1
- package/dist/bin/mcp-stdio-server.cjs +135 -5
- package/dist/bin/mcp-stdio-server.cjs.map +1 -1
- package/dist/bin/mcp-stdio-server.js +2 -2
- package/dist/chunk-2CQXHSWC.js +7 -0
- package/dist/chunk-2CQXHSWC.js.map +1 -0
- package/dist/{chunk-ROS67BNV.js → chunk-BSYPATSM.js} +21 -2
- package/dist/chunk-BSYPATSM.js.map +1 -0
- package/dist/{chunk-AYTOCZBS.js → chunk-F44RBOJ5.js} +6 -3
- package/dist/chunk-F44RBOJ5.js.map +1 -0
- package/dist/{chunk-DKJ2XCY7.js → chunk-Q4DFONIK.js} +136 -6
- package/dist/chunk-Q4DFONIK.js.map +1 -0
- package/dist/{chunk-BQTWXY6G.js → chunk-TIPUIEJN.js} +2 -2
- package/dist/{chunk-7GCCOT3M.js → chunk-TL7YTFLH.js} +15 -1
- package/dist/chunk-TL7YTFLH.js.map +1 -0
- package/dist/{db-BVHYI57K.js → db-P5X6UQ3E.js} +2 -2
- package/dist/{server-LUNOI26E.js → server-VD2TD3AD.js} +773 -184
- package/dist/server-VD2TD3AD.js.map +1 -0
- package/dist/{worker-TDJQ6TH3.js → worker-OZSWIS3F.js} +3 -3
- package/package.json +1 -1
- package/dist/chunk-7GCCOT3M.js.map +0 -1
- package/dist/chunk-AYTOCZBS.js.map +0 -1
- package/dist/chunk-DKJ2XCY7.js.map +0 -1
- package/dist/chunk-RDROUQ4E.js +0 -7
- package/dist/chunk-RDROUQ4E.js.map +0 -1
- package/dist/chunk-ROS67BNV.js.map +0 -1
- package/dist/server-LUNOI26E.js.map +0 -1
- /package/dist/{chunk-BQTWXY6G.js.map → chunk-TIPUIEJN.js.map} +0 -0
- /package/dist/{db-BVHYI57K.js.map → db-P5X6UQ3E.js.map} +0 -0
- /package/dist/{worker-TDJQ6TH3.js.map → worker-OZSWIS3F.js.map} +0 -0
package/dist/bin/api-server.cjs
CHANGED
|
@@ -7099,6 +7099,20 @@ async function migrate() {
|
|
|
7099
7099
|
await db.execute(`ALTER TABLE users ADD COLUMN concurrency_stripe_sub_id TEXT`);
|
|
7100
7100
|
} catch {
|
|
7101
7101
|
}
|
|
7102
|
+
await db.execute(`
|
|
7103
|
+
CREATE TABLE IF NOT EXISTS concurrency_locks (
|
|
7104
|
+
id TEXT PRIMARY KEY,
|
|
7105
|
+
user_id INTEGER NOT NULL REFERENCES users(id),
|
|
7106
|
+
operation TEXT NOT NULL,
|
|
7107
|
+
status TEXT NOT NULL DEFAULT 'active',
|
|
7108
|
+
acquired_at TEXT NOT NULL DEFAULT (datetime('now')),
|
|
7109
|
+
expires_at TEXT NOT NULL,
|
|
7110
|
+
released_at TEXT,
|
|
7111
|
+
metadata TEXT
|
|
7112
|
+
)
|
|
7113
|
+
`);
|
|
7114
|
+
await db.execute(`CREATE INDEX IF NOT EXISTS concurrency_locks_user_status ON concurrency_locks(user_id, status, expires_at)`);
|
|
7115
|
+
await db.execute(`CREATE INDEX IF NOT EXISTS concurrency_locks_operation ON concurrency_locks(operation, status, expires_at)`);
|
|
7102
7116
|
await db.execute(`
|
|
7103
7117
|
CREATE TABLE IF NOT EXISTS ledger (
|
|
7104
7118
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
@@ -8672,6 +8686,19 @@ var init_site_audit_routes = __esm({
|
|
|
8672
8686
|
function browserActiveCostMc(activeMs) {
|
|
8673
8687
|
return Math.round(activeMs * MC_PER_BROWSER_MS);
|
|
8674
8688
|
}
|
|
8689
|
+
function concurrencySlotBillingInfo() {
|
|
8690
|
+
const billingUrl = process.env.CONCURRENCY_BILLING_URL ?? "https://mcpscraper.dev/billing";
|
|
8691
|
+
return {
|
|
8692
|
+
product: "Extra concurrency slot",
|
|
8693
|
+
price_label: CONCURRENCY_SLOT_PRICE_LABEL,
|
|
8694
|
+
unit_amount_usd: CONCURRENCY_SLOT_PRICE_USD,
|
|
8695
|
+
currency: CONCURRENCY_SLOT_PRICE_CURRENCY,
|
|
8696
|
+
interval: CONCURRENCY_SLOT_PRICE_INTERVAL,
|
|
8697
|
+
billing_url: billingUrl,
|
|
8698
|
+
terminal_command: CONCURRENCY_SLOT_TERMINAL_COMMAND,
|
|
8699
|
+
terminal_command_with_api_key_env: `MCP_SCRAPER_API_KEY=sk_live_your_key ${CONCURRENCY_SLOT_TERMINAL_COMMAND}`
|
|
8700
|
+
};
|
|
8701
|
+
}
|
|
8675
8702
|
function mcToCredits(mc) {
|
|
8676
8703
|
return mc / MC_PER_CREDIT;
|
|
8677
8704
|
}
|
|
@@ -8688,7 +8715,7 @@ function insufficientBalanceResponse(balanceMc, requiredMc) {
|
|
|
8688
8715
|
topup_url: topupUrl
|
|
8689
8716
|
};
|
|
8690
8717
|
}
|
|
8691
|
-
var MC_COSTS, MC_PER_BROWSER_MS, BROWSER_OPEN_MIN_BALANCE_MC, MC_PER_CREDIT, CREDIT_COST_CATALOG, CONCURRENCY_PRICE_ID, FREE_SIGNUP_MC, FREE_MONTHLY_REFRESH_MC, BALANCE_PRICE_IDS, BALANCE_PACK_LABELS, LedgerOperation;
|
|
8718
|
+
var MC_COSTS, MC_PER_BROWSER_MS, BROWSER_OPEN_MIN_BALANCE_MC, MC_PER_CREDIT, CREDIT_COST_CATALOG, CONCURRENCY_PRICE_ID, CONCURRENCY_SLOT_PRICE_USD, CONCURRENCY_SLOT_PRICE_CURRENCY, CONCURRENCY_SLOT_PRICE_INTERVAL, CONCURRENCY_SLOT_PRICE_LABEL, CONCURRENCY_SLOT_TERMINAL_COMMAND, FREE_SIGNUP_MC, FREE_MONTHLY_REFRESH_MC, BALANCE_PRICE_IDS, BALANCE_PACK_LABELS, LedgerOperation;
|
|
8692
8719
|
var init_rates = __esm({
|
|
8693
8720
|
"src/api/rates.ts"() {
|
|
8694
8721
|
"use strict";
|
|
@@ -8815,6 +8842,11 @@ var init_rates = __esm({
|
|
|
8815
8842
|
}
|
|
8816
8843
|
];
|
|
8817
8844
|
CONCURRENCY_PRICE_ID = "price_1Ta1NRS8aAcsk3TGwsRnYbix";
|
|
8845
|
+
CONCURRENCY_SLOT_PRICE_USD = 5;
|
|
8846
|
+
CONCURRENCY_SLOT_PRICE_CURRENCY = "usd";
|
|
8847
|
+
CONCURRENCY_SLOT_PRICE_INTERVAL = "month";
|
|
8848
|
+
CONCURRENCY_SLOT_PRICE_LABEL = "$5/month";
|
|
8849
|
+
CONCURRENCY_SLOT_TERMINAL_COMMAND = "npx -y -p mcp-scraper@latest mcp-scraper-cli billing concurrency checkout";
|
|
8818
8850
|
FREE_SIGNUP_MC = 5e5;
|
|
8819
8851
|
FREE_MONTHLY_REFRESH_MC = 25e4;
|
|
8820
8852
|
BALANCE_PRICE_IDS = {
|
|
@@ -10383,6 +10415,187 @@ var init_server_schemas = __esm({
|
|
|
10383
10415
|
}
|
|
10384
10416
|
});
|
|
10385
10417
|
|
|
10418
|
+
// src/api/concurrency-gates.ts
|
|
10419
|
+
function concurrencyLimitForUser(user) {
|
|
10420
|
+
return 1 + Math.max(0, Number(user.extra_concurrency_slots ?? 0));
|
|
10421
|
+
}
|
|
10422
|
+
function concurrencyLimitExceededResponse(gate) {
|
|
10423
|
+
const upgrade = concurrencySlotBillingInfo();
|
|
10424
|
+
const message = `You have ${gate.active} active operation${gate.active !== 1 ? "s" : ""}. Your account allows ${gate.limit} concurrent operation${gate.limit !== 1 ? "s" : ""}. Wait for one to finish, or add an extra concurrency slot for ${upgrade.price_label} at ${upgrade.billing_url}. Terminal upgrade: ${upgrade.terminal_command}`;
|
|
10425
|
+
return {
|
|
10426
|
+
error: message,
|
|
10427
|
+
message,
|
|
10428
|
+
error_code: "concurrency_limit_exceeded",
|
|
10429
|
+
error_type: "concurrency_limit",
|
|
10430
|
+
retryable: true,
|
|
10431
|
+
active: gate.active,
|
|
10432
|
+
limit: gate.limit,
|
|
10433
|
+
operation: gate.operation,
|
|
10434
|
+
upgrade,
|
|
10435
|
+
upgrade_url: upgrade.billing_url,
|
|
10436
|
+
upgrade_command: upgrade.terminal_command
|
|
10437
|
+
};
|
|
10438
|
+
}
|
|
10439
|
+
function isConcurrencyLimitBypassed(user) {
|
|
10440
|
+
const raw = process.env.CONCURRENCY_LIMIT_BYPASS_EMAILS ?? process.env.HARVEST_LIMIT_BYPASS_EMAILS ?? "";
|
|
10441
|
+
return raw.split(",").map((email) => email.trim()).filter(Boolean).includes(user.email);
|
|
10442
|
+
}
|
|
10443
|
+
function lockTtlModifier(ttlSeconds) {
|
|
10444
|
+
const ttl = Number.isFinite(ttlSeconds) ? Math.max(30, Math.min(Math.trunc(Number(ttlSeconds)), MAX_LOCK_TTL_SECONDS)) : DEFAULT_LOCK_TTL_SECONDS;
|
|
10445
|
+
return `+${ttl} seconds`;
|
|
10446
|
+
}
|
|
10447
|
+
function safeMetadata(metadata) {
|
|
10448
|
+
if (!metadata) return null;
|
|
10449
|
+
try {
|
|
10450
|
+
return JSON.stringify(metadata).slice(0, 4e3);
|
|
10451
|
+
} catch {
|
|
10452
|
+
return null;
|
|
10453
|
+
}
|
|
10454
|
+
}
|
|
10455
|
+
async function ensureConcurrencyGateSchema() {
|
|
10456
|
+
if (!schemaReady) {
|
|
10457
|
+
schemaReady = (async () => {
|
|
10458
|
+
const db = getDb();
|
|
10459
|
+
await db.execute(`
|
|
10460
|
+
CREATE TABLE IF NOT EXISTS concurrency_locks (
|
|
10461
|
+
id TEXT PRIMARY KEY,
|
|
10462
|
+
user_id INTEGER NOT NULL REFERENCES users(id),
|
|
10463
|
+
operation TEXT NOT NULL,
|
|
10464
|
+
status TEXT NOT NULL DEFAULT 'active',
|
|
10465
|
+
acquired_at TEXT NOT NULL DEFAULT (datetime('now')),
|
|
10466
|
+
expires_at TEXT NOT NULL,
|
|
10467
|
+
released_at TEXT,
|
|
10468
|
+
metadata TEXT
|
|
10469
|
+
)
|
|
10470
|
+
`);
|
|
10471
|
+
await db.execute(`CREATE INDEX IF NOT EXISTS concurrency_locks_user_status ON concurrency_locks(user_id, status, expires_at)`);
|
|
10472
|
+
await db.execute(`CREATE INDEX IF NOT EXISTS concurrency_locks_operation ON concurrency_locks(operation, status, expires_at)`);
|
|
10473
|
+
})();
|
|
10474
|
+
}
|
|
10475
|
+
return schemaReady;
|
|
10476
|
+
}
|
|
10477
|
+
async function expireConcurrencyLocksForUser(userId) {
|
|
10478
|
+
await ensureConcurrencyGateSchema();
|
|
10479
|
+
await getDb().execute({
|
|
10480
|
+
sql: `UPDATE concurrency_locks
|
|
10481
|
+
SET status = 'expired', released_at = datetime('now')
|
|
10482
|
+
WHERE user_id = ? AND status = 'active' AND expires_at <= datetime('now')`,
|
|
10483
|
+
args: [userId]
|
|
10484
|
+
});
|
|
10485
|
+
}
|
|
10486
|
+
async function countActiveUsageForUser(userId) {
|
|
10487
|
+
await ensureConcurrencyGateSchema();
|
|
10488
|
+
await expireConcurrencyLocksForUser(userId);
|
|
10489
|
+
const res = await getDb().execute({
|
|
10490
|
+
sql: `SELECT
|
|
10491
|
+
(
|
|
10492
|
+
SELECT COUNT(*)
|
|
10493
|
+
FROM jobs
|
|
10494
|
+
WHERE user_id = ?
|
|
10495
|
+
AND (status = 'pending' OR (status = 'running' AND started_at > datetime('now', '-10 minutes')))
|
|
10496
|
+
) +
|
|
10497
|
+
(
|
|
10498
|
+
SELECT COUNT(*)
|
|
10499
|
+
FROM concurrency_locks
|
|
10500
|
+
WHERE user_id = ? AND status = 'active' AND expires_at > datetime('now')
|
|
10501
|
+
) AS n`,
|
|
10502
|
+
args: [userId, userId]
|
|
10503
|
+
});
|
|
10504
|
+
return Number(res.rows[0]?.n ?? 0);
|
|
10505
|
+
}
|
|
10506
|
+
async function reuseExistingConcurrencyGate(userId, lockId, ttlSeconds) {
|
|
10507
|
+
if (!/^cl_[a-f0-9]{24,32}$/.test(lockId)) return false;
|
|
10508
|
+
await ensureConcurrencyGateSchema();
|
|
10509
|
+
await expireConcurrencyLocksForUser(userId);
|
|
10510
|
+
const res = await getDb().execute({
|
|
10511
|
+
sql: `UPDATE concurrency_locks
|
|
10512
|
+
SET expires_at = datetime('now', ?)
|
|
10513
|
+
WHERE id = ?
|
|
10514
|
+
AND user_id = ?
|
|
10515
|
+
AND status = 'active'
|
|
10516
|
+
AND expires_at > datetime('now')`,
|
|
10517
|
+
args: [lockTtlModifier(ttlSeconds), lockId, userId]
|
|
10518
|
+
});
|
|
10519
|
+
return Number(res.rowsAffected ?? 0) > 0;
|
|
10520
|
+
}
|
|
10521
|
+
async function acquireConcurrencyGate(user, operation, options = {}) {
|
|
10522
|
+
await ensureConcurrencyGateSchema();
|
|
10523
|
+
const limit = concurrencyLimitForUser(user);
|
|
10524
|
+
if (isConcurrencyLimitBypassed(user)) {
|
|
10525
|
+
return { ok: true, lockId: null, active: 0, limit, operation, bypassed: true };
|
|
10526
|
+
}
|
|
10527
|
+
if (options.reuseLockId && await reuseExistingConcurrencyGate(user.id, options.reuseLockId, options.ttlSeconds)) {
|
|
10528
|
+
return { ok: true, lockId: null, active: await countActiveUsageForUser(user.id), limit, operation, reused: true };
|
|
10529
|
+
}
|
|
10530
|
+
await expireConcurrencyLocksForUser(user.id);
|
|
10531
|
+
const lockId = `cl_${(0, import_node_crypto2.randomUUID)().replace(/-/g, "").slice(0, 24)}`;
|
|
10532
|
+
const res = await getDb().execute({
|
|
10533
|
+
sql: `INSERT INTO concurrency_locks (id, user_id, operation, status, expires_at, metadata)
|
|
10534
|
+
SELECT ?, ?, ?, 'active', datetime('now', ?), ?
|
|
10535
|
+
WHERE (
|
|
10536
|
+
(
|
|
10537
|
+
SELECT COUNT(*)
|
|
10538
|
+
FROM jobs
|
|
10539
|
+
WHERE user_id = ?
|
|
10540
|
+
AND (status = 'pending' OR (status = 'running' AND started_at > datetime('now', '-10 minutes')))
|
|
10541
|
+
) +
|
|
10542
|
+
(
|
|
10543
|
+
SELECT COUNT(*)
|
|
10544
|
+
FROM concurrency_locks
|
|
10545
|
+
WHERE user_id = ? AND status = 'active' AND expires_at > datetime('now')
|
|
10546
|
+
)
|
|
10547
|
+
) < ?`,
|
|
10548
|
+
args: [
|
|
10549
|
+
lockId,
|
|
10550
|
+
user.id,
|
|
10551
|
+
operation,
|
|
10552
|
+
lockTtlModifier(options.ttlSeconds),
|
|
10553
|
+
safeMetadata(options.metadata),
|
|
10554
|
+
user.id,
|
|
10555
|
+
user.id,
|
|
10556
|
+
limit
|
|
10557
|
+
]
|
|
10558
|
+
});
|
|
10559
|
+
const active = await countActiveUsageForUser(user.id);
|
|
10560
|
+
if (Number(res.rowsAffected ?? 0) > 0) {
|
|
10561
|
+
return { ok: true, lockId, active, limit, operation };
|
|
10562
|
+
}
|
|
10563
|
+
return { ok: false, active, limit, operation, retryAfterSeconds: DEFAULT_RETRY_AFTER_SECONDS };
|
|
10564
|
+
}
|
|
10565
|
+
async function releaseConcurrencyGate(lockId) {
|
|
10566
|
+
if (!lockId) return;
|
|
10567
|
+
await ensureConcurrencyGateSchema();
|
|
10568
|
+
await getDb().execute({
|
|
10569
|
+
sql: `UPDATE concurrency_locks
|
|
10570
|
+
SET status = 'released', released_at = datetime('now')
|
|
10571
|
+
WHERE id = ? AND status = 'active'`,
|
|
10572
|
+
args: [lockId]
|
|
10573
|
+
});
|
|
10574
|
+
}
|
|
10575
|
+
async function extendConcurrencyGate(lockId, ttlSeconds = DEFAULT_LOCK_TTL_SECONDS) {
|
|
10576
|
+
if (!lockId) return;
|
|
10577
|
+
await ensureConcurrencyGateSchema();
|
|
10578
|
+
await getDb().execute({
|
|
10579
|
+
sql: `UPDATE concurrency_locks
|
|
10580
|
+
SET expires_at = datetime('now', ?)
|
|
10581
|
+
WHERE id = ? AND status = 'active' AND expires_at > datetime('now')`,
|
|
10582
|
+
args: [lockTtlModifier(ttlSeconds), lockId]
|
|
10583
|
+
});
|
|
10584
|
+
}
|
|
10585
|
+
var import_node_crypto2, DEFAULT_LOCK_TTL_SECONDS, DEFAULT_RETRY_AFTER_SECONDS, MAX_LOCK_TTL_SECONDS, schemaReady;
|
|
10586
|
+
var init_concurrency_gates = __esm({
|
|
10587
|
+
"src/api/concurrency-gates.ts"() {
|
|
10588
|
+
"use strict";
|
|
10589
|
+
import_node_crypto2 = require("crypto");
|
|
10590
|
+
init_db();
|
|
10591
|
+
init_rates();
|
|
10592
|
+
DEFAULT_LOCK_TTL_SECONDS = 15 * 60;
|
|
10593
|
+
DEFAULT_RETRY_AFTER_SECONDS = 30;
|
|
10594
|
+
MAX_LOCK_TTL_SECONDS = 24 * 60 * 60;
|
|
10595
|
+
schemaReady = null;
|
|
10596
|
+
}
|
|
10597
|
+
});
|
|
10598
|
+
|
|
10386
10599
|
// src/api/youtube-routes.ts
|
|
10387
10600
|
function buildTranscriptMarkdown(result) {
|
|
10388
10601
|
const lines = [];
|
|
@@ -10444,6 +10657,7 @@ var init_youtube_routes = __esm({
|
|
|
10444
10657
|
init_errors();
|
|
10445
10658
|
init_api_auth();
|
|
10446
10659
|
init_server_schemas();
|
|
10660
|
+
init_concurrency_gates();
|
|
10447
10661
|
youtubeApp = new import_hono2.Hono();
|
|
10448
10662
|
youtubeApp.post("/harvest", createApiKeyAuth(), async (c) => {
|
|
10449
10663
|
const raw = await c.req.json().catch(() => ({}));
|
|
@@ -10457,9 +10671,16 @@ var init_youtube_routes = __esm({
|
|
|
10457
10671
|
return c.json({ error: "channel mode requires channelHandle (@handle or UC... ID)" }, 400);
|
|
10458
10672
|
}
|
|
10459
10673
|
const user = c.get("user");
|
|
10460
|
-
const
|
|
10461
|
-
|
|
10674
|
+
const gate = await acquireConcurrencyGate(user, "youtube_harvest", {
|
|
10675
|
+
reuseLockId: c.req.header("x-mcp-scraper-concurrency-lock"),
|
|
10676
|
+
metadata: { mode, query, channelHandle }
|
|
10677
|
+
});
|
|
10678
|
+
if (!gate.ok) return c.json(concurrencyLimitExceededResponse(gate), 429, { "Retry-After": String(gate.retryAfterSeconds) });
|
|
10679
|
+
let debited = false;
|
|
10462
10680
|
try {
|
|
10681
|
+
const { ok: ytOk, balance_mc: ytBal } = await debitMc(user.id, MC_COSTS.yt_channel, LedgerOperation.YT_CHANNEL, query ?? channelHandle ?? "");
|
|
10682
|
+
if (!ytOk) return c.json(insufficientBalanceResponse(ytBal, MC_COSTS.yt_channel), 402);
|
|
10683
|
+
debited = true;
|
|
10463
10684
|
const normalizedChannelUrl = mode === "channel" ? buildYouTubeChannelVideosUrl(channelHandle.trim()) : void 0;
|
|
10464
10685
|
const result = await ytHarvest({
|
|
10465
10686
|
mode,
|
|
@@ -10484,13 +10705,15 @@ var init_youtube_routes = __esm({
|
|
|
10484
10705
|
return c.json({ error: msg }, 400);
|
|
10485
10706
|
}
|
|
10486
10707
|
if (err instanceof CaptchaError || msg.includes("CAPTCHA") || msg.includes("blocked")) {
|
|
10487
|
-
await creditMc(user.id, MC_COSTS.yt_channel, LedgerOperation.YT_CHANNEL_REFUND, "failed call");
|
|
10708
|
+
if (debited) await creditMc(user.id, MC_COSTS.yt_channel, LedgerOperation.YT_CHANNEL_REFUND, "failed call");
|
|
10488
10709
|
await logRequestEvent({ userId: user.id, source: "youtube_harvest", status: "failed", query: mode === "search" ? query?.trim() ?? "" : channelHandle?.trim() ?? "", error: msg });
|
|
10489
10710
|
return c.json({ error: msg }, 503);
|
|
10490
10711
|
}
|
|
10491
|
-
await creditMc(user.id, MC_COSTS.yt_channel, LedgerOperation.YT_CHANNEL_REFUND, "failed call");
|
|
10712
|
+
if (debited) await creditMc(user.id, MC_COSTS.yt_channel, LedgerOperation.YT_CHANNEL_REFUND, "failed call");
|
|
10492
10713
|
await logRequestEvent({ userId: user.id, source: "youtube_harvest", status: "failed", query: mode === "search" ? query?.trim() ?? "" : channelHandle?.trim() ?? "", error: msg });
|
|
10493
10714
|
return c.json({ error: msg }, 500);
|
|
10715
|
+
} finally {
|
|
10716
|
+
await releaseConcurrencyGate(gate.lockId);
|
|
10494
10717
|
}
|
|
10495
10718
|
});
|
|
10496
10719
|
youtubeApp.post("/transcribe", createApiKeyAuth(), async (c) => {
|
|
@@ -10502,9 +10725,16 @@ var init_youtube_routes = __esm({
|
|
|
10502
10725
|
const user = c.get("user");
|
|
10503
10726
|
const holdMins = 5;
|
|
10504
10727
|
const holdMc = MC_COSTS.yt_transcription * holdMins;
|
|
10505
|
-
const
|
|
10506
|
-
|
|
10728
|
+
const gate = await acquireConcurrencyGate(user, "youtube_transcribe", {
|
|
10729
|
+
reuseLockId: c.req.header("x-mcp-scraper-concurrency-lock"),
|
|
10730
|
+
metadata: { videoId: id }
|
|
10731
|
+
});
|
|
10732
|
+
if (!gate.ok) return c.json(concurrencyLimitExceededResponse(gate), 429, { "Retry-After": String(gate.retryAfterSeconds) });
|
|
10733
|
+
let debited = false;
|
|
10507
10734
|
try {
|
|
10735
|
+
const { ok: trOk, balance_mc: trBal } = await debitMc(user.id, holdMc, LedgerOperation.TRANSCRIPTION_HOLD, id);
|
|
10736
|
+
if (!trOk) return c.json(insufficientBalanceResponse(trBal, holdMc), 402);
|
|
10737
|
+
debited = true;
|
|
10508
10738
|
const result = await fetchCaptions(id);
|
|
10509
10739
|
const lastChunk = result.chunks?.[result.chunks.length - 1];
|
|
10510
10740
|
const chunkSecs = lastChunk && Number.isFinite(lastChunk.timestamp[1]) ? lastChunk.timestamp[1] : 0;
|
|
@@ -10529,10 +10759,12 @@ var init_youtube_routes = __esm({
|
|
|
10529
10759
|
html: buildTranscriptHtml(result)
|
|
10530
10760
|
});
|
|
10531
10761
|
} catch (err) {
|
|
10532
|
-
await creditMc(user.id, holdMc, LedgerOperation.TRANSCRIPTION_REFUND, "failed call");
|
|
10762
|
+
if (debited) await creditMc(user.id, holdMc, LedgerOperation.TRANSCRIPTION_REFUND, "failed call");
|
|
10533
10763
|
const msg = err instanceof Error ? err.message : String(err);
|
|
10534
10764
|
await logRequestEvent({ userId: user.id, source: "youtube_transcribe", status: "failed", query: id, error: msg });
|
|
10535
10765
|
return c.json({ error: msg }, 500);
|
|
10766
|
+
} finally {
|
|
10767
|
+
await releaseConcurrencyGate(gate.lockId);
|
|
10536
10768
|
}
|
|
10537
10769
|
});
|
|
10538
10770
|
}
|
|
@@ -10549,6 +10781,7 @@ var init_screenshot_routes = __esm({
|
|
|
10549
10781
|
init_screenshot();
|
|
10550
10782
|
init_api_auth();
|
|
10551
10783
|
init_url_utils();
|
|
10784
|
+
init_concurrency_gates();
|
|
10552
10785
|
ScreenshotBodySchema = import_zod14.z.object({
|
|
10553
10786
|
url: import_zod14.z.string().trim().min(1, "url is required"),
|
|
10554
10787
|
device: import_zod14.z.string().trim().optional(),
|
|
@@ -10564,6 +10797,7 @@ var init_screenshot_routes = __esm({
|
|
|
10564
10797
|
}
|
|
10565
10798
|
const body = parsed.data;
|
|
10566
10799
|
const urlCheck = await validatePublicHttpUrl(body.url, { field: "url", requireHttps: false });
|
|
10800
|
+
let targetUrl = urlCheck.parsed?.href;
|
|
10567
10801
|
if (urlCheck.error) {
|
|
10568
10802
|
if (!body.allowLocal) {
|
|
10569
10803
|
return c.json({ error_code: "invalid_request", message: urlCheck.error }, 400);
|
|
@@ -10577,25 +10811,17 @@ var init_screenshot_routes = __esm({
|
|
|
10577
10811
|
if (parsedFallback.protocol !== "http:" && parsedFallback.protocol !== "https:") {
|
|
10578
10812
|
return c.json({ error: "url must use http or https" }, 400);
|
|
10579
10813
|
}
|
|
10580
|
-
|
|
10581
|
-
try {
|
|
10582
|
-
const buf = await captureScreenshot(parsedFallback.href, browserServiceApiKey(), device2);
|
|
10583
|
-
return new Response(new Uint8Array(buf), {
|
|
10584
|
-
status: 200,
|
|
10585
|
-
headers: {
|
|
10586
|
-
"Content-Type": "image/png",
|
|
10587
|
-
"Content-Length": String(buf.length),
|
|
10588
|
-
"Cache-Control": "no-store"
|
|
10589
|
-
}
|
|
10590
|
-
});
|
|
10591
|
-
} catch (err) {
|
|
10592
|
-
const msg = err instanceof Error ? err.message : String(err);
|
|
10593
|
-
return c.json({ error: msg }, 502);
|
|
10594
|
-
}
|
|
10814
|
+
targetUrl = parsedFallback.href;
|
|
10595
10815
|
}
|
|
10596
10816
|
const device = body.device === "mobile" ? "mobile" : "desktop";
|
|
10817
|
+
const user = c.get("user");
|
|
10818
|
+
const gate = await acquireConcurrencyGate(user, "screenshot", {
|
|
10819
|
+
reuseLockId: c.req.header("x-mcp-scraper-concurrency-lock"),
|
|
10820
|
+
metadata: { url: targetUrl, device }
|
|
10821
|
+
});
|
|
10822
|
+
if (!gate.ok) return c.json(concurrencyLimitExceededResponse(gate), 429, { "Retry-After": String(gate.retryAfterSeconds) });
|
|
10597
10823
|
try {
|
|
10598
|
-
const buf = await captureScreenshot(
|
|
10824
|
+
const buf = await captureScreenshot(targetUrl, browserServiceApiKey(), device);
|
|
10599
10825
|
return new Response(new Uint8Array(buf), {
|
|
10600
10826
|
status: 200,
|
|
10601
10827
|
headers: {
|
|
@@ -10607,6 +10833,8 @@ var init_screenshot_routes = __esm({
|
|
|
10607
10833
|
} catch (err) {
|
|
10608
10834
|
const msg = err instanceof Error ? err.message : String(err);
|
|
10609
10835
|
return c.json({ error: msg }, 502);
|
|
10836
|
+
} finally {
|
|
10837
|
+
await releaseConcurrencyGate(gate.lockId);
|
|
10610
10838
|
}
|
|
10611
10839
|
});
|
|
10612
10840
|
}
|
|
@@ -11390,6 +11618,132 @@ var init_FacebookAdGraphql = __esm({
|
|
|
11390
11618
|
}
|
|
11391
11619
|
});
|
|
11392
11620
|
|
|
11621
|
+
// src/extractor/FacebookOrganicVideoExtractor.ts
|
|
11622
|
+
function htmlDecode(value) {
|
|
11623
|
+
return value.replace(/&/g, "&").replace(/"/g, '"').replace(/'/g, "'").replace(/</g, "<").replace(/>/g, ">");
|
|
11624
|
+
}
|
|
11625
|
+
function decodeEscapedValue(raw) {
|
|
11626
|
+
let out = raw;
|
|
11627
|
+
try {
|
|
11628
|
+
out = JSON.parse(`"${raw.replace(/"/g, '\\"')}"`);
|
|
11629
|
+
} catch {
|
|
11630
|
+
out = raw.replace(/\\u0025/g, "%").replace(/\\u0026/g, "&").replace(/\\u003D/g, "=").replace(/\\u003d/g, "=").replace(/\\"/g, '"').replace(/\\\//g, "/");
|
|
11631
|
+
}
|
|
11632
|
+
return htmlDecode(out);
|
|
11633
|
+
}
|
|
11634
|
+
function facebookVideoIdFromUrl(url) {
|
|
11635
|
+
try {
|
|
11636
|
+
const parsed = new URL(url);
|
|
11637
|
+
const path6 = parsed.pathname;
|
|
11638
|
+
const direct = path6.match(/\/(?:reel|videos|watch)\/(?:[^/]+\/)?(\d{8,})/i);
|
|
11639
|
+
if (direct) return direct[1];
|
|
11640
|
+
const queryVideoId = parsed.searchParams.get("v") ?? parsed.searchParams.get("video_id");
|
|
11641
|
+
if (queryVideoId && /^\d{8,}$/.test(queryVideoId)) return queryVideoId;
|
|
11642
|
+
const numericTail = path6.match(/\/(\d{8,})\/?$/);
|
|
11643
|
+
if (numericTail) return numericTail[1];
|
|
11644
|
+
} catch {
|
|
11645
|
+
}
|
|
11646
|
+
return null;
|
|
11647
|
+
}
|
|
11648
|
+
function parseEfg(url) {
|
|
11649
|
+
try {
|
|
11650
|
+
const efg = new URL(url).searchParams.get("efg");
|
|
11651
|
+
if (!efg) return null;
|
|
11652
|
+
return JSON.parse(Buffer.from(efg, "base64").toString("utf8"));
|
|
11653
|
+
} catch {
|
|
11654
|
+
return null;
|
|
11655
|
+
}
|
|
11656
|
+
}
|
|
11657
|
+
function manifestDurationSec(htmlWindow) {
|
|
11658
|
+
const decoded = decodeEscapedValue(htmlWindow);
|
|
11659
|
+
const match = decoded.match(/mediaPresentationDuration="PT([\d.]+)S"/);
|
|
11660
|
+
return match ? Number(match[1]) : null;
|
|
11661
|
+
}
|
|
11662
|
+
function metadataContent(html, key) {
|
|
11663
|
+
const escaped = key.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
|
11664
|
+
const re = new RegExp(`<meta\\s+[^>]*(?:property|name)=["']${escaped}["'][^>]*content=["']([^"']+)["'][^>]*>`, "i");
|
|
11665
|
+
const m = html.match(re);
|
|
11666
|
+
return m ? htmlDecode(m[1]).trim() : null;
|
|
11667
|
+
}
|
|
11668
|
+
function ownerNameFromHtml(html) {
|
|
11669
|
+
const ogTitle = metadataContent(html, "og:title");
|
|
11670
|
+
const pipe = ogTitle?.split("|").map((s) => s.trim()).filter(Boolean);
|
|
11671
|
+
if (pipe && pipe.length > 1) return pipe[pipe.length - 1] ?? null;
|
|
11672
|
+
return null;
|
|
11673
|
+
}
|
|
11674
|
+
function extractFacebookOrganicVideo(html, pageUrl, sourceUrl = pageUrl, quality = "best") {
|
|
11675
|
+
const videoId = facebookVideoIdFromUrl(pageUrl) ?? facebookVideoIdFromUrl(sourceUrl);
|
|
11676
|
+
const manifestIndex = videoId ? html.indexOf(`dash_mpd_debug.mpd?v=${videoId}`) : -1;
|
|
11677
|
+
const searchWindow = manifestIndex >= 0 ? html.slice(Math.max(0, manifestIndex - 5e4), Math.min(html.length, manifestIndex + 5e4)) : html;
|
|
11678
|
+
const targetDuration = manifestDurationSec(searchWindow);
|
|
11679
|
+
const candidates = [];
|
|
11680
|
+
const seen = /* @__PURE__ */ new Set();
|
|
11681
|
+
const re = /browser_native_(sd|hd)_url\\?"\s*:\s*\\?"([^"<]+?)\\?"/g;
|
|
11682
|
+
let match;
|
|
11683
|
+
while ((match = re.exec(searchWindow)) !== null) {
|
|
11684
|
+
const url = decodeEscapedValue(match[2]);
|
|
11685
|
+
if (!url || seen.has(url)) continue;
|
|
11686
|
+
seen.add(url);
|
|
11687
|
+
const parsed = parseEfg(url);
|
|
11688
|
+
let bitrate = null;
|
|
11689
|
+
try {
|
|
11690
|
+
const value = new URL(url).searchParams.get("bitrate");
|
|
11691
|
+
bitrate = value ? Number(value) : null;
|
|
11692
|
+
} catch {
|
|
11693
|
+
}
|
|
11694
|
+
candidates.push({
|
|
11695
|
+
url,
|
|
11696
|
+
quality: match[1],
|
|
11697
|
+
bitrate: Number.isFinite(bitrate) ? bitrate : null,
|
|
11698
|
+
durationSec: typeof parsed?.duration_s === "number" ? parsed.duration_s : null,
|
|
11699
|
+
assetId: typeof parsed?.xpv_asset_id === "number" ? parsed.xpv_asset_id : null,
|
|
11700
|
+
vencodeTag: typeof parsed?.vencode_tag === "string" ? parsed.vencode_tag : null
|
|
11701
|
+
});
|
|
11702
|
+
}
|
|
11703
|
+
const targetCandidates = candidates.filter((candidate) => {
|
|
11704
|
+
if (!targetDuration || candidate.durationSec == null) return true;
|
|
11705
|
+
return Math.abs(candidate.durationSec - targetDuration) <= 2;
|
|
11706
|
+
});
|
|
11707
|
+
const selectable = targetCandidates.length ? targetCandidates : candidates;
|
|
11708
|
+
const selected = [...selectable].filter((candidate) => quality === "best" || candidate.quality === quality).sort((a, b) => {
|
|
11709
|
+
const aq = a.quality === "hd" ? 1 : 0;
|
|
11710
|
+
const bq = b.quality === "hd" ? 1 : 0;
|
|
11711
|
+
if (quality === "best" && aq !== bq) return bq - aq;
|
|
11712
|
+
return (b.bitrate ?? 0) - (a.bitrate ?? 0);
|
|
11713
|
+
})[0];
|
|
11714
|
+
if (!selected) {
|
|
11715
|
+
throw new Error("No progressive Facebook video URL was found in the rendered page state");
|
|
11716
|
+
}
|
|
11717
|
+
return {
|
|
11718
|
+
sourceUrl,
|
|
11719
|
+
pageUrl,
|
|
11720
|
+
videoId,
|
|
11721
|
+
title: metadataContent(html, "og:title") ?? metadataContent(html, "twitter:title"),
|
|
11722
|
+
description: metadataContent(html, "og:description") ?? metadataContent(html, "description") ?? metadataContent(html, "twitter:description"),
|
|
11723
|
+
ownerName: ownerNameFromHtml(html),
|
|
11724
|
+
videoUrl: selected.url,
|
|
11725
|
+
selectedQuality: selected.quality,
|
|
11726
|
+
bitrate: selected.bitrate,
|
|
11727
|
+
durationSec: targetDuration ?? selected.durationSec,
|
|
11728
|
+
assetId: selected.assetId,
|
|
11729
|
+
vencodeTag: selected.vencodeTag,
|
|
11730
|
+
candidates: selectable,
|
|
11731
|
+
extractedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
11732
|
+
};
|
|
11733
|
+
}
|
|
11734
|
+
async function extractFacebookOrganicVideoFromPage(page, sourceUrl, quality = "best") {
|
|
11735
|
+
await page.waitForLoadState("domcontentloaded", { timeout: 15e3 }).catch(() => {
|
|
11736
|
+
});
|
|
11737
|
+
await page.waitForTimeout(1500);
|
|
11738
|
+
const html = await page.content();
|
|
11739
|
+
return extractFacebookOrganicVideo(html, page.url(), sourceUrl, quality);
|
|
11740
|
+
}
|
|
11741
|
+
var init_FacebookOrganicVideoExtractor = __esm({
|
|
11742
|
+
"src/extractor/FacebookOrganicVideoExtractor.ts"() {
|
|
11743
|
+
"use strict";
|
|
11744
|
+
}
|
|
11745
|
+
});
|
|
11746
|
+
|
|
11393
11747
|
// src/locations.ts
|
|
11394
11748
|
var LOCATIONS;
|
|
11395
11749
|
var init_locations = __esm({
|
|
@@ -11933,6 +12287,39 @@ var init_kernel_proxy_resolver = __esm({
|
|
|
11933
12287
|
function invalidRequest(message) {
|
|
11934
12288
|
return { error_code: "invalid_request", message };
|
|
11935
12289
|
}
|
|
12290
|
+
function isAllowedFacebookOrganicUrl(url) {
|
|
12291
|
+
const hostname = url.hostname.toLowerCase().replace(/^www\./, "").replace(/^m\./, "");
|
|
12292
|
+
return hostname === "facebook.com" || hostname === "fb.watch";
|
|
12293
|
+
}
|
|
12294
|
+
function facebookTranscriptMarkdown(title, text, chunks, durationMs) {
|
|
12295
|
+
const fmtTs2 = (s) => `${Math.floor(s / 60)}:${String(Math.floor(s % 60)).padStart(2, "0")}`;
|
|
12296
|
+
const lines = [title, "", `*Transcribed in ${(durationMs / 1e3).toFixed(1)}s*`, "", "## Full Text", "", text, ""];
|
|
12297
|
+
if (chunks.length) {
|
|
12298
|
+
lines.push("## Timestamped Segments", "");
|
|
12299
|
+
for (const ch of chunks) {
|
|
12300
|
+
lines.push(`**[${fmtTs2(ch.timestamp[0])} -> ${fmtTs2(ch.timestamp[1])}]** ${ch.text.trim()}`, "");
|
|
12301
|
+
}
|
|
12302
|
+
}
|
|
12303
|
+
return lines.join("\n");
|
|
12304
|
+
}
|
|
12305
|
+
async function transcribeFacebookVideoUrl(videoUrl, markdownTitle = "# Facebook Video Transcript") {
|
|
12306
|
+
const startMs = Date.now();
|
|
12307
|
+
const result = await import_client3.fal.subscribe("fal-ai/wizper", {
|
|
12308
|
+
input: { audio_url: videoUrl, task: "transcribe", language: "en" },
|
|
12309
|
+
logs: false,
|
|
12310
|
+
pollInterval: 3e3
|
|
12311
|
+
});
|
|
12312
|
+
const data = result.data;
|
|
12313
|
+
const text = data.text ?? "";
|
|
12314
|
+
const chunks = data.chunks ?? [];
|
|
12315
|
+
const durationMs = Date.now() - startMs;
|
|
12316
|
+
return {
|
|
12317
|
+
text,
|
|
12318
|
+
chunks,
|
|
12319
|
+
durationMs,
|
|
12320
|
+
markdown: facebookTranscriptMarkdown(markdownTitle, text, chunks, durationMs)
|
|
12321
|
+
};
|
|
12322
|
+
}
|
|
11936
12323
|
async function detectSoftBlock(driver) {
|
|
11937
12324
|
const page = driver.getPage();
|
|
11938
12325
|
const bodyText = await page.evaluate(() => document.body?.innerText ?? "").catch(() => "");
|
|
@@ -11962,7 +12349,7 @@ async function kernelLaunchOptsResidential() {
|
|
|
11962
12349
|
}
|
|
11963
12350
|
return { headless: true, kernelApiKey: browserServiceApiKey(), kernelProxyId: proxyId, viewport: { width: 1280, height: 900 }, locale: "en-US" };
|
|
11964
12351
|
}
|
|
11965
|
-
var import_hono4, import_zod15, import_client3, FacebookAdBodySchema, FacebookPageIntelBodySchema, FacebookTranscribeBodySchema, FacebookSearchBodySchema, FacebookMediaBodySchema, facebookAdApp, ALLOWED_MEDIA_HOSTS;
|
|
12352
|
+
var import_hono4, import_zod15, import_client3, FacebookAdBodySchema, FacebookPageIntelBodySchema, FacebookTranscribeBodySchema, FacebookVideoTranscribeBodySchema, FacebookSearchBodySchema, FacebookMediaBodySchema, facebookAdApp, ALLOWED_MEDIA_HOSTS;
|
|
11966
12353
|
var init_facebook_ad_routes = __esm({
|
|
11967
12354
|
"src/api/facebook-ad-routes.ts"() {
|
|
11968
12355
|
"use strict";
|
|
@@ -11974,10 +12361,12 @@ var init_facebook_ad_routes = __esm({
|
|
|
11974
12361
|
init_BrowserDriver();
|
|
11975
12362
|
init_FacebookAdExtractor();
|
|
11976
12363
|
init_FacebookAdGraphql();
|
|
12364
|
+
init_FacebookOrganicVideoExtractor();
|
|
11977
12365
|
init_kernel_proxy_resolver();
|
|
11978
12366
|
import_client3 = require("@fal-ai/client");
|
|
11979
12367
|
init_api_auth();
|
|
11980
12368
|
init_url_utils();
|
|
12369
|
+
init_concurrency_gates();
|
|
11981
12370
|
FacebookAdBodySchema = import_zod15.z.object({
|
|
11982
12371
|
url: import_zod15.z.string().trim().optional(),
|
|
11983
12372
|
libraryId: import_zod15.z.string().trim().optional(),
|
|
@@ -11995,6 +12384,10 @@ var init_facebook_ad_routes = __esm({
|
|
|
11995
12384
|
FacebookTranscribeBodySchema = import_zod15.z.object({
|
|
11996
12385
|
videoUrl: import_zod15.z.string().trim().min(1, "videoUrl is required")
|
|
11997
12386
|
});
|
|
12387
|
+
FacebookVideoTranscribeBodySchema = import_zod15.z.object({
|
|
12388
|
+
url: import_zod15.z.string().trim().min(1, "url is required"),
|
|
12389
|
+
quality: import_zod15.z.enum(["best", "hd", "sd"]).default("best")
|
|
12390
|
+
});
|
|
11998
12391
|
FacebookSearchBodySchema = import_zod15.z.object({
|
|
11999
12392
|
query: import_zod15.z.string().trim().min(1, "query is required"),
|
|
12000
12393
|
country: import_zod15.z.string().trim().toUpperCase().optional(),
|
|
@@ -12016,10 +12409,17 @@ var init_facebook_ad_routes = __esm({
|
|
|
12016
12409
|
const libraryId = FacebookAdExtractor.resolveLibraryId(raw2);
|
|
12017
12410
|
if (!libraryId) return c.json({ error: "Could not resolve a valid Facebook Ad Library ID from the provided input" }, 400);
|
|
12018
12411
|
const fbUser = c.get("user");
|
|
12019
|
-
const
|
|
12020
|
-
|
|
12412
|
+
const gate = await acquireConcurrencyGate(fbUser, "facebook_ad", {
|
|
12413
|
+
reuseLockId: c.req.header("x-mcp-scraper-concurrency-lock"),
|
|
12414
|
+
metadata: { libraryId }
|
|
12415
|
+
});
|
|
12416
|
+
if (!gate.ok) return c.json(concurrencyLimitExceededResponse(gate), 429, { "Retry-After": String(gate.retryAfterSeconds) });
|
|
12021
12417
|
const driver = new BrowserDriver();
|
|
12418
|
+
let debited = false;
|
|
12022
12419
|
try {
|
|
12420
|
+
const { ok: adOk, balance_mc: adBal } = await debitMc(fbUser.id, MC_COSTS.fb_ad, LedgerOperation.FB_AD, raw2);
|
|
12421
|
+
if (!adOk) return c.json(insufficientBalanceResponse(adBal, MC_COSTS.fb_ad), 402);
|
|
12422
|
+
debited = true;
|
|
12023
12423
|
await driver.launch(kernelLaunchOpts());
|
|
12024
12424
|
const extractor = new FacebookAdExtractor(driver);
|
|
12025
12425
|
const result = await extractor.extract(libraryId, { openModal: body.openModal !== false });
|
|
@@ -12033,7 +12433,7 @@ var init_facebook_ad_routes = __esm({
|
|
|
12033
12433
|
});
|
|
12034
12434
|
return c.json(result);
|
|
12035
12435
|
} catch (err) {
|
|
12036
|
-
await creditMc(fbUser.id, MC_COSTS.fb_ad, LedgerOperation.FB_AD_REFUND, "failed call");
|
|
12436
|
+
if (debited) await creditMc(fbUser.id, MC_COSTS.fb_ad, LedgerOperation.FB_AD_REFUND, "failed call");
|
|
12037
12437
|
const msg = err instanceof Error ? err.message : String(err);
|
|
12038
12438
|
await logRequestEvent({ userId: fbUser.id, source: "facebook_ad", status: "failed", query: raw2, error: msg });
|
|
12039
12439
|
if (msg.toLowerCase().includes("blocked") || msg.toLowerCase().includes("captcha")) {
|
|
@@ -12042,24 +12442,26 @@ var init_facebook_ad_routes = __esm({
|
|
|
12042
12442
|
return c.json({ error: msg }, 500);
|
|
12043
12443
|
} finally {
|
|
12044
12444
|
await driver.close();
|
|
12445
|
+
await releaseConcurrencyGate(gate.lockId);
|
|
12045
12446
|
}
|
|
12046
12447
|
});
|
|
12047
12448
|
facebookAdApp.post("/page-intel", createApiKeyAuth(), async (c) => {
|
|
12048
12449
|
const raw = await c.req.json().catch(() => ({}));
|
|
12049
12450
|
const parsed = FacebookPageIntelBodySchema.safeParse(raw);
|
|
12050
|
-
if (!parsed.success)
|
|
12051
|
-
return c.json(invalidRequest(parsed.error.issues[0]?.message ?? "Invalid request"), 400);
|
|
12052
|
-
}
|
|
12451
|
+
if (!parsed.success) return c.json(invalidRequest(parsed.error.issues[0]?.message ?? "Invalid request"), 400);
|
|
12053
12452
|
const body = parsed.data;
|
|
12054
12453
|
const maxAds = Math.min(200, Math.max(1, body.maxAds ?? 50));
|
|
12055
12454
|
const country = body.country?.trim().toUpperCase() ?? "US";
|
|
12056
12455
|
const listingUrl = buildPageIntelUrl(body, country);
|
|
12057
12456
|
const fbUser = c.get("user");
|
|
12058
|
-
const
|
|
12059
|
-
if (!
|
|
12457
|
+
const gate = await acquireConcurrencyGate(fbUser, "facebook_page_intel", { reuseLockId: c.req.header("x-mcp-scraper-concurrency-lock"), metadata: { pageId: body.pageId, query: body.query, libraryId: body.libraryId, country } });
|
|
12458
|
+
if (!gate.ok) return c.json(concurrencyLimitExceededResponse(gate), 429, { "Retry-After": String(gate.retryAfterSeconds) });
|
|
12060
12459
|
const driver = new BrowserDriver();
|
|
12061
|
-
let refunded = false;
|
|
12460
|
+
let refunded = false, debited = false;
|
|
12062
12461
|
try {
|
|
12462
|
+
const { ok: fbOk, balance_mc: fbBal } = await debitMc(fbUser.id, MC_COSTS.fb_ad, LedgerOperation.FB_AD, body.pageId ?? body.query ?? body.libraryId ?? "");
|
|
12463
|
+
if (!fbOk) return c.json(insufficientBalanceResponse(fbBal, MC_COSTS.fb_ad), 402);
|
|
12464
|
+
debited = true;
|
|
12063
12465
|
await driver.launch(await kernelLaunchOptsResidential());
|
|
12064
12466
|
await driver.navigateTo(listingUrl);
|
|
12065
12467
|
const extractor = new FacebookAdExtractor(driver);
|
|
@@ -12074,7 +12476,7 @@ var init_facebook_ad_routes = __esm({
|
|
|
12074
12476
|
return c.json(result);
|
|
12075
12477
|
} catch (err) {
|
|
12076
12478
|
const msg = err instanceof Error ? err.message : String(err);
|
|
12077
|
-
if (!refunded) await creditMc(fbUser.id, MC_COSTS.fb_ad, LedgerOperation.FB_AD_REFUND, "failed call");
|
|
12479
|
+
if (debited && !refunded) await creditMc(fbUser.id, MC_COSTS.fb_ad, LedgerOperation.FB_AD_REFUND, "failed call");
|
|
12078
12480
|
await logRequestEvent({ userId: fbUser.id, source: "facebook_page_intel", status: "failed", query: body.pageId ?? body.query ?? body.libraryId ?? "", error: msg });
|
|
12079
12481
|
if (msg.toLowerCase().includes("blocked") || msg.toLowerCase().includes("captcha")) {
|
|
12080
12482
|
return c.json({ error: msg }, 503);
|
|
@@ -12082,6 +12484,7 @@ var init_facebook_ad_routes = __esm({
|
|
|
12082
12484
|
return c.json({ error: msg }, 500);
|
|
12083
12485
|
} finally {
|
|
12084
12486
|
await driver.close();
|
|
12487
|
+
await releaseConcurrencyGate(gate.lockId);
|
|
12085
12488
|
}
|
|
12086
12489
|
});
|
|
12087
12490
|
facebookAdApp.post("/transcribe", createApiKeyAuth(), async (c) => {
|
|
@@ -12097,35 +12500,83 @@ var init_facebook_ad_routes = __esm({
|
|
|
12097
12500
|
}
|
|
12098
12501
|
const videoUrl = urlCheck.parsed.href;
|
|
12099
12502
|
const fbUser = c.get("user");
|
|
12100
|
-
const
|
|
12101
|
-
|
|
12503
|
+
const gate = await acquireConcurrencyGate(fbUser, "facebook_transcribe", {
|
|
12504
|
+
reuseLockId: c.req.header("x-mcp-scraper-concurrency-lock"),
|
|
12505
|
+
metadata: { videoUrl }
|
|
12506
|
+
});
|
|
12507
|
+
if (!gate.ok) return c.json(concurrencyLimitExceededResponse(gate), 429, { "Retry-After": String(gate.retryAfterSeconds) });
|
|
12102
12508
|
import_client3.fal.config({ credentials: process.env.FAL_KEY });
|
|
12509
|
+
let debited = false;
|
|
12103
12510
|
try {
|
|
12104
|
-
const
|
|
12105
|
-
|
|
12106
|
-
|
|
12107
|
-
|
|
12108
|
-
|
|
12109
|
-
|
|
12110
|
-
const data = result.data;
|
|
12111
|
-
const text = data.text ?? "";
|
|
12112
|
-
const chunks = data.chunks ?? [];
|
|
12113
|
-
const durationMs = Date.now() - startMs;
|
|
12114
|
-
const fmtTs2 = (s) => `${Math.floor(s / 60)}:${String(Math.floor(s % 60)).padStart(2, "0")}`;
|
|
12115
|
-
const lines = ["# Facebook Ad Transcript", "", `*Transcribed in ${(durationMs / 1e3).toFixed(1)}s*`, "", "## Full Text", "", text, ""];
|
|
12116
|
-
if (chunks.length) {
|
|
12117
|
-
lines.push("## Timestamped Segments", "");
|
|
12118
|
-
for (const ch of chunks) {
|
|
12119
|
-
lines.push(`**[${fmtTs2(ch.timestamp[0])} \u2192 ${fmtTs2(ch.timestamp[1])}]** ${ch.text.trim()}`, "");
|
|
12120
|
-
}
|
|
12121
|
-
}
|
|
12122
|
-
await logRequestEvent({ userId: fbUser.id, source: "facebook_transcribe", status: "done", query: videoUrl, resultCount: chunks.length, result: { text, chunks, durationMs } });
|
|
12123
|
-
return c.json({ text, chunks, durationMs, markdown: lines.join("\n") });
|
|
12511
|
+
const { ok, balance_mc } = await debitMc(fbUser.id, MC_COSTS.fb_transcribe, LedgerOperation.FB_TRANSCRIBE, videoUrl);
|
|
12512
|
+
if (!ok) return c.json(insufficientBalanceResponse(balance_mc, MC_COSTS.fb_transcribe), 402);
|
|
12513
|
+
debited = true;
|
|
12514
|
+
const transcript = await transcribeFacebookVideoUrl(videoUrl, "# Facebook Ad Transcript");
|
|
12515
|
+
await logRequestEvent({ userId: fbUser.id, source: "facebook_transcribe", status: "done", query: videoUrl, resultCount: transcript.chunks.length, result: { text: transcript.text, chunks: transcript.chunks, durationMs: transcript.durationMs } });
|
|
12516
|
+
return c.json(transcript);
|
|
12124
12517
|
} catch (err) {
|
|
12125
12518
|
const msg = err instanceof Error ? err.message : String(err);
|
|
12126
|
-
await creditMc(fbUser.id, MC_COSTS.fb_transcribe, LedgerOperation.FB_TRANSCRIBE_REFUND, "failed call");
|
|
12519
|
+
if (debited) await creditMc(fbUser.id, MC_COSTS.fb_transcribe, LedgerOperation.FB_TRANSCRIBE_REFUND, "failed call");
|
|
12127
12520
|
await logRequestEvent({ userId: fbUser.id, source: "facebook_transcribe", status: "failed", query: videoUrl, error: msg });
|
|
12128
12521
|
return c.json({ error: msg }, 500);
|
|
12522
|
+
} finally {
|
|
12523
|
+
await releaseConcurrencyGate(gate.lockId);
|
|
12524
|
+
}
|
|
12525
|
+
});
|
|
12526
|
+
facebookAdApp.post("/video-transcribe", createApiKeyAuth(), async (c) => {
|
|
12527
|
+
const raw = await c.req.json().catch(() => ({}));
|
|
12528
|
+
const parsed = FacebookVideoTranscribeBodySchema.safeParse(raw);
|
|
12529
|
+
if (!parsed.success) {
|
|
12530
|
+
return c.json(invalidRequest(parsed.error.issues[0]?.message ?? "Invalid request"), 400);
|
|
12531
|
+
}
|
|
12532
|
+
const body = parsed.data;
|
|
12533
|
+
const urlCheck = await validatePublicHttpUrl(body.url, { field: "url", requireHttps: false });
|
|
12534
|
+
if (urlCheck.error) {
|
|
12535
|
+
return c.json(invalidRequest(urlCheck.error), 400);
|
|
12536
|
+
}
|
|
12537
|
+
const sourceUrl = urlCheck.parsed;
|
|
12538
|
+
if (!isAllowedFacebookOrganicUrl(sourceUrl)) {
|
|
12539
|
+
return c.json(invalidRequest("url must be a Facebook organic video, reel, or share URL"), 400);
|
|
12540
|
+
}
|
|
12541
|
+
const fbUser = c.get("user");
|
|
12542
|
+
const gate = await acquireConcurrencyGate(fbUser, "facebook_video_transcribe", {
|
|
12543
|
+
reuseLockId: c.req.header("x-mcp-scraper-concurrency-lock"),
|
|
12544
|
+
metadata: { url: sourceUrl.href }
|
|
12545
|
+
});
|
|
12546
|
+
if (!gate.ok) return c.json(concurrencyLimitExceededResponse(gate), 429, { "Retry-After": String(gate.retryAfterSeconds) });
|
|
12547
|
+
import_client3.fal.config({ credentials: process.env.FAL_KEY });
|
|
12548
|
+
const driver = new BrowserDriver();
|
|
12549
|
+
let debited = false;
|
|
12550
|
+
try {
|
|
12551
|
+
const { ok, balance_mc } = await debitMc(fbUser.id, MC_COSTS.fb_transcribe, LedgerOperation.FB_TRANSCRIBE, sourceUrl.href);
|
|
12552
|
+
if (!ok) return c.json(insufficientBalanceResponse(balance_mc, MC_COSTS.fb_transcribe), 402);
|
|
12553
|
+
debited = true;
|
|
12554
|
+
await driver.launch(await kernelLaunchOptsResidential());
|
|
12555
|
+
await driver.navigateTo(sourceUrl.href);
|
|
12556
|
+
const page = driver.getPage();
|
|
12557
|
+
const video = await extractFacebookOrganicVideoFromPage(page, sourceUrl.href, body.quality);
|
|
12558
|
+
const transcript = await transcribeFacebookVideoUrl(video.videoUrl, "# Facebook Organic Video Transcript");
|
|
12559
|
+
const result = {
|
|
12560
|
+
...video,
|
|
12561
|
+
videoDurationSec: video.durationSec,
|
|
12562
|
+
text: transcript.text,
|
|
12563
|
+
chunks: transcript.chunks,
|
|
12564
|
+
durationMs: transcript.durationMs,
|
|
12565
|
+
markdown: transcript.markdown
|
|
12566
|
+
};
|
|
12567
|
+
await logRequestEvent({ userId: fbUser.id, source: "facebook_video_transcribe", status: "done", query: sourceUrl.href, resultCount: transcript.chunks.length, result });
|
|
12568
|
+
return c.json(result);
|
|
12569
|
+
} catch (err) {
|
|
12570
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
12571
|
+
if (debited) await creditMc(fbUser.id, MC_COSTS.fb_transcribe, LedgerOperation.FB_TRANSCRIBE_REFUND, "failed call");
|
|
12572
|
+
await logRequestEvent({ userId: fbUser.id, source: "facebook_video_transcribe", status: "failed", query: sourceUrl.href, error: msg });
|
|
12573
|
+
if (msg.toLowerCase().includes("blocked") || msg.toLowerCase().includes("captcha")) {
|
|
12574
|
+
return c.json({ error: msg }, 503);
|
|
12575
|
+
}
|
|
12576
|
+
return c.json({ error: msg }, 500);
|
|
12577
|
+
} finally {
|
|
12578
|
+
await driver.close();
|
|
12579
|
+
await releaseConcurrencyGate(gate.lockId);
|
|
12129
12580
|
}
|
|
12130
12581
|
});
|
|
12131
12582
|
facebookAdApp.post("/search", createApiKeyAuth(), async (c) => {
|
|
@@ -12139,11 +12590,18 @@ var init_facebook_ad_routes = __esm({
|
|
|
12139
12590
|
const maxResults = Math.min(20, Math.max(1, body.maxResults ?? 10));
|
|
12140
12591
|
const searchUrl = `https://www.facebook.com/ads/library/?active_status=all&ad_type=all&country=${country}&q=${encodeURIComponent(body.query.trim())}&search_type=keyword_unordered`;
|
|
12141
12592
|
const fbUser = c.get("user");
|
|
12142
|
-
const
|
|
12143
|
-
|
|
12593
|
+
const gate = await acquireConcurrencyGate(fbUser, "facebook_search", {
|
|
12594
|
+
reuseLockId: c.req.header("x-mcp-scraper-concurrency-lock"),
|
|
12595
|
+
metadata: { query: body.query.trim(), country }
|
|
12596
|
+
});
|
|
12597
|
+
if (!gate.ok) return c.json(concurrencyLimitExceededResponse(gate), 429, { "Retry-After": String(gate.retryAfterSeconds) });
|
|
12144
12598
|
const driver = new BrowserDriver();
|
|
12145
12599
|
let searchRefunded = false;
|
|
12600
|
+
let debited = false;
|
|
12146
12601
|
try {
|
|
12602
|
+
const { ok, balance_mc } = await debitMc(fbUser.id, MC_COSTS.fb_search, LedgerOperation.FB_SEARCH, body.query.trim());
|
|
12603
|
+
if (!ok) return c.json(insufficientBalanceResponse(balance_mc, MC_COSTS.fb_search), 402);
|
|
12604
|
+
debited = true;
|
|
12147
12605
|
await driver.launch(await kernelLaunchOptsResidential());
|
|
12148
12606
|
const page = driver.getPage();
|
|
12149
12607
|
const collated = await collectAdLibraryResults(page, searchUrl, Math.max(maxResults * 4, 40));
|
|
@@ -12204,7 +12662,7 @@ var init_facebook_ad_routes = __esm({
|
|
|
12204
12662
|
return c.json(searchResult);
|
|
12205
12663
|
} catch (err) {
|
|
12206
12664
|
const msg = err instanceof Error ? err.message : String(err);
|
|
12207
|
-
if (!searchRefunded) await creditMc(fbUser.id, MC_COSTS.fb_search, LedgerOperation.FB_SEARCH_REFUND, "failed call");
|
|
12665
|
+
if (debited && !searchRefunded) await creditMc(fbUser.id, MC_COSTS.fb_search, LedgerOperation.FB_SEARCH_REFUND, "failed call");
|
|
12208
12666
|
await logRequestEvent({ userId: fbUser.id, source: "facebook_search", status: "failed", query: body.query.trim(), error: msg });
|
|
12209
12667
|
if (msg.toLowerCase().includes("blocked") || msg.toLowerCase().includes("captcha")) {
|
|
12210
12668
|
return c.json({ error: msg }, 503);
|
|
@@ -12212,6 +12670,7 @@ var init_facebook_ad_routes = __esm({
|
|
|
12212
12670
|
return c.json({ error: msg }, 500);
|
|
12213
12671
|
} finally {
|
|
12214
12672
|
await driver.close();
|
|
12673
|
+
await releaseConcurrencyGate(gate.lockId);
|
|
12215
12674
|
}
|
|
12216
12675
|
});
|
|
12217
12676
|
ALLOWED_MEDIA_HOSTS = ["fbcdn.net", "cdninstagram.com", "scontent.facebook.com", "scontent.cdninstagram.com"];
|
|
@@ -13215,6 +13674,7 @@ var init_maps_routes = __esm({
|
|
|
13215
13674
|
init_errors();
|
|
13216
13675
|
init_browser_service_env();
|
|
13217
13676
|
init_maps_search_rotation();
|
|
13677
|
+
init_concurrency_gates();
|
|
13218
13678
|
mapsApp = new import_hono5.Hono();
|
|
13219
13679
|
mapsApp.post("/search", createApiKeyAuth(), async (c) => {
|
|
13220
13680
|
const user = c.get("user");
|
|
@@ -13226,14 +13686,21 @@ var init_maps_routes = __esm({
|
|
|
13226
13686
|
if (!parsed.success) {
|
|
13227
13687
|
return c.json({ error: parsed.error.issues[0]?.message ?? "Invalid request" }, 400);
|
|
13228
13688
|
}
|
|
13229
|
-
const
|
|
13230
|
-
|
|
13231
|
-
|
|
13232
|
-
|
|
13233
|
-
|
|
13234
|
-
|
|
13235
|
-
if (!ok) return c.json(insufficientBalanceResponse(balance_mc, MC_COSTS.maps_search), 402);
|
|
13689
|
+
const gate = await acquireConcurrencyGate(user, "maps_search", {
|
|
13690
|
+
reuseLockId: c.req.header("x-mcp-scraper-concurrency-lock"),
|
|
13691
|
+
metadata: { query: parsed.data.query, location: parsed.data.location }
|
|
13692
|
+
});
|
|
13693
|
+
if (!gate.ok) return c.json(concurrencyLimitExceededResponse(gate), 429, { "Retry-After": String(gate.retryAfterSeconds) });
|
|
13694
|
+
let debited = false;
|
|
13236
13695
|
try {
|
|
13696
|
+
const { ok, balance_mc } = await debitMc(
|
|
13697
|
+
user.id,
|
|
13698
|
+
MC_COSTS.maps_search,
|
|
13699
|
+
LedgerOperation.MAPS_SEARCH,
|
|
13700
|
+
[parsed.data.query, parsed.data.location].filter(Boolean).join(" ")
|
|
13701
|
+
);
|
|
13702
|
+
if (!ok) return c.json(insufficientBalanceResponse(balance_mc, MC_COSTS.maps_search), 402);
|
|
13703
|
+
debited = true;
|
|
13237
13704
|
const result = await runMapsSearchWithRotation({
|
|
13238
13705
|
...parsed.data,
|
|
13239
13706
|
configuredKernelProxyId: browserServiceProxyId()
|
|
@@ -13249,7 +13716,7 @@ var init_maps_routes = __esm({
|
|
|
13249
13716
|
});
|
|
13250
13717
|
return c.json(result);
|
|
13251
13718
|
} catch (err) {
|
|
13252
|
-
await creditMc(user.id, MC_COSTS.maps_search, LedgerOperation.REFUND, "failed maps_search call");
|
|
13719
|
+
if (debited) await creditMc(user.id, MC_COSTS.maps_search, LedgerOperation.REFUND, "failed maps_search call");
|
|
13253
13720
|
const msg = err instanceof Error ? err.message : String(err);
|
|
13254
13721
|
await logRequestEvent({
|
|
13255
13722
|
userId: user.id,
|
|
@@ -13261,6 +13728,8 @@ var init_maps_routes = __esm({
|
|
|
13261
13728
|
result: err instanceof MapsSearchRotationError ? { attempts: err.attempts } : void 0
|
|
13262
13729
|
});
|
|
13263
13730
|
return mapsErrorResponse(c, err, "maps_search_failed");
|
|
13731
|
+
} finally {
|
|
13732
|
+
await releaseConcurrencyGate(gate.lockId);
|
|
13264
13733
|
}
|
|
13265
13734
|
});
|
|
13266
13735
|
mapsApp.post("/place", createApiKeyAuth(), async (c) => {
|
|
@@ -13273,17 +13742,24 @@ var init_maps_routes = __esm({
|
|
|
13273
13742
|
if (!parsed.success) {
|
|
13274
13743
|
return c.json({ error: parsed.error.issues[0]?.message ?? "Invalid request" }, 400);
|
|
13275
13744
|
}
|
|
13276
|
-
const
|
|
13277
|
-
|
|
13278
|
-
|
|
13279
|
-
|
|
13280
|
-
|
|
13281
|
-
);
|
|
13282
|
-
if (!ok) return c.json(insufficientBalanceResponse(balance_mc, MC_COSTS.maps_place), 402);
|
|
13745
|
+
const gate = await acquireConcurrencyGate(user, "maps_place", {
|
|
13746
|
+
reuseLockId: c.req.header("x-mcp-scraper-concurrency-lock"),
|
|
13747
|
+
metadata: { businessName: parsed.data.businessName, location: parsed.data.location }
|
|
13748
|
+
});
|
|
13749
|
+
if (!gate.ok) return c.json(concurrencyLimitExceededResponse(gate), 429, { "Retry-After": String(gate.retryAfterSeconds) });
|
|
13283
13750
|
const driver = new BrowserDriver();
|
|
13284
13751
|
const extractor = new MapsExtractor(driver);
|
|
13285
13752
|
let reviewDebitMc = 0;
|
|
13753
|
+
let debited = false;
|
|
13286
13754
|
try {
|
|
13755
|
+
const { ok, balance_mc } = await debitMc(
|
|
13756
|
+
user.id,
|
|
13757
|
+
MC_COSTS.maps_place,
|
|
13758
|
+
LedgerOperation.MAPS_PLACE,
|
|
13759
|
+
`${parsed.data.businessName} ${parsed.data.location}`
|
|
13760
|
+
);
|
|
13761
|
+
if (!ok) return c.json(insufficientBalanceResponse(balance_mc, MC_COSTS.maps_place), 402);
|
|
13762
|
+
debited = true;
|
|
13287
13763
|
const result = await extractor.extract(parsed.data);
|
|
13288
13764
|
const reviewCount = Array.isArray(result.reviews) ? result.reviews.length : 0;
|
|
13289
13765
|
if (reviewCount > 0) {
|
|
@@ -13311,7 +13787,7 @@ var init_maps_routes = __esm({
|
|
|
13311
13787
|
});
|
|
13312
13788
|
return c.json(result);
|
|
13313
13789
|
} catch (err) {
|
|
13314
|
-
await creditMc(user.id, MC_COSTS.maps_place, LedgerOperation.REFUND, "failed maps_place call");
|
|
13790
|
+
if (debited) await creditMc(user.id, MC_COSTS.maps_place, LedgerOperation.REFUND, "failed maps_place call");
|
|
13315
13791
|
if (reviewDebitMc > 0) {
|
|
13316
13792
|
await creditMc(
|
|
13317
13793
|
user.id,
|
|
@@ -13332,6 +13808,7 @@ var init_maps_routes = __esm({
|
|
|
13332
13808
|
return mapsErrorResponse(c, msg, "maps_place_failed");
|
|
13333
13809
|
} finally {
|
|
13334
13810
|
await driver.close();
|
|
13811
|
+
await releaseConcurrencyGate(gate.lockId);
|
|
13335
13812
|
}
|
|
13336
13813
|
});
|
|
13337
13814
|
}
|
|
@@ -13969,6 +14446,8 @@ function formatCreditsInfo(raw, input) {
|
|
|
13969
14446
|
const costs = d.costs ?? [];
|
|
13970
14447
|
const matched = d.matched_cost;
|
|
13971
14448
|
const ledger = d.ledger ?? [];
|
|
14449
|
+
const concurrencyRaw = d.concurrency;
|
|
14450
|
+
const upgradeRaw = concurrencyRaw?.upgrade;
|
|
13972
14451
|
const costRows = costs.map((c) => {
|
|
13973
14452
|
const notes = c.notes ? ` ${c.notes}` : "";
|
|
13974
14453
|
return `| ${c.label} | ${c.credits} | ${c.unit}${notes} |`;
|
|
@@ -13984,10 +14463,20 @@ function formatCreditsInfo(raw, input) {
|
|
|
13984
14463
|
${matched.notes}` : ""}` : input.item ? `
|
|
13985
14464
|
## Matched Cost
|
|
13986
14465
|
No exact cost match found for "${input.item}". See the full cost table below.` : "";
|
|
14466
|
+
const concurrencySection = concurrencyRaw ? [
|
|
14467
|
+
`
|
|
14468
|
+
## Concurrency`,
|
|
14469
|
+
`**Current limit:** ${concurrencyRaw.current_limit ?? "unknown"} concurrent operation${concurrencyRaw.current_limit === 1 ? "" : "s"}`,
|
|
14470
|
+
`**Extra slots:** ${concurrencyRaw.current_extra_slots ?? "unknown"}`,
|
|
14471
|
+
`**Extra concurrency slot:** ${upgradeRaw?.price_label ?? "$5/month"}`,
|
|
14472
|
+
`**Upgrade in terminal:** \`${upgradeRaw?.terminal_command ?? "npx -y -p mcp-scraper@latest mcp-scraper-cli billing concurrency checkout"}\``,
|
|
14473
|
+
`**Billing URL:** ${upgradeRaw?.billing_url ?? "https://mcpscraper.dev/billing"}`
|
|
14474
|
+
].join("\n") : "";
|
|
13987
14475
|
const full = [
|
|
13988
14476
|
`# Credits`,
|
|
13989
14477
|
`**Balance:** ${balance ?? "unknown"} credits`,
|
|
13990
14478
|
matchedSection,
|
|
14479
|
+
concurrencySection,
|
|
13991
14480
|
costs.length ? `
|
|
13992
14481
|
## Cost Table
|
|
13993
14482
|
| Item | Credits | Unit |
|
|
@@ -14016,7 +14505,22 @@ ${ledgerRows}` : ""
|
|
|
14016
14505
|
operation: String(row.operation ?? ""),
|
|
14017
14506
|
credits: row.amount_mc / 1e3,
|
|
14018
14507
|
description: row.description ?? null
|
|
14019
|
-
}))
|
|
14508
|
+
})),
|
|
14509
|
+
concurrency: concurrencyRaw && upgradeRaw ? {
|
|
14510
|
+
currentExtraSlots: Number(concurrencyRaw.current_extra_slots ?? 0),
|
|
14511
|
+
currentLimit: Number(concurrencyRaw.current_limit ?? 1),
|
|
14512
|
+
hasSubscription: concurrencyRaw.has_subscription === true,
|
|
14513
|
+
upgrade: {
|
|
14514
|
+
product: String(upgradeRaw.product ?? "Extra concurrency slot"),
|
|
14515
|
+
priceLabel: String(upgradeRaw.price_label ?? "$5/month"),
|
|
14516
|
+
unitAmountUsd: Number(upgradeRaw.unit_amount_usd ?? 5),
|
|
14517
|
+
currency: String(upgradeRaw.currency ?? "usd"),
|
|
14518
|
+
interval: String(upgradeRaw.interval ?? "month"),
|
|
14519
|
+
billingUrl: String(upgradeRaw.billing_url ?? "https://mcpscraper.dev/billing"),
|
|
14520
|
+
terminalCommand: String(upgradeRaw.terminal_command ?? "npx -y -p mcp-scraper@latest mcp-scraper-cli billing concurrency checkout"),
|
|
14521
|
+
terminalCommandWithApiKeyEnv: String(upgradeRaw.terminal_command_with_api_key_env ?? "MCP_SCRAPER_API_KEY=sk_live_your_key npx -y -p mcp-scraper@latest mcp-scraper-cli billing concurrency checkout")
|
|
14522
|
+
}
|
|
14523
|
+
} : null
|
|
14020
14524
|
}
|
|
14021
14525
|
};
|
|
14022
14526
|
}
|
|
@@ -14302,6 +14806,62 @@ ${chunkRows}` : "",
|
|
|
14302
14806
|
].filter(Boolean).join("\n");
|
|
14303
14807
|
return oneBlock(full);
|
|
14304
14808
|
}
|
|
14809
|
+
function formatFacebookVideoTranscribe(raw, input) {
|
|
14810
|
+
const parsed = parseData(raw);
|
|
14811
|
+
if ("error" in parsed) return { content: [{ type: "text", text: parsed.error }], isError: true };
|
|
14812
|
+
const d = parsed.data;
|
|
14813
|
+
const text = d.text ?? "";
|
|
14814
|
+
const chunks = d.chunks ?? [];
|
|
14815
|
+
const wordCount = text.trim() ? text.trim().split(/\s+/).length : 0;
|
|
14816
|
+
const durSec = d.durationMs ? (d.durationMs / 1e3).toFixed(0) : "\u2014";
|
|
14817
|
+
const videoDuration = typeof d.videoDurationSec === "number" ? `${Math.round(d.videoDurationSec)}s` : "\u2014";
|
|
14818
|
+
const label = d.videoId ? `Facebook Organic Video \`${d.videoId}\`` : "Facebook Organic Video";
|
|
14819
|
+
const chunkRows = chunks.slice(0, 50).map((c) => {
|
|
14820
|
+
const sec = Number.isFinite(c.timestamp[0]) ? Math.floor(c.timestamp[0]) : 0;
|
|
14821
|
+
const mm = String(Math.floor(sec / 60)).padStart(2, "0");
|
|
14822
|
+
const ss = String(sec % 60).padStart(2, "0");
|
|
14823
|
+
return `| ${mm}:${ss} | ${cell(truncate(c.text, 120))} |`;
|
|
14824
|
+
}).join("\n");
|
|
14825
|
+
const full = [
|
|
14826
|
+
`# ${label} Transcript`,
|
|
14827
|
+
d.ownerName ? `**Owner:** ${d.ownerName}` : "",
|
|
14828
|
+
`**Video duration:** ${videoDuration} \xB7 **Transcribed in:** ${durSec}s \xB7 **${wordCount} words**`,
|
|
14829
|
+
d.pageUrl ? `**Page URL:** ${d.pageUrl}` : `**Page URL:** ${input.url}`,
|
|
14830
|
+
d.videoUrl ? `**Extracted MP4:** \`${d.videoUrl}\`` : "",
|
|
14831
|
+
`
|
|
14832
|
+
## Full Transcript
|
|
14833
|
+
${text}`,
|
|
14834
|
+
chunks.length ? `
|
|
14835
|
+
## Timestamped Chunks
|
|
14836
|
+
| Time | Text |
|
|
14837
|
+
|------|------|
|
|
14838
|
+
${chunkRows}` : "",
|
|
14839
|
+
`
|
|
14840
|
+
---
|
|
14841
|
+
\u{1F4A1} To download the extracted MP4, call the Facebook media endpoint with the extracted MP4 URL.`
|
|
14842
|
+
].filter(Boolean).join("\n");
|
|
14843
|
+
return {
|
|
14844
|
+
...oneBlock(full),
|
|
14845
|
+
structuredContent: {
|
|
14846
|
+
sourceUrl: d.sourceUrl ?? input.url,
|
|
14847
|
+
pageUrl: d.pageUrl ?? input.url,
|
|
14848
|
+
videoId: d.videoId ?? null,
|
|
14849
|
+
ownerName: d.ownerName ?? null,
|
|
14850
|
+
selectedQuality: d.selectedQuality ?? input.quality ?? "best",
|
|
14851
|
+
bitrate: typeof d.bitrate === "number" ? d.bitrate : null,
|
|
14852
|
+
videoDurationSec: typeof d.videoDurationSec === "number" ? d.videoDurationSec : null,
|
|
14853
|
+
videoUrl: d.videoUrl ?? "",
|
|
14854
|
+
wordCount,
|
|
14855
|
+
chunkCount: chunks.length,
|
|
14856
|
+
transcriptText: text,
|
|
14857
|
+
chunks: chunks.map((c) => ({
|
|
14858
|
+
startSec: Number.isFinite(c.timestamp[0]) ? c.timestamp[0] : 0,
|
|
14859
|
+
endSec: Number.isFinite(c.timestamp[1]) ? c.timestamp[1] : 0,
|
|
14860
|
+
text: c.text
|
|
14861
|
+
}))
|
|
14862
|
+
}
|
|
14863
|
+
};
|
|
14864
|
+
}
|
|
14305
14865
|
var import_node_fs3, import_node_os3, import_node_path5, reportSavingEnabled;
|
|
14306
14866
|
var init_mcp_response_formatter = __esm({
|
|
14307
14867
|
"src/mcp/mcp-response-formatter.ts"() {
|
|
@@ -14811,6 +15371,7 @@ var init_directory_routes = __esm({
|
|
|
14811
15371
|
init_directory_workflow();
|
|
14812
15372
|
init_location_db();
|
|
14813
15373
|
init_browser_service_env();
|
|
15374
|
+
init_concurrency_gates();
|
|
14814
15375
|
directoryApp = new import_hono6.Hono();
|
|
14815
15376
|
directoryApp.post("/run", createApiKeyAuth(), async (c) => {
|
|
14816
15377
|
const user = c.get("user");
|
|
@@ -14826,18 +15387,27 @@ var init_directory_routes = __esm({
|
|
|
14826
15387
|
if (!kernelApiKey && parsed.data.proxyMode !== "none") {
|
|
14827
15388
|
return c.json({ error: "Browser service API key is required for directory workflow Maps searches unless proxyMode is none" }, 503);
|
|
14828
15389
|
}
|
|
14829
|
-
const
|
|
14830
|
-
|
|
14831
|
-
|
|
14832
|
-
|
|
14833
|
-
|
|
14834
|
-
|
|
14835
|
-
|
|
14836
|
-
|
|
14837
|
-
);
|
|
14838
|
-
if (!debit.ok) return c.json(insufficientBalanceResponse(debit.balance_mc, requiredMc), 402);
|
|
14839
|
-
}
|
|
15390
|
+
const gate = await acquireConcurrencyGate(user, "directory_workflow", {
|
|
15391
|
+
reuseLockId: c.req.header("x-mcp-scraper-concurrency-lock"),
|
|
15392
|
+
ttlSeconds: 30 * 60,
|
|
15393
|
+
metadata: { query: parsed.data.query, state: parsed.data.state }
|
|
15394
|
+
});
|
|
15395
|
+
if (!gate.ok) return c.json(concurrencyLimitExceededResponse(gate), 429, { "Retry-After": String(gate.retryAfterSeconds) });
|
|
15396
|
+
let debited = false;
|
|
15397
|
+
let requiredMc = 0;
|
|
14840
15398
|
try {
|
|
15399
|
+
const plan = await resolveDirectoryMarkets(parsed.data);
|
|
15400
|
+
requiredMc = plan.markets.length * MC_COSTS.maps_search;
|
|
15401
|
+
if (requiredMc > 0) {
|
|
15402
|
+
const debit = await debitMc(
|
|
15403
|
+
user.id,
|
|
15404
|
+
requiredMc,
|
|
15405
|
+
LedgerOperation.MAPS_SEARCH,
|
|
15406
|
+
`directory_workflow ${parsed.data.query} ${parsed.data.state} ${plan.markets.length} cities`
|
|
15407
|
+
);
|
|
15408
|
+
if (!debit.ok) return c.json(insufficientBalanceResponse(debit.balance_mc, requiredMc), 402);
|
|
15409
|
+
debited = true;
|
|
15410
|
+
}
|
|
14841
15411
|
const result = await runDirectoryWorkflowFromPlan(parsed.data, plan);
|
|
14842
15412
|
const failedCities = result.cities.filter((city) => city.status === "failed").length;
|
|
14843
15413
|
if (failedCities > 0) {
|
|
@@ -14854,7 +15424,7 @@ var init_directory_routes = __esm({
|
|
|
14854
15424
|
});
|
|
14855
15425
|
return c.json(result);
|
|
14856
15426
|
} catch (err) {
|
|
14857
|
-
if (requiredMc > 0) {
|
|
15427
|
+
if (debited && requiredMc > 0) {
|
|
14858
15428
|
await creditMc(user.id, requiredMc, LedgerOperation.REFUND, "failed directory_workflow call");
|
|
14859
15429
|
}
|
|
14860
15430
|
const message = err instanceof Error ? err.message : String(err);
|
|
@@ -14867,6 +15437,8 @@ var init_directory_routes = __esm({
|
|
|
14867
15437
|
error: message
|
|
14868
15438
|
});
|
|
14869
15439
|
return c.json({ error: message, error_code: "directory_workflow_failed", retryable: true }, 500);
|
|
15440
|
+
} finally {
|
|
15441
|
+
await releaseConcurrencyGate(gate.lockId);
|
|
14870
15442
|
}
|
|
14871
15443
|
});
|
|
14872
15444
|
}
|
|
@@ -15015,18 +15587,21 @@ var init_http_client2 = __esm({
|
|
|
15015
15587
|
"src/workflows/http-client.ts"() {
|
|
15016
15588
|
"use strict";
|
|
15017
15589
|
WorkflowHttpClient = class {
|
|
15018
|
-
constructor(apiUrl, apiKey, fetchImpl = fetch) {
|
|
15590
|
+
constructor(apiUrl, apiKey, fetchImpl = fetch, extraHeaders = {}) {
|
|
15019
15591
|
this.apiUrl = apiUrl;
|
|
15020
15592
|
this.apiKey = apiKey;
|
|
15021
15593
|
this.fetchImpl = fetchImpl;
|
|
15594
|
+
this.extraHeaders = extraHeaders;
|
|
15022
15595
|
}
|
|
15023
15596
|
apiUrl;
|
|
15024
15597
|
apiKey;
|
|
15025
15598
|
fetchImpl;
|
|
15599
|
+
extraHeaders;
|
|
15026
15600
|
async post(path6, body, timeoutMs = 18e4) {
|
|
15027
15601
|
const res = await this.fetchImpl(`${this.apiUrl.replace(/\/$/, "")}${path6}`, {
|
|
15028
15602
|
method: "POST",
|
|
15029
15603
|
headers: {
|
|
15604
|
+
...this.extraHeaders,
|
|
15030
15605
|
"Content-Type": "application/json",
|
|
15031
15606
|
"x-api-key": this.apiKey
|
|
15032
15607
|
},
|
|
@@ -16374,7 +16949,7 @@ async function runWorkflow(id, rawInput, options = {}) {
|
|
|
16374
16949
|
if (!apiKey) throw new Error("MCP_SCRAPER_API_KEY is required for workflow runs. Pass --api-key or set the environment variable.");
|
|
16375
16950
|
const apiUrl = options.apiUrl?.trim() || process.env.MCP_SCRAPER_API_URL?.trim() || "https://mcpscraper.dev";
|
|
16376
16951
|
const artifacts = await ArtifactWriter.create(definition.id, definition.title, input, options.outputDir, options.runId);
|
|
16377
|
-
const client2 = new WorkflowHttpClient(apiUrl, apiKey, options.fetchImpl);
|
|
16952
|
+
const client2 = new WorkflowHttpClient(apiUrl, apiKey, options.fetchImpl, options.headers);
|
|
16378
16953
|
try {
|
|
16379
16954
|
return await definition.run(input, {
|
|
16380
16955
|
runId: artifacts.runId,
|
|
@@ -16473,7 +17048,7 @@ async function readManifestFromSummary(summary) {
|
|
|
16473
17048
|
function webhookSignature(body, timestamp2) {
|
|
16474
17049
|
const secret2 = process.env.MCP_SCRAPER_WEBHOOK_SECRET?.trim();
|
|
16475
17050
|
if (!secret2) return null;
|
|
16476
|
-
return (0,
|
|
17051
|
+
return (0, import_node_crypto3.createHmac)("sha256", secret2).update(`${timestamp2}.${body}`).digest("hex");
|
|
16477
17052
|
}
|
|
16478
17053
|
async function deliverWorkflowWebhook(input) {
|
|
16479
17054
|
if (!input.webhookUrl) return;
|
|
@@ -16527,7 +17102,8 @@ async function executeWorkflowRun(input) {
|
|
|
16527
17102
|
apiKey: input.user.api_key,
|
|
16528
17103
|
apiUrl: input.apiUrl,
|
|
16529
17104
|
outputDir: hostedWorkflowOutputDir(),
|
|
16530
|
-
runId: input.runId
|
|
17105
|
+
runId: input.runId,
|
|
17106
|
+
headers: input.concurrencyLockId ? { "x-mcp-scraper-concurrency-lock": input.concurrencyLockId } : void 0
|
|
16531
17107
|
});
|
|
16532
17108
|
const manifest = await readManifestFromSummary(summary);
|
|
16533
17109
|
await completeWorkflowRunRecord(input.runId, summary.status, manifest ?? { summary });
|
|
@@ -16556,19 +17132,28 @@ async function dispatchDueWorkflowSchedules(apiUrl, limit = 3) {
|
|
|
16556
17132
|
continue;
|
|
16557
17133
|
}
|
|
16558
17134
|
const scheduledFor = schedule.next_run_at ?? now;
|
|
16559
|
-
const
|
|
16560
|
-
|
|
16561
|
-
scheduleId: schedule.id,
|
|
16562
|
-
workflowId: schedule.workflow_id,
|
|
16563
|
-
workflowInput: schedule.input,
|
|
16564
|
-
idempotencyKey: `${schedule.id}:${scheduledFor}`
|
|
17135
|
+
const gate = await acquireConcurrencyGate(user, "workflow_schedule_dispatch", {
|
|
17136
|
+
ttlSeconds: 60 * 60,
|
|
17137
|
+
metadata: { scheduleId: schedule.id, workflowId: schedule.workflow_id }
|
|
16565
17138
|
});
|
|
16566
|
-
|
|
16567
|
-
|
|
16568
|
-
results.push({ scheduleId: schedule.id, runId: run.id, status: run.status });
|
|
17139
|
+
if (!gate.ok) {
|
|
17140
|
+
results.push({ scheduleId: schedule.id, status: "deferred", error: concurrencyLimitExceededResponse(gate).message });
|
|
16569
17141
|
continue;
|
|
16570
17142
|
}
|
|
17143
|
+
let run = null;
|
|
16571
17144
|
try {
|
|
17145
|
+
run = await createWorkflowRun({
|
|
17146
|
+
userId: user.id,
|
|
17147
|
+
scheduleId: schedule.id,
|
|
17148
|
+
workflowId: schedule.workflow_id,
|
|
17149
|
+
workflowInput: schedule.input,
|
|
17150
|
+
idempotencyKey: `${schedule.id}:${scheduledFor}`
|
|
17151
|
+
});
|
|
17152
|
+
await markWorkflowScheduleRan(schedule.id, now, addCadence(scheduledFor, schedule.cadence));
|
|
17153
|
+
if (run.status !== "queued") {
|
|
17154
|
+
results.push({ scheduleId: schedule.id, runId: run.id, status: run.status });
|
|
17155
|
+
continue;
|
|
17156
|
+
}
|
|
16572
17157
|
const executed = await executeWorkflowRun({
|
|
16573
17158
|
runId: run.id,
|
|
16574
17159
|
user,
|
|
@@ -16576,20 +17161,23 @@ async function dispatchDueWorkflowSchedules(apiUrl, limit = 3) {
|
|
|
16576
17161
|
workflowInput: schedule.input,
|
|
16577
17162
|
apiUrl,
|
|
16578
17163
|
scheduleId: schedule.id,
|
|
16579
|
-
webhookUrl: schedule.webhook_url
|
|
17164
|
+
webhookUrl: schedule.webhook_url,
|
|
17165
|
+
concurrencyLockId: gate.lockId
|
|
16580
17166
|
});
|
|
16581
17167
|
results.push({ scheduleId: schedule.id, runId: executed.run.id, status: executed.run.status });
|
|
16582
17168
|
} catch (err) {
|
|
16583
|
-
results.push({ scheduleId: schedule.id, runId: run
|
|
17169
|
+
results.push({ scheduleId: schedule.id, runId: run?.id, status: "failed", error: err instanceof Error ? err.message : String(err) });
|
|
17170
|
+
} finally {
|
|
17171
|
+
await releaseConcurrencyGate(gate.lockId);
|
|
16584
17172
|
}
|
|
16585
17173
|
}
|
|
16586
17174
|
return { dispatched: results.length, results };
|
|
16587
17175
|
}
|
|
16588
|
-
var
|
|
17176
|
+
var import_node_crypto3, import_promises7, import_hono7, import_zod24, workflowApp, WorkflowInputSchema, WorkflowIdSchema, CadenceSchema, ScheduleStatusSchema, RunBodySchema, ScheduleCreateSchema, SchedulePatchSchema;
|
|
16589
17177
|
var init_workflow_routes = __esm({
|
|
16590
17178
|
"src/api/workflow-routes.ts"() {
|
|
16591
17179
|
"use strict";
|
|
16592
|
-
|
|
17180
|
+
import_node_crypto3 = require("crypto");
|
|
16593
17181
|
import_promises7 = require("fs/promises");
|
|
16594
17182
|
import_hono7 = require("hono");
|
|
16595
17183
|
import_zod24 = require("zod");
|
|
@@ -16598,6 +17186,7 @@ var init_workflow_routes = __esm({
|
|
|
16598
17186
|
init_api_auth();
|
|
16599
17187
|
init_db();
|
|
16600
17188
|
init_url_utils();
|
|
17189
|
+
init_concurrency_gates();
|
|
16601
17190
|
workflowApp = new import_hono7.Hono();
|
|
16602
17191
|
WorkflowInputSchema = import_zod24.z.record(import_zod24.z.unknown()).default({});
|
|
16603
17192
|
WorkflowIdSchema = import_zod24.z.string().min(1);
|
|
@@ -16641,20 +17230,29 @@ var init_workflow_routes = __esm({
|
|
|
16641
17230
|
} catch (err) {
|
|
16642
17231
|
return c.json({ error: err instanceof Error ? err.message : String(err) }, 400);
|
|
16643
17232
|
}
|
|
16644
|
-
const
|
|
17233
|
+
const gate = await acquireConcurrencyGate(user, "workflow_run", {
|
|
17234
|
+
ttlSeconds: 60 * 60,
|
|
17235
|
+
metadata: { workflowId: parsed.data.workflowId }
|
|
17236
|
+
});
|
|
17237
|
+
if (!gate.ok) return c.json(concurrencyLimitExceededResponse(gate), 429, { "Retry-After": String(gate.retryAfterSeconds) });
|
|
17238
|
+
let run = null;
|
|
16645
17239
|
try {
|
|
17240
|
+
run = await createWorkflowRun({ userId: user.id, workflowId: parsed.data.workflowId, workflowInput: input });
|
|
16646
17241
|
const executed = await executeWorkflowRun({
|
|
16647
17242
|
runId: run.id,
|
|
16648
17243
|
user,
|
|
16649
17244
|
workflowId: parsed.data.workflowId,
|
|
16650
17245
|
workflowInput: input,
|
|
16651
17246
|
apiUrl: originFromUrl(c.req.url),
|
|
16652
|
-
webhookUrl
|
|
17247
|
+
webhookUrl,
|
|
17248
|
+
concurrencyLockId: gate.lockId
|
|
16653
17249
|
});
|
|
16654
17250
|
return c.json({ run: exposeRun(c, executed.run, executed.artifacts), summary: executed.summary });
|
|
16655
17251
|
} catch (err) {
|
|
16656
|
-
const failed = await getWorkflowRun(run.id, user.id);
|
|
17252
|
+
const failed = run ? await getWorkflowRun(run.id, user.id) : null;
|
|
16657
17253
|
return c.json({ run: failed, error: err instanceof Error ? err.message : String(err) }, 500);
|
|
17254
|
+
} finally {
|
|
17255
|
+
await releaseConcurrencyGate(gate.lockId);
|
|
16658
17256
|
}
|
|
16659
17257
|
});
|
|
16660
17258
|
workflowApp.get("/runs", createApiKeyAuth(), async (c) => {
|
|
@@ -16752,8 +17350,14 @@ var init_workflow_routes = __esm({
|
|
|
16752
17350
|
const user = c.get("user");
|
|
16753
17351
|
const schedule = await getWorkflowSchedule(c.req.param("id"), user.id);
|
|
16754
17352
|
if (!schedule) return c.json({ error: "Schedule not found" }, 404);
|
|
16755
|
-
const
|
|
17353
|
+
const gate = await acquireConcurrencyGate(user, "workflow_schedule_run", {
|
|
17354
|
+
ttlSeconds: 60 * 60,
|
|
17355
|
+
metadata: { scheduleId: schedule.id, workflowId: schedule.workflow_id }
|
|
17356
|
+
});
|
|
17357
|
+
if (!gate.ok) return c.json(concurrencyLimitExceededResponse(gate), 429, { "Retry-After": String(gate.retryAfterSeconds) });
|
|
17358
|
+
let run = null;
|
|
16756
17359
|
try {
|
|
17360
|
+
run = await createWorkflowRun({ userId: user.id, scheduleId: schedule.id, workflowId: schedule.workflow_id, workflowInput: schedule.input });
|
|
16757
17361
|
const executed = await executeWorkflowRun({
|
|
16758
17362
|
runId: run.id,
|
|
16759
17363
|
user,
|
|
@@ -16761,12 +17365,15 @@ var init_workflow_routes = __esm({
|
|
|
16761
17365
|
workflowInput: schedule.input,
|
|
16762
17366
|
apiUrl: originFromUrl(c.req.url),
|
|
16763
17367
|
scheduleId: schedule.id,
|
|
16764
|
-
webhookUrl: schedule.webhook_url
|
|
17368
|
+
webhookUrl: schedule.webhook_url,
|
|
17369
|
+
concurrencyLockId: gate.lockId
|
|
16765
17370
|
});
|
|
16766
17371
|
return c.json({ run: exposeRun(c, executed.run, executed.artifacts), summary: executed.summary });
|
|
16767
17372
|
} catch (err) {
|
|
16768
|
-
const failed = await getWorkflowRun(run.id, user.id);
|
|
17373
|
+
const failed = run ? await getWorkflowRun(run.id, user.id) : null;
|
|
16769
17374
|
return c.json({ run: failed, error: err instanceof Error ? err.message : String(err) }, 500);
|
|
17375
|
+
} finally {
|
|
17376
|
+
await releaseConcurrencyGate(gate.lockId);
|
|
16770
17377
|
}
|
|
16771
17378
|
});
|
|
16772
17379
|
workflowApp.post("/cron/dispatch", async (c) => {
|
|
@@ -16782,7 +17389,7 @@ var init_workflow_routes = __esm({
|
|
|
16782
17389
|
// src/serp-intelligence/page-snapshot-extractor.ts
|
|
16783
17390
|
function sha256(value) {
|
|
16784
17391
|
if (!value) return null;
|
|
16785
|
-
return (0,
|
|
17392
|
+
return (0, import_node_crypto4.createHash)("sha256").update(value).digest("hex");
|
|
16786
17393
|
}
|
|
16787
17394
|
function countWords(markdown) {
|
|
16788
17395
|
const matches = markdown.trim().match(/\b[\p{L}\p{N}][\p{L}\p{N}'-]*\b/gu);
|
|
@@ -17082,11 +17689,11 @@ async function capturePageSnapshots(targets, options = {}) {
|
|
|
17082
17689
|
}
|
|
17083
17690
|
};
|
|
17084
17691
|
}
|
|
17085
|
-
var
|
|
17692
|
+
var import_node_crypto4, import_p_limit3, DEFAULT_TIMEOUT_MS, DEFAULT_MAX_CONCURRENCY, DEFAULT_MAX_CONTENT_CHARS;
|
|
17086
17693
|
var init_page_snapshot_extractor = __esm({
|
|
17087
17694
|
"src/serp-intelligence/page-snapshot-extractor.ts"() {
|
|
17088
17695
|
"use strict";
|
|
17089
|
-
|
|
17696
|
+
import_node_crypto4 = require("crypto");
|
|
17090
17697
|
import_p_limit3 = __toESM(require("p-limit"), 1);
|
|
17091
17698
|
init_kpo_extractor();
|
|
17092
17699
|
init_url_utils();
|
|
@@ -19382,26 +19989,6 @@ async function enforceSerpIntelligenceRateLimit(c, userId) {
|
|
|
19382
19989
|
{ "Retry-After": String(result.resetSeconds) }
|
|
19383
19990
|
);
|
|
19384
19991
|
}
|
|
19385
|
-
async function enforceSerpIntelligenceConcurrency(user) {
|
|
19386
|
-
const active = await countActiveJobsForUser(user.id);
|
|
19387
|
-
const limit = 1 + (user.extra_concurrency_slots ?? 0);
|
|
19388
|
-
if (active < limit) return null;
|
|
19389
|
-
return new Response(
|
|
19390
|
-
JSON.stringify(structuredError({
|
|
19391
|
-
error_code: "concurrency_limit_exceeded",
|
|
19392
|
-
error_type: "concurrency_limit",
|
|
19393
|
-
message: `You have ${active} active job${active !== 1 ? "s" : ""}. Your account allows ${limit} concurrent job${limit !== 1 ? "s" : ""}.`,
|
|
19394
|
-
retryable: true
|
|
19395
|
-
})),
|
|
19396
|
-
{
|
|
19397
|
-
status: 429,
|
|
19398
|
-
headers: {
|
|
19399
|
-
"Content-Type": "application/json",
|
|
19400
|
-
"Retry-After": "30"
|
|
19401
|
-
}
|
|
19402
|
-
}
|
|
19403
|
-
);
|
|
19404
|
-
}
|
|
19405
19992
|
function pageSnapshotTargetsFromBody(body) {
|
|
19406
19993
|
return body.targets?.length ? body.targets : body.urls.map((url) => ({
|
|
19407
19994
|
url,
|
|
@@ -19421,6 +20008,7 @@ var init_serp_intelligence_routes = __esm({
|
|
|
19421
20008
|
init_api_auth();
|
|
19422
20009
|
init_db();
|
|
19423
20010
|
init_rates();
|
|
20011
|
+
init_concurrency_gates();
|
|
19424
20012
|
SERP_INTELLIGENCE_RATE_LIMIT = 60;
|
|
19425
20013
|
SERP_INTELLIGENCE_RATE_WINDOW_SECONDS = 60;
|
|
19426
20014
|
POST_CAPTURE_ROUTE_LABEL = "POST /capture";
|
|
@@ -19435,17 +20023,22 @@ var init_serp_intelligence_routes = __esm({
|
|
|
19435
20023
|
if (!parsed.success) return c.json(formatZodError(parsed.error), 400);
|
|
19436
20024
|
const limited = await enforceSerpIntelligenceRateLimit(c, user.id);
|
|
19437
20025
|
if (limited) return limited;
|
|
19438
|
-
const
|
|
19439
|
-
|
|
20026
|
+
const gate = await acquireConcurrencyGate(user, "serp_intelligence_capture", {
|
|
20027
|
+
reuseLockId: c.req.header("x-mcp-scraper-concurrency-lock"),
|
|
20028
|
+
metadata: { query: parsed.data.query, location: parsed.data.location }
|
|
20029
|
+
});
|
|
20030
|
+
if (!gate.ok) return c.json(concurrencyLimitExceededResponse(gate), 429, { "Retry-After": String(gate.retryAfterSeconds) });
|
|
19440
20031
|
const cost = MC_COSTS.serp;
|
|
19441
|
-
|
|
19442
|
-
user.id,
|
|
19443
|
-
cost,
|
|
19444
|
-
LedgerOperation.SERP,
|
|
19445
|
-
parsed.data.query
|
|
19446
|
-
);
|
|
19447
|
-
if (!ok) return c.json(insufficientBalanceResponse(balance_mc, cost), 402);
|
|
20032
|
+
let debited = false;
|
|
19448
20033
|
try {
|
|
20034
|
+
const { ok, balance_mc } = await debitMc(
|
|
20035
|
+
user.id,
|
|
20036
|
+
cost,
|
|
20037
|
+
LedgerOperation.SERP,
|
|
20038
|
+
parsed.data.query
|
|
20039
|
+
);
|
|
20040
|
+
if (!ok) return c.json(insufficientBalanceResponse(balance_mc, cost), 402);
|
|
20041
|
+
debited = true;
|
|
19449
20042
|
const result = await captureSerpIntelligenceSnapshot(parsed.data, {
|
|
19450
20043
|
kernelApiKey: browserServiceApiKey(),
|
|
19451
20044
|
kernelProxyId: browserServiceProxyId(),
|
|
@@ -19463,7 +20056,7 @@ var init_serp_intelligence_routes = __esm({
|
|
|
19463
20056
|
});
|
|
19464
20057
|
return c.json(result);
|
|
19465
20058
|
} catch (error) {
|
|
19466
|
-
await creditMc(user.id, cost, LedgerOperation.REFUND, "failed serp intelligence capture");
|
|
20059
|
+
if (debited) await creditMc(user.id, cost, LedgerOperation.REFUND, "failed serp intelligence capture");
|
|
19467
20060
|
const body = error instanceof SerpIntelligenceCaptureError ? error.toJSON() : structuredError({
|
|
19468
20061
|
error_code: "capture_failed",
|
|
19469
20062
|
error_type: "capture_error",
|
|
@@ -19480,6 +20073,8 @@ var init_serp_intelligence_routes = __esm({
|
|
|
19480
20073
|
});
|
|
19481
20074
|
const status = error instanceof SerpIntelligenceCaptureError ? error.httpStatus : 500;
|
|
19482
20075
|
return c.json(body, status);
|
|
20076
|
+
} finally {
|
|
20077
|
+
await releaseConcurrencyGate(gate.lockId);
|
|
19483
20078
|
}
|
|
19484
20079
|
});
|
|
19485
20080
|
serpIntelligenceApp.post("/page-snapshots", async (c) => {
|
|
@@ -19490,18 +20085,23 @@ var init_serp_intelligence_routes = __esm({
|
|
|
19490
20085
|
if (!parsed.success) return c.json(formatZodError(parsed.error), 400);
|
|
19491
20086
|
const limited = await enforceSerpIntelligenceRateLimit(c, user.id);
|
|
19492
20087
|
if (limited) return limited;
|
|
19493
|
-
const concurrencyLimited = await enforceSerpIntelligenceConcurrency(user);
|
|
19494
|
-
if (concurrencyLimited) return concurrencyLimited;
|
|
19495
20088
|
const targets = pageSnapshotTargetsFromBody(parsed.data);
|
|
20089
|
+
const gate = await acquireConcurrencyGate(user, "serp_intelligence_page_snapshots", {
|
|
20090
|
+
reuseLockId: c.req.header("x-mcp-scraper-concurrency-lock"),
|
|
20091
|
+
metadata: { targets: targets.length }
|
|
20092
|
+
});
|
|
20093
|
+
if (!gate.ok) return c.json(concurrencyLimitExceededResponse(gate), 429, { "Retry-After": String(gate.retryAfterSeconds) });
|
|
19496
20094
|
const cost = targets.length * MC_COSTS.page_scrape;
|
|
19497
|
-
|
|
19498
|
-
user.id,
|
|
19499
|
-
cost,
|
|
19500
|
-
LedgerOperation.EXTRACT_URL,
|
|
19501
|
-
`serp intelligence page snapshots: ${targets.length}`
|
|
19502
|
-
);
|
|
19503
|
-
if (!ok) return c.json(insufficientBalanceResponse(balance_mc, cost), 402);
|
|
20095
|
+
let debited = false;
|
|
19504
20096
|
try {
|
|
20097
|
+
const { ok, balance_mc } = await debitMc(
|
|
20098
|
+
user.id,
|
|
20099
|
+
cost,
|
|
20100
|
+
LedgerOperation.EXTRACT_URL,
|
|
20101
|
+
`serp intelligence page snapshots: ${targets.length}`
|
|
20102
|
+
);
|
|
20103
|
+
if (!ok) return c.json(insufficientBalanceResponse(balance_mc, cost), 402);
|
|
20104
|
+
debited = true;
|
|
19505
20105
|
const result = await capturePageSnapshots(targets, {
|
|
19506
20106
|
kernelApiKey: browserServiceApiKey(),
|
|
19507
20107
|
timeoutMs: parsed.data.timeoutMs,
|
|
@@ -19518,7 +20118,7 @@ var init_serp_intelligence_routes = __esm({
|
|
|
19518
20118
|
});
|
|
19519
20119
|
return c.json(result);
|
|
19520
20120
|
} catch (error) {
|
|
19521
|
-
await creditMc(user.id, cost, LedgerOperation.REFUND, "failed serp intelligence page snapshots");
|
|
20121
|
+
if (debited) await creditMc(user.id, cost, LedgerOperation.REFUND, "failed serp intelligence page snapshots");
|
|
19522
20122
|
const body = structuredError({
|
|
19523
20123
|
error_code: "page_snapshot_failed",
|
|
19524
20124
|
error_type: "capture_error",
|
|
@@ -19533,6 +20133,8 @@ var init_serp_intelligence_routes = __esm({
|
|
|
19533
20133
|
error: body.message
|
|
19534
20134
|
});
|
|
19535
20135
|
return c.json(body, 500);
|
|
20136
|
+
} finally {
|
|
20137
|
+
await releaseConcurrencyGate(gate.lockId);
|
|
19536
20138
|
}
|
|
19537
20139
|
});
|
|
19538
20140
|
}
|
|
@@ -19543,12 +20145,12 @@ var PACKAGE_VERSION;
|
|
|
19543
20145
|
var init_version = __esm({
|
|
19544
20146
|
"src/version.ts"() {
|
|
19545
20147
|
"use strict";
|
|
19546
|
-
PACKAGE_VERSION = "0.2.
|
|
20148
|
+
PACKAGE_VERSION = "0.2.15";
|
|
19547
20149
|
}
|
|
19548
20150
|
});
|
|
19549
20151
|
|
|
19550
20152
|
// src/mcp/mcp-tool-schemas.ts
|
|
19551
|
-
var import_zod26, HarvestPaaInputSchema, ExtractUrlInputSchema, MapSiteUrlsInputSchema, ExtractSiteInputSchema, YoutubeHarvestInputSchema, YoutubeTranscribeInputSchema, FacebookPageIntelInputSchema, FacebookAdSearchInputSchema, FacebookAdTranscribeInputSchema, MapsPlaceIntelInputSchema, MapsSearchInputSchema, DirectoryWorkflowInputSchema, RankTrackerModeSchema, RankTrackerBlueprintInputSchema, NullableString, MapsSearchAttemptOutput, MapsSearchOutputSchema, DirectoryMapsBusinessOutput, DirectoryWorkflowOutputSchema, RankTrackerToolPlanOutput, RankTrackerTableOutput, RankTrackerCronJobOutput, RankTrackerBlueprintOutputSchema, OrganicResultOutput, AiOverviewOutput, EntityIdsOutput, HarvestPaaOutputSchema, SearchSerpOutputSchema, ExtractUrlOutputSchema, ExtractSiteOutputSchema, MapsPlaceIntelOutputSchema, CreditsInfoOutputSchema, MapSiteUrlsOutputSchema, YoutubeHarvestOutputSchema, FacebookAdSearchOutputSchema, FacebookPageIntelOutputSchema, CreditsInfoInputSchema, SearchSerpInputSchema, CaptureSerpSnapshotInputSchema, ScreenshotInputSchema, CaptureSerpPageSnapshotsInputSchema;
|
|
20153
|
+
var import_zod26, HarvestPaaInputSchema, ExtractUrlInputSchema, MapSiteUrlsInputSchema, ExtractSiteInputSchema, YoutubeHarvestInputSchema, YoutubeTranscribeInputSchema, FacebookPageIntelInputSchema, FacebookAdSearchInputSchema, FacebookAdTranscribeInputSchema, FacebookVideoTranscribeInputSchema, MapsPlaceIntelInputSchema, MapsSearchInputSchema, DirectoryWorkflowInputSchema, RankTrackerModeSchema, RankTrackerBlueprintInputSchema, NullableString, MapsSearchAttemptOutput, MapsSearchOutputSchema, DirectoryMapsBusinessOutput, DirectoryWorkflowOutputSchema, RankTrackerToolPlanOutput, RankTrackerTableOutput, RankTrackerCronJobOutput, RankTrackerBlueprintOutputSchema, OrganicResultOutput, AiOverviewOutput, EntityIdsOutput, HarvestPaaOutputSchema, SearchSerpOutputSchema, ExtractUrlOutputSchema, ExtractSiteOutputSchema, MapsPlaceIntelOutputSchema, CreditsInfoOutputSchema, MapSiteUrlsOutputSchema, YoutubeHarvestOutputSchema, FacebookAdSearchOutputSchema, FacebookPageIntelOutputSchema, FacebookVideoTranscribeOutputSchema, CreditsInfoInputSchema, SearchSerpInputSchema, CaptureSerpSnapshotInputSchema, ScreenshotInputSchema, CaptureSerpPageSnapshotsInputSchema;
|
|
19552
20154
|
var init_mcp_tool_schemas = __esm({
|
|
19553
20155
|
"src/mcp/mcp-tool-schemas.ts"() {
|
|
19554
20156
|
"use strict";
|
|
@@ -19605,6 +20207,10 @@ var init_mcp_tool_schemas = __esm({
|
|
|
19605
20207
|
FacebookAdTranscribeInputSchema = {
|
|
19606
20208
|
videoUrl: import_zod26.z.string().url().describe("Facebook CDN video URL from a facebook_page_intel result")
|
|
19607
20209
|
};
|
|
20210
|
+
FacebookVideoTranscribeInputSchema = {
|
|
20211
|
+
url: import_zod26.z.string().url().describe("Organic Facebook reel, video, watch, or share URL. The tool renders the page, extracts the best public Facebook CDN MP4 URL, then transcribes it."),
|
|
20212
|
+
quality: import_zod26.z.enum(["best", "hd", "sd"]).default("best").describe("Preferred progressive MP4 quality. Use best by default; hd prefers the highest HD progressive URL; sd forces the SD URL.")
|
|
20213
|
+
};
|
|
19608
20214
|
MapsPlaceIntelInputSchema = {
|
|
19609
20215
|
businessName: import_zod26.z.string().min(1).describe('Business name only. If user says "Elite Roofing Denver CO", use businessName="Elite Roofing" and location="Denver, CO".'),
|
|
19610
20216
|
location: import_zod26.z.string().min(1).describe('City/region/country where the business should be searched, e.g. "Denver, CO". Infer from the user request when possible.'),
|
|
@@ -19898,7 +20504,22 @@ var init_mcp_tool_schemas = __esm({
|
|
|
19898
20504
|
operation: import_zod26.z.string(),
|
|
19899
20505
|
credits: import_zod26.z.number(),
|
|
19900
20506
|
description: NullableString
|
|
19901
|
-
}))
|
|
20507
|
+
})),
|
|
20508
|
+
concurrency: import_zod26.z.object({
|
|
20509
|
+
currentExtraSlots: import_zod26.z.number().int().min(0),
|
|
20510
|
+
currentLimit: import_zod26.z.number().int().min(1),
|
|
20511
|
+
hasSubscription: import_zod26.z.boolean(),
|
|
20512
|
+
upgrade: import_zod26.z.object({
|
|
20513
|
+
product: import_zod26.z.string(),
|
|
20514
|
+
priceLabel: import_zod26.z.string(),
|
|
20515
|
+
unitAmountUsd: import_zod26.z.number(),
|
|
20516
|
+
currency: import_zod26.z.string(),
|
|
20517
|
+
interval: import_zod26.z.string(),
|
|
20518
|
+
billingUrl: import_zod26.z.string().url(),
|
|
20519
|
+
terminalCommand: import_zod26.z.string(),
|
|
20520
|
+
terminalCommandWithApiKeyEnv: import_zod26.z.string()
|
|
20521
|
+
})
|
|
20522
|
+
}).nullable()
|
|
19902
20523
|
};
|
|
19903
20524
|
MapSiteUrlsOutputSchema = {
|
|
19904
20525
|
startUrl: import_zod26.z.string(),
|
|
@@ -19955,8 +20576,26 @@ var init_mcp_tool_schemas = __esm({
|
|
|
19955
20576
|
variations: import_zod26.z.number().int().nullable()
|
|
19956
20577
|
}))
|
|
19957
20578
|
};
|
|
20579
|
+
FacebookVideoTranscribeOutputSchema = {
|
|
20580
|
+
sourceUrl: import_zod26.z.string().url(),
|
|
20581
|
+
pageUrl: import_zod26.z.string().url(),
|
|
20582
|
+
videoId: NullableString,
|
|
20583
|
+
ownerName: NullableString,
|
|
20584
|
+
selectedQuality: import_zod26.z.string(),
|
|
20585
|
+
bitrate: import_zod26.z.number().int().nullable(),
|
|
20586
|
+
videoDurationSec: import_zod26.z.number().nullable(),
|
|
20587
|
+
videoUrl: import_zod26.z.string().url(),
|
|
20588
|
+
wordCount: import_zod26.z.number().int().min(0),
|
|
20589
|
+
chunkCount: import_zod26.z.number().int().min(0),
|
|
20590
|
+
transcriptText: import_zod26.z.string(),
|
|
20591
|
+
chunks: import_zod26.z.array(import_zod26.z.object({
|
|
20592
|
+
startSec: import_zod26.z.number(),
|
|
20593
|
+
endSec: import_zod26.z.number(),
|
|
20594
|
+
text: import_zod26.z.string()
|
|
20595
|
+
}))
|
|
20596
|
+
};
|
|
19958
20597
|
CreditsInfoInputSchema = {
|
|
19959
|
-
item: import_zod26.z.string().optional().describe('Optional tool, action, or feature to look up, e.g. "maps reviews", "extract_url",
|
|
20598
|
+
item: import_zod26.z.string().optional().describe('Optional tool, action, or feature to look up, e.g. "maps reviews", "extract_url", "YouTube transcription", or "concurrency"'),
|
|
19960
20599
|
includeLedger: import_zod26.z.boolean().default(false).describe("Whether to include recent credit ledger entries")
|
|
19961
20600
|
};
|
|
19962
20601
|
SearchSerpInputSchema = {
|
|
@@ -20462,6 +21101,13 @@ function registerPaaExtractorMcpTools(server, executor, options = {}) {
|
|
|
20462
21101
|
inputSchema: FacebookAdTranscribeInputSchema,
|
|
20463
21102
|
annotations: liveWebToolAnnotations("Facebook Ad Transcription")
|
|
20464
21103
|
}, async (input) => formatFacebookAdTranscribe(await executor.facebookAdTranscribe(input), input));
|
|
21104
|
+
server.registerTool("facebook_video_transcribe", {
|
|
21105
|
+
title: "Facebook Organic Video Transcription",
|
|
21106
|
+
description: withReportNote("Transcribe audio from an organic Facebook reel, video, watch, or share URL. Renders the Facebook page, extracts the best public Facebook CDN MP4 URL from the page state, then returns the full transcript, timestamped chunks, and extracted MP4 URL."),
|
|
21107
|
+
inputSchema: FacebookVideoTranscribeInputSchema,
|
|
21108
|
+
outputSchema: FacebookVideoTranscribeOutputSchema,
|
|
21109
|
+
annotations: liveWebToolAnnotations("Facebook Organic Video Transcription")
|
|
21110
|
+
}, async (input) => formatFacebookVideoTranscribe(await executor.facebookVideoTranscribe(input), input));
|
|
20465
21111
|
server.registerTool("maps_place_intel", {
|
|
20466
21112
|
title: "Google Maps Business Profile Details",
|
|
20467
21113
|
description: withReportNote('Extract Google Maps business intelligence for one known/named business: rating, review count, category, address, phone, website, hours, booking URL, review histogram, review topics, about attributes, entity IDs, and optional review cards. Do not use this for category searches, local market prospect lists, or requests for multiple GMB/GBP profiles; use maps_search first for those. Split business name from location (e.g. "Elite Roofing Denver CO" => businessName "Elite Roofing", location "Denver, CO"). Pass includeReviews true when the user asks for reviews/customer pain.'),
|
|
@@ -20492,7 +21138,7 @@ function registerPaaExtractorMcpTools(server, executor, options = {}) {
|
|
|
20492
21138
|
}, async (input) => buildRankTrackerBlueprint(input));
|
|
20493
21139
|
server.registerTool("credits_info", {
|
|
20494
21140
|
title: "MCP Scraper Credits & Costs",
|
|
20495
|
-
description: "Answer questions about MCP Scraper credits: current credit balance, what a specific tool/action costs, the full cost table, and
|
|
21141
|
+
description: "Answer questions about MCP Scraper credits, usage limits, and concurrency upgrades: current credit balance, what a specific tool/action costs, the full cost table, current concurrency limit, the extra-slot price, billing URL, and the terminal checkout command. Does not expose payment methods or credit card information.",
|
|
20496
21142
|
inputSchema: CreditsInfoInputSchema,
|
|
20497
21143
|
outputSchema: CreditsInfoOutputSchema,
|
|
20498
21144
|
annotations: {
|
|
@@ -20610,6 +21256,9 @@ var init_http_mcp_tool_executor = __esm({
|
|
|
20610
21256
|
facebookAdTranscribe(input) {
|
|
20611
21257
|
return this.call("/facebook/transcribe", input);
|
|
20612
21258
|
}
|
|
21259
|
+
facebookVideoTranscribe(input) {
|
|
21260
|
+
return this.call("/facebook/video-transcribe", input, this.httpTimeoutOverrideMs ?? 24e4);
|
|
21261
|
+
}
|
|
20613
21262
|
mapsPlaceIntel(input) {
|
|
20614
21263
|
return this.call("/maps/place", input);
|
|
20615
21264
|
}
|
|
@@ -20725,6 +21374,7 @@ async function migrateBrowserAgent() {
|
|
|
20725
21374
|
status TEXT NOT NULL DEFAULT 'open',
|
|
20726
21375
|
label TEXT,
|
|
20727
21376
|
user_id INTEGER,
|
|
21377
|
+
concurrency_lock_id TEXT,
|
|
20728
21378
|
created_at TEXT NOT NULL DEFAULT (datetime('now')),
|
|
20729
21379
|
closed_at TEXT,
|
|
20730
21380
|
last_action_at TEXT,
|
|
@@ -20734,6 +21384,11 @@ async function migrateBrowserAgent() {
|
|
|
20734
21384
|
`);
|
|
20735
21385
|
await db.execute(`CREATE INDEX IF NOT EXISTS browser_agent_sessions_status ON browser_agent_sessions(status)`);
|
|
20736
21386
|
await db.execute(`CREATE INDEX IF NOT EXISTS browser_agent_sessions_user ON browser_agent_sessions(user_id)`);
|
|
21387
|
+
try {
|
|
21388
|
+
await db.execute(`ALTER TABLE browser_agent_sessions ADD COLUMN concurrency_lock_id TEXT`);
|
|
21389
|
+
} catch {
|
|
21390
|
+
}
|
|
21391
|
+
await db.execute(`CREATE INDEX IF NOT EXISTS browser_agent_sessions_concurrency_lock ON browser_agent_sessions(concurrency_lock_id)`);
|
|
20737
21392
|
await db.execute(`
|
|
20738
21393
|
CREATE TABLE IF NOT EXISTS browser_agent_actions (
|
|
20739
21394
|
id TEXT PRIMARY KEY,
|
|
@@ -20761,11 +21416,11 @@ async function migrateBrowserAgent() {
|
|
|
20761
21416
|
}
|
|
20762
21417
|
async function createSessionRow(input) {
|
|
20763
21418
|
const db = getDb();
|
|
20764
|
-
const id = `bas_${(0,
|
|
21419
|
+
const id = `bas_${(0, import_node_crypto5.randomUUID)().replace(/-/g, "").slice(0, 20)}`;
|
|
20765
21420
|
await db.execute({
|
|
20766
|
-
sql: `INSERT INTO browser_agent_sessions (id, runtime_session_id, live_view_url, cdp_ws_url, status, label, user_id, last_action_at)
|
|
20767
|
-
VALUES (?, ?, ?, ?, 'open', ?, ?, datetime('now'))`,
|
|
20768
|
-
args: [id, input.runtimeSessionId, input.liveViewUrl, input.cdpWsUrl, input.label, input.userId]
|
|
21421
|
+
sql: `INSERT INTO browser_agent_sessions (id, runtime_session_id, live_view_url, cdp_ws_url, status, label, user_id, concurrency_lock_id, last_action_at)
|
|
21422
|
+
VALUES (?, ?, ?, ?, 'open', ?, ?, ?, datetime('now'))`,
|
|
21423
|
+
args: [id, input.runtimeSessionId, input.liveViewUrl, input.cdpWsUrl, input.label, input.userId, input.concurrencyLockId ?? null]
|
|
20769
21424
|
});
|
|
20770
21425
|
const row = await getSessionRow(id);
|
|
20771
21426
|
if (!row) throw new Error("session insert failed");
|
|
@@ -20822,7 +21477,7 @@ async function recordAction(input) {
|
|
|
20822
21477
|
sql: `INSERT INTO browser_agent_actions (id, session_id, type, params_json, ok, error)
|
|
20823
21478
|
VALUES (?, ?, ?, ?, ?, ?)`,
|
|
20824
21479
|
args: [
|
|
20825
|
-
`baa_${(0,
|
|
21480
|
+
`baa_${(0, import_node_crypto5.randomUUID)().replace(/-/g, "").slice(0, 20)}`,
|
|
20826
21481
|
input.sessionId,
|
|
20827
21482
|
input.type,
|
|
20828
21483
|
input.params == null ? null : JSON.stringify(input.params),
|
|
@@ -20862,11 +21517,11 @@ async function listReplayRows(sessionId) {
|
|
|
20862
21517
|
});
|
|
20863
21518
|
return res.rows;
|
|
20864
21519
|
}
|
|
20865
|
-
var
|
|
21520
|
+
var import_node_crypto5, _ready;
|
|
20866
21521
|
var init_browser_agent_db = __esm({
|
|
20867
21522
|
"src/api/browser-agent-db.ts"() {
|
|
20868
21523
|
"use strict";
|
|
20869
|
-
|
|
21524
|
+
import_node_crypto5 = require("crypto");
|
|
20870
21525
|
init_db();
|
|
20871
21526
|
_ready = false;
|
|
20872
21527
|
}
|
|
@@ -21186,6 +21841,12 @@ var init_browser_agent_service = __esm({
|
|
|
21186
21841
|
});
|
|
21187
21842
|
|
|
21188
21843
|
// src/api/browser-agent-routes.ts
|
|
21844
|
+
function browserSessionLockTtlSeconds(timeoutSeconds) {
|
|
21845
|
+
if (typeof timeoutSeconds === "number" && Number.isFinite(timeoutSeconds) && timeoutSeconds > 0) {
|
|
21846
|
+
return Math.max(30 * 60, Math.min(Math.trunc(timeoutSeconds) + 5 * 60, 24 * 60 * 60));
|
|
21847
|
+
}
|
|
21848
|
+
return DEFAULT_BROWSER_SESSION_LOCK_TTL_SECONDS;
|
|
21849
|
+
}
|
|
21189
21850
|
async function charge(sessionId, userId, startedAtMs) {
|
|
21190
21851
|
const elapsedMs = Date.now() - startedAtMs;
|
|
21191
21852
|
const { active_ms, billed_mc } = await addActiveMs(sessionId, elapsedMs);
|
|
@@ -21241,6 +21902,8 @@ async function loadOpenSession(id, userId) {
|
|
|
21241
21902
|
const row = await getSessionRow(id);
|
|
21242
21903
|
if (!row) return null;
|
|
21243
21904
|
if (row.user_id != null && row.user_id !== userId) return null;
|
|
21905
|
+
if (row.status !== "open") return null;
|
|
21906
|
+
await extendConcurrencyGate(row.concurrency_lock_id, DEFAULT_BROWSER_SESSION_LOCK_TTL_SECONDS);
|
|
21244
21907
|
return row;
|
|
21245
21908
|
}
|
|
21246
21909
|
function buildBrowserAgentRoutes() {
|
|
@@ -21256,23 +21919,39 @@ function buildBrowserAgentRoutes() {
|
|
|
21256
21919
|
return c.json(insufficientBalanceResponse(Number(user.balance_mc ?? 0), BROWSER_OPEN_MIN_BALANCE_MC), 402);
|
|
21257
21920
|
}
|
|
21258
21921
|
const body = await c.req.json().catch(() => ({}));
|
|
21922
|
+
const timeoutSeconds = typeof body.timeout_seconds === "number" ? body.timeout_seconds : void 0;
|
|
21923
|
+
const gate = await acquireConcurrencyGate(user, "browser_agent_session", {
|
|
21924
|
+
ttlSeconds: browserSessionLockTtlSeconds(timeoutSeconds),
|
|
21925
|
+
metadata: { label: typeof body.label === "string" ? body.label : null }
|
|
21926
|
+
});
|
|
21927
|
+
if (!gate.ok) return c.json(concurrencyLimitExceededResponse(gate), 429, { "Retry-After": String(gate.retryAfterSeconds) });
|
|
21928
|
+
let runtimeSessionId = null;
|
|
21259
21929
|
try {
|
|
21260
21930
|
const created = await createSession({
|
|
21261
|
-
timeoutSeconds
|
|
21931
|
+
timeoutSeconds,
|
|
21262
21932
|
proxyId: typeof body.proxy_id === "string" ? body.proxy_id : void 0,
|
|
21263
21933
|
profileName: typeof body.profile === "string" ? body.profile : void 0,
|
|
21264
21934
|
disableDefaultProxy: body.disable_default_proxy === true,
|
|
21265
21935
|
viewport: body.viewport && typeof body.viewport === "object" ? body.viewport : void 0
|
|
21266
21936
|
});
|
|
21937
|
+
runtimeSessionId = created.runtimeSessionId;
|
|
21267
21938
|
const row = await createSessionRow({
|
|
21268
21939
|
runtimeSessionId: created.runtimeSessionId,
|
|
21269
21940
|
liveViewUrl: created.liveViewUrl,
|
|
21270
21941
|
cdpWsUrl: created.cdpWsUrl,
|
|
21271
21942
|
label: typeof body.label === "string" ? body.label : null,
|
|
21272
|
-
userId: user.id
|
|
21943
|
+
userId: user.id,
|
|
21944
|
+
concurrencyLockId: gate.lockId
|
|
21273
21945
|
});
|
|
21274
21946
|
return c.json({ ...publicSession(row), watch_url: `/console/${row.id}` });
|
|
21275
21947
|
} catch (err) {
|
|
21948
|
+
if (runtimeSessionId) {
|
|
21949
|
+
try {
|
|
21950
|
+
await closeSession(runtimeSessionId);
|
|
21951
|
+
} catch {
|
|
21952
|
+
}
|
|
21953
|
+
}
|
|
21954
|
+
await releaseConcurrencyGate(gate.lockId);
|
|
21276
21955
|
return c.json(failure(err), 502);
|
|
21277
21956
|
}
|
|
21278
21957
|
});
|
|
@@ -21303,6 +21982,7 @@ function buildBrowserAgentRoutes() {
|
|
|
21303
21982
|
} catch {
|
|
21304
21983
|
}
|
|
21305
21984
|
await markSessionClosed(row.id);
|
|
21985
|
+
await releaseConcurrencyGate(row.concurrency_lock_id);
|
|
21306
21986
|
return c.json({ ok: true });
|
|
21307
21987
|
});
|
|
21308
21988
|
app2.post("/sessions/:id/goto", async (c) => {
|
|
@@ -21560,7 +22240,7 @@ function buildBrowserAgentRoutes() {
|
|
|
21560
22240
|
});
|
|
21561
22241
|
return app2;
|
|
21562
22242
|
}
|
|
21563
|
-
var import_hono10, auth;
|
|
22243
|
+
var import_hono10, auth, DEFAULT_BROWSER_SESSION_LOCK_TTL_SECONDS;
|
|
21564
22244
|
var init_browser_agent_routes = __esm({
|
|
21565
22245
|
"src/api/browser-agent-routes.ts"() {
|
|
21566
22246
|
"use strict";
|
|
@@ -21571,7 +22251,9 @@ var init_browser_agent_routes = __esm({
|
|
|
21571
22251
|
init_rates();
|
|
21572
22252
|
init_browser_agent_db();
|
|
21573
22253
|
init_browser_agent_service();
|
|
22254
|
+
init_concurrency_gates();
|
|
21574
22255
|
auth = createApiKeyAuth();
|
|
22256
|
+
DEFAULT_BROWSER_SESSION_LOCK_TTL_SECONDS = 2 * 60 * 60;
|
|
21575
22257
|
}
|
|
21576
22258
|
});
|
|
21577
22259
|
|
|
@@ -22073,14 +22755,14 @@ function getSessionSecret() {
|
|
|
22073
22755
|
function safeEqualHex(a, b) {
|
|
22074
22756
|
if (a.length !== b.length) return false;
|
|
22075
22757
|
try {
|
|
22076
|
-
return (0,
|
|
22758
|
+
return (0, import_node_crypto6.timingSafeEqual)(Buffer.from(a, "hex"), Buffer.from(b, "hex"));
|
|
22077
22759
|
} catch {
|
|
22078
22760
|
return false;
|
|
22079
22761
|
}
|
|
22080
22762
|
}
|
|
22081
22763
|
function signSession(userId) {
|
|
22082
22764
|
const payload = String(userId);
|
|
22083
|
-
const sig = (0,
|
|
22765
|
+
const sig = (0, import_node_crypto6.createHmac)("sha256", secret()).update(payload).digest("hex");
|
|
22084
22766
|
return `${payload}.${sig}`;
|
|
22085
22767
|
}
|
|
22086
22768
|
function verifySession(token) {
|
|
@@ -22088,16 +22770,16 @@ function verifySession(token) {
|
|
|
22088
22770
|
if (dot === -1) return null;
|
|
22089
22771
|
const payload = token.slice(0, dot);
|
|
22090
22772
|
const sig = token.slice(dot + 1);
|
|
22091
|
-
const expected = (0,
|
|
22773
|
+
const expected = (0, import_node_crypto6.createHmac)("sha256", secret()).update(payload).digest("hex");
|
|
22092
22774
|
if (!safeEqualHex(sig, expected)) return null;
|
|
22093
22775
|
const id = parseInt(payload);
|
|
22094
22776
|
return isNaN(id) ? null : id;
|
|
22095
22777
|
}
|
|
22096
|
-
var
|
|
22778
|
+
var import_node_crypto6, isProduction, secret;
|
|
22097
22779
|
var init_session = __esm({
|
|
22098
22780
|
"src/api/session.ts"() {
|
|
22099
22781
|
"use strict";
|
|
22100
|
-
|
|
22782
|
+
import_node_crypto6 = require("crypto");
|
|
22101
22783
|
isProduction = () => process.env.NODE_ENV === "production" || process.env.VERCEL === "1";
|
|
22102
22784
|
secret = () => getSessionSecret();
|
|
22103
22785
|
}
|
|
@@ -22306,14 +22988,23 @@ function countPaaQuestions2(result) {
|
|
|
22306
22988
|
function paaCostForQuestionCount2(questionCount) {
|
|
22307
22989
|
return Math.max(1, questionCount) * MC_COSTS.paa;
|
|
22308
22990
|
}
|
|
22309
|
-
async function checkHarvestLimits(
|
|
22310
|
-
if (
|
|
22311
|
-
|
|
22312
|
-
const
|
|
22313
|
-
|
|
22991
|
+
async function checkHarvestLimits(user, reuseLockId) {
|
|
22992
|
+
if (reuseLockId && await reuseExistingConcurrencyGate(user.id, reuseLockId, 30 * 60)) return null;
|
|
22993
|
+
if (isConcurrencyLimitBypassed(user)) return null;
|
|
22994
|
+
const limit = concurrencyLimitForUser(user);
|
|
22995
|
+
const active = await countActiveUsageForUser(user.id);
|
|
22996
|
+
if (active >= limit) {
|
|
22997
|
+
return concurrencyLimitExceededResponse({
|
|
22998
|
+
ok: false,
|
|
22999
|
+
active,
|
|
23000
|
+
limit,
|
|
23001
|
+
operation: "harvest",
|
|
23002
|
+
retryAfterSeconds: 30
|
|
23003
|
+
});
|
|
23004
|
+
}
|
|
22314
23005
|
return null;
|
|
22315
23006
|
}
|
|
22316
|
-
var import_resend, import_hono12, import_hono13, import_factory6, import_cookie, import_stripe2, secureCookies, isProduction2, sessionCookieOptions, requireAllowedOrigin, auth2, adminAuth, sessionAuth, app, STRIPE_API_VERSION,
|
|
23007
|
+
var import_resend, import_hono12, import_hono13, import_factory6, import_cookie, import_stripe2, secureCookies, isProduction2, sessionCookieOptions, requireAllowedOrigin, auth2, adminAuth, sessionAuth, app, STRIPE_API_VERSION, SYNC_HARVEST_TIMEOUT_OVERRIDE_MS;
|
|
22317
23008
|
var init_server = __esm({
|
|
22318
23009
|
"src/api/server.ts"() {
|
|
22319
23010
|
"use strict";
|
|
@@ -22359,6 +23050,7 @@ var init_server = __esm({
|
|
|
22359
23050
|
init_harvest_problems();
|
|
22360
23051
|
init_harvest_attempt_events();
|
|
22361
23052
|
init_session();
|
|
23053
|
+
init_concurrency_gates();
|
|
22362
23054
|
secureCookies = process.env.NODE_ENV === "production" || process.env.VERCEL === "1";
|
|
22363
23055
|
isProduction2 = secureCookies;
|
|
22364
23056
|
sessionCookieOptions = {
|
|
@@ -22555,9 +23247,6 @@ var init_server = __esm({
|
|
|
22555
23247
|
await revokeApiKey(user.id);
|
|
22556
23248
|
return c.json({ ok: true });
|
|
22557
23249
|
});
|
|
22558
|
-
BYPASS_EMAILS = new Set(
|
|
22559
|
-
(process.env.HARVEST_LIMIT_BYPASS_EMAILS ?? "").split(",").map((e) => e.trim()).filter(Boolean)
|
|
22560
|
-
);
|
|
22561
23250
|
SYNC_HARVEST_TIMEOUT_OVERRIDE_MS = (() => {
|
|
22562
23251
|
const raw = process.env.SYNC_HARVEST_TIMEOUT_MS;
|
|
22563
23252
|
const parsed = raw === void 0 ? NaN : Number(raw);
|
|
@@ -22575,8 +23264,8 @@ var init_server = __esm({
|
|
|
22575
23264
|
if (checked.error) return c.json({ error: checked.error }, 400);
|
|
22576
23265
|
body.callback_url = checked.parsed?.href;
|
|
22577
23266
|
}
|
|
22578
|
-
const limitErr = await checkHarvestLimits(user
|
|
22579
|
-
if (limitErr) return c.json(limitErr, 429);
|
|
23267
|
+
const limitErr = await checkHarvestLimits(user, c.req.header("x-mcp-scraper-concurrency-lock"));
|
|
23268
|
+
if (limitErr) return c.json(limitErr, 429, { "Retry-After": "30" });
|
|
22580
23269
|
const options = {
|
|
22581
23270
|
query: body.query.trim(),
|
|
22582
23271
|
location: body.location,
|
|
@@ -22611,8 +23300,8 @@ var init_server = __esm({
|
|
|
22611
23300
|
const bodyResult = HarvestBodySchema.safeParse(raw);
|
|
22612
23301
|
if (!bodyResult.success) return c.json({ error: bodyResult.error.issues[0]?.message ?? "Invalid request" }, 400);
|
|
22613
23302
|
const body = bodyResult.data;
|
|
22614
|
-
const limitErr = await checkHarvestLimits(user
|
|
22615
|
-
if (limitErr) return c.json(limitErr, 429);
|
|
23303
|
+
const limitErr = await checkHarvestLimits(user, c.req.header("x-mcp-scraper-concurrency-lock"));
|
|
23304
|
+
if (limitErr) return c.json(limitErr, 429, { "Retry-After": "30" });
|
|
22616
23305
|
const options = {
|
|
22617
23306
|
query: body.query.trim(),
|
|
22618
23307
|
location: body.location,
|
|
@@ -22771,9 +23460,16 @@ var init_server = __esm({
|
|
|
22771
23460
|
}
|
|
22772
23461
|
})();
|
|
22773
23462
|
const user = c.get("user");
|
|
22774
|
-
const
|
|
22775
|
-
|
|
23463
|
+
const gate = await acquireConcurrencyGate(user, "extract_url", {
|
|
23464
|
+
reuseLockId: c.req.header("x-mcp-scraper-concurrency-lock"),
|
|
23465
|
+
metadata: { url: canonicalUrl }
|
|
23466
|
+
});
|
|
23467
|
+
if (!gate.ok) return c.json(concurrencyLimitExceededResponse(gate), 429, { "Retry-After": String(gate.retryAfterSeconds) });
|
|
23468
|
+
let debited = false;
|
|
22776
23469
|
try {
|
|
23470
|
+
const { ok: euOk, balance_mc: euBal } = await debitMc(user.id, MC_COSTS.page_scrape, LedgerOperation.EXTRACT_URL, new URL(canonicalUrl).hostname);
|
|
23471
|
+
if (!euOk) return c.json(insufficientBalanceResponse(euBal, MC_COSTS.page_scrape), 402);
|
|
23472
|
+
debited = true;
|
|
22777
23473
|
const kernelApiKey = browserServiceApiKey();
|
|
22778
23474
|
const device = screenshotDevice === "mobile" ? "mobile" : "desktop";
|
|
22779
23475
|
const [result, pageData] = await Promise.all([
|
|
@@ -22791,9 +23487,11 @@ var init_server = __esm({
|
|
|
22791
23487
|
return c.json({ ...result, screenshot: screenshotMeta, branding: brandingData, media: mediaMeta });
|
|
22792
23488
|
} catch (err) {
|
|
22793
23489
|
const msg = err instanceof Error ? err.message : String(err);
|
|
22794
|
-
await creditMc(user.id, MC_COSTS.page_scrape, LedgerOperation.EXTRACT_URL, "failed call");
|
|
23490
|
+
if (debited) await creditMc(user.id, MC_COSTS.page_scrape, LedgerOperation.EXTRACT_URL, "failed call");
|
|
22795
23491
|
await logRequestEvent({ userId: user.id, source: "extract_url", status: "failed", query: canonicalUrl, error: msg });
|
|
22796
23492
|
return c.json({ error: msg }, 500);
|
|
23493
|
+
} finally {
|
|
23494
|
+
await releaseConcurrencyGate(gate.lockId);
|
|
22797
23495
|
}
|
|
22798
23496
|
});
|
|
22799
23497
|
app.post("/map-urls", auth2, async (c) => {
|
|
@@ -22805,9 +23503,16 @@ var init_server = __esm({
|
|
|
22805
23503
|
if (checked.error || !checked.parsed) return c.json({ error: checked.error ?? "Invalid URL" }, 400);
|
|
22806
23504
|
const parsed = checked.parsed;
|
|
22807
23505
|
const user = c.get("user");
|
|
22808
|
-
const
|
|
22809
|
-
|
|
23506
|
+
const gate = await acquireConcurrencyGate(user, "map_urls", {
|
|
23507
|
+
reuseLockId: c.req.header("x-mcp-scraper-concurrency-lock"),
|
|
23508
|
+
metadata: { url: parsed.href }
|
|
23509
|
+
});
|
|
23510
|
+
if (!gate.ok) return c.json(concurrencyLimitExceededResponse(gate), 429, { "Retry-After": String(gate.retryAfterSeconds) });
|
|
23511
|
+
let debited = false;
|
|
22810
23512
|
try {
|
|
23513
|
+
const { ok: mapOk, balance_mc: mapBal } = await debitMc(user.id, MC_COSTS.url_map, LedgerOperation.URL_MAP, parsed.hostname);
|
|
23514
|
+
if (!mapOk) return c.json(insufficientBalanceResponse(mapBal, MC_COSTS.url_map), 402);
|
|
23515
|
+
debited = true;
|
|
22811
23516
|
const result = await spiderSite({
|
|
22812
23517
|
startUrl: parsed.href,
|
|
22813
23518
|
maxUrls: Math.min(2e3, Math.max(1, body.maxUrls ?? 500)),
|
|
@@ -22825,7 +23530,7 @@ var init_server = __esm({
|
|
|
22825
23530
|
return c.json(result);
|
|
22826
23531
|
} catch (err) {
|
|
22827
23532
|
const msg = err instanceof Error ? err.message : String(err);
|
|
22828
|
-
await creditMc(user.id, MC_COSTS.url_map, LedgerOperation.URL_MAP_REFUND, "failed call");
|
|
23533
|
+
if (debited) await creditMc(user.id, MC_COSTS.url_map, LedgerOperation.URL_MAP_REFUND, "failed call");
|
|
22829
23534
|
await logRequestEvent({
|
|
22830
23535
|
userId: user.id,
|
|
22831
23536
|
source: "map_urls",
|
|
@@ -22834,6 +23539,8 @@ var init_server = __esm({
|
|
|
22834
23539
|
error: msg
|
|
22835
23540
|
});
|
|
22836
23541
|
return c.json({ error: msg }, 500);
|
|
23542
|
+
} finally {
|
|
23543
|
+
await releaseConcurrencyGate(gate.lockId);
|
|
22837
23544
|
}
|
|
22838
23545
|
});
|
|
22839
23546
|
app.post("/extract-site", auth2, async (c) => {
|
|
@@ -22846,9 +23553,16 @@ var init_server = __esm({
|
|
|
22846
23553
|
const parsed = checked.parsed;
|
|
22847
23554
|
const user = c.get("user");
|
|
22848
23555
|
const siteHoldMc = MC_COSTS.page_scrape * 10;
|
|
22849
|
-
const
|
|
22850
|
-
|
|
23556
|
+
const gate = await acquireConcurrencyGate(user, "extract_site", {
|
|
23557
|
+
reuseLockId: c.req.header("x-mcp-scraper-concurrency-lock"),
|
|
23558
|
+
metadata: { url: parsed.href }
|
|
23559
|
+
});
|
|
23560
|
+
if (!gate.ok) return c.json(concurrencyLimitExceededResponse(gate), 429, { "Retry-After": String(gate.retryAfterSeconds) });
|
|
23561
|
+
let debited = false;
|
|
22851
23562
|
try {
|
|
23563
|
+
const { ok: siteOk, balance_mc: siteBal } = await debitMc(user.id, siteHoldMc, LedgerOperation.EXTRACT_SITE_HOLD, parsed.hostname);
|
|
23564
|
+
if (!siteOk) return c.json(insufficientBalanceResponse(siteBal, siteHoldMc), 402);
|
|
23565
|
+
debited = true;
|
|
22852
23566
|
const result = await extractSite({
|
|
22853
23567
|
startUrl: parsed.href,
|
|
22854
23568
|
maxPages: Math.min(200, Math.max(1, body.maxPages ?? 100)),
|
|
@@ -22869,7 +23583,7 @@ var init_server = __esm({
|
|
|
22869
23583
|
});
|
|
22870
23584
|
return c.json(result);
|
|
22871
23585
|
} catch (err) {
|
|
22872
|
-
await creditMc(user.id, siteHoldMc, LedgerOperation.EXTRACT_SITE_REFUND, "failed call");
|
|
23586
|
+
if (debited) await creditMc(user.id, siteHoldMc, LedgerOperation.EXTRACT_SITE_REFUND, "failed call");
|
|
22873
23587
|
const msg = err instanceof Error ? err.message : String(err);
|
|
22874
23588
|
await logRequestEvent({
|
|
22875
23589
|
userId: user.id,
|
|
@@ -22879,6 +23593,8 @@ var init_server = __esm({
|
|
|
22879
23593
|
error: msg
|
|
22880
23594
|
});
|
|
22881
23595
|
return c.json({ error: msg }, 500);
|
|
23596
|
+
} finally {
|
|
23597
|
+
await releaseConcurrencyGate(gate.lockId);
|
|
22882
23598
|
}
|
|
22883
23599
|
});
|
|
22884
23600
|
app.post("/billing/checkout", requireAllowedOrigin, sessionAuth, async (c) => {
|
|
@@ -22947,6 +23663,60 @@ var init_server = __esm({
|
|
|
22947
23663
|
return c.json({ error: message }, 500);
|
|
22948
23664
|
}
|
|
22949
23665
|
});
|
|
23666
|
+
app.post("/billing/concurrency/terminal-checkout", auth2, async (c) => {
|
|
23667
|
+
try {
|
|
23668
|
+
const user = c.get("user");
|
|
23669
|
+
const upgrade = concurrencySlotBillingInfo();
|
|
23670
|
+
if (user.concurrency_stripe_sub_id) {
|
|
23671
|
+
return c.json({
|
|
23672
|
+
error: "Already subscribed. Your account already has an active extra concurrency slot subscription.",
|
|
23673
|
+
error_code: "concurrency_subscription_exists",
|
|
23674
|
+
concurrency: {
|
|
23675
|
+
extra_slots: user.extra_concurrency_slots,
|
|
23676
|
+
limit: concurrencyLimitForUser(user),
|
|
23677
|
+
has_subscription: true,
|
|
23678
|
+
upgrade
|
|
23679
|
+
}
|
|
23680
|
+
}, 409);
|
|
23681
|
+
}
|
|
23682
|
+
const secret2 = process.env.STRIPE_SECRET_KEY?.trim();
|
|
23683
|
+
if (!secret2) return c.json({ error: "Stripe is not configured." }, 503);
|
|
23684
|
+
const stripeClient = new import_stripe2.default(secret2, { apiVersion: STRIPE_API_VERSION });
|
|
23685
|
+
let customerId = user.stripe_customer_id;
|
|
23686
|
+
if (!customerId) {
|
|
23687
|
+
const customer = await stripeClient.customers.create({ email: user.email });
|
|
23688
|
+
customerId = customer.id;
|
|
23689
|
+
await setStripeCustomerId(user.id, customerId);
|
|
23690
|
+
}
|
|
23691
|
+
const session = await stripeClient.checkout.sessions.create({
|
|
23692
|
+
customer: customerId,
|
|
23693
|
+
mode: "subscription",
|
|
23694
|
+
line_items: [{ price: CONCURRENCY_PRICE_ID, quantity: 1 }],
|
|
23695
|
+
automatic_tax: { enabled: true },
|
|
23696
|
+
billing_address_collection: "required",
|
|
23697
|
+
customer_update: { address: "auto", name: "auto" },
|
|
23698
|
+
tax_id_collection: { enabled: true },
|
|
23699
|
+
success_url: `${appOrigin()}/billing?slot_added=1&source=terminal`,
|
|
23700
|
+
cancel_url: `${appOrigin()}/billing?slot_cancelled=1&source=terminal`
|
|
23701
|
+
});
|
|
23702
|
+
if (!session.url) return c.json({ error: "Stripe did not return a checkout URL." }, 502);
|
|
23703
|
+
return c.json({
|
|
23704
|
+
checkout_url: session.url,
|
|
23705
|
+
price: upgrade,
|
|
23706
|
+
concurrency: {
|
|
23707
|
+
current_extra_slots: user.extra_concurrency_slots,
|
|
23708
|
+
current_limit: concurrencyLimitForUser(user),
|
|
23709
|
+
after_checkout_extra_slots: user.extra_concurrency_slots + 1,
|
|
23710
|
+
after_checkout_limit: concurrencyLimitForUser({ extra_concurrency_slots: user.extra_concurrency_slots + 1 })
|
|
23711
|
+
},
|
|
23712
|
+
next_step: "Open checkout_url in a browser, complete checkout, then restart or retry the MCP request."
|
|
23713
|
+
});
|
|
23714
|
+
} catch (err) {
|
|
23715
|
+
const message = err instanceof Error ? err.message : "Unable to start terminal checkout.";
|
|
23716
|
+
console.error("[billing/concurrency/terminal-checkout]", message);
|
|
23717
|
+
return c.json({ error: message }, 500);
|
|
23718
|
+
}
|
|
23719
|
+
});
|
|
22950
23720
|
app.post("/billing/concurrency/cancel", requireAllowedOrigin, sessionAuth, async (c) => {
|
|
22951
23721
|
const user = c.get("sessionUser");
|
|
22952
23722
|
if (!user.concurrency_stripe_sub_id) return c.json({ error: "No active concurrency subscription." }, 404);
|
|
@@ -22954,9 +23724,10 @@ var init_server = __esm({
|
|
|
22954
23724
|
if (!stripeSecret) return c.json({ error: "Stripe is not configured." }, 503);
|
|
22955
23725
|
const stripeClient = new import_stripe2.default(stripeSecret, { apiVersion: STRIPE_API_VERSION });
|
|
22956
23726
|
await stripeClient.subscriptions.cancel(user.concurrency_stripe_sub_id);
|
|
22957
|
-
|
|
23727
|
+
const nextSlots = Math.max(0, user.extra_concurrency_slots - 1);
|
|
23728
|
+
await setExtraConcurrencySlots(user.id, nextSlots);
|
|
22958
23729
|
await setConcurrencySubId(user.id, null);
|
|
22959
|
-
return c.json({ ok: true, concurrency_limit:
|
|
23730
|
+
return c.json({ ok: true, extra_concurrency_slots: nextSlots, concurrency_limit: 1 + nextSlots });
|
|
22960
23731
|
});
|
|
22961
23732
|
app.get("/billing/balance", auth2, async (c) => {
|
|
22962
23733
|
const user = c.get("user");
|
|
@@ -22991,6 +23762,12 @@ var init_server = __esm({
|
|
|
22991
23762
|
item: body.item ?? null,
|
|
22992
23763
|
matched_cost: matchedCost ? (({ aliases, ...cost }) => cost)(matchedCost) : null,
|
|
22993
23764
|
costs,
|
|
23765
|
+
concurrency: {
|
|
23766
|
+
current_extra_slots: user.extra_concurrency_slots,
|
|
23767
|
+
current_limit: concurrencyLimitForUser(user),
|
|
23768
|
+
has_subscription: !!user.concurrency_stripe_sub_id,
|
|
23769
|
+
upgrade: concurrencySlotBillingInfo()
|
|
23770
|
+
},
|
|
22994
23771
|
ledger
|
|
22995
23772
|
});
|
|
22996
23773
|
});
|