mcp-scraper 0.2.13 → 0.2.14
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +9 -0
- package/dist/bin/api-server.cjs +664 -178
- package/dist/bin/api-server.cjs.map +1 -1
- package/dist/bin/api-server.js +3 -3
- package/dist/bin/browser-agent-stdio-server.cjs +1 -1
- package/dist/bin/browser-agent-stdio-server.cjs.map +1 -1
- package/dist/bin/browser-agent-stdio-server.js +2 -2
- package/dist/bin/mcp-scraper-cli.cjs +46 -3
- package/dist/bin/mcp-scraper-cli.cjs.map +1 -1
- package/dist/bin/mcp-scraper-cli.js +42 -2
- package/dist/bin/mcp-scraper-cli.js.map +1 -1
- package/dist/bin/mcp-scraper-combined-stdio-server.cjs +47 -5
- package/dist/bin/mcp-scraper-combined-stdio-server.cjs.map +1 -1
- package/dist/bin/mcp-scraper-combined-stdio-server.js +3 -3
- package/dist/bin/mcp-scraper-install.cjs +2 -1
- package/dist/bin/mcp-scraper-install.cjs.map +1 -1
- package/dist/bin/mcp-scraper-install.js +2 -1
- package/dist/bin/mcp-scraper-install.js.map +1 -1
- package/dist/bin/mcp-stdio-server.cjs +47 -5
- package/dist/bin/mcp-stdio-server.cjs.map +1 -1
- package/dist/bin/mcp-stdio-server.js +2 -2
- package/dist/{chunk-ROS67BNV.js → chunk-BSYPATSM.js} +21 -2
- package/dist/chunk-BSYPATSM.js.map +1 -0
- package/dist/{chunk-AYTOCZBS.js → chunk-F44RBOJ5.js} +6 -3
- package/dist/chunk-F44RBOJ5.js.map +1 -0
- package/dist/{chunk-DKJ2XCY7.js → chunk-IWQTNY2E.js} +48 -6
- package/dist/chunk-IWQTNY2E.js.map +1 -0
- package/dist/chunk-ROCXJOVY.js +7 -0
- package/dist/chunk-ROCXJOVY.js.map +1 -0
- package/dist/{chunk-7GCCOT3M.js → chunk-TL7YTFLH.js} +15 -1
- package/dist/chunk-TL7YTFLH.js.map +1 -0
- package/dist/{chunk-BQTWXY6G.js → chunk-ZV2XXYR7.js} +2 -2
- package/dist/{db-BVHYI57K.js → db-P5X6UQ3E.js} +2 -2
- package/dist/{server-LUNOI26E.js → server-7QP6HQJJ.js} +556 -164
- package/dist/server-7QP6HQJJ.js.map +1 -0
- package/dist/{worker-TDJQ6TH3.js → worker-OZSWIS3F.js} +3 -3
- package/package.json +1 -1
- package/dist/chunk-7GCCOT3M.js.map +0 -1
- package/dist/chunk-AYTOCZBS.js.map +0 -1
- package/dist/chunk-DKJ2XCY7.js.map +0 -1
- package/dist/chunk-RDROUQ4E.js +0 -7
- package/dist/chunk-RDROUQ4E.js.map +0 -1
- package/dist/chunk-ROS67BNV.js.map +0 -1
- package/dist/server-LUNOI26E.js.map +0 -1
- /package/dist/{chunk-BQTWXY6G.js.map → chunk-ZV2XXYR7.js.map} +0 -0
- /package/dist/{db-BVHYI57K.js.map → db-P5X6UQ3E.js.map} +0 -0
- /package/dist/{worker-TDJQ6TH3.js.map → worker-OZSWIS3F.js.map} +0 -0
package/dist/bin/api-server.cjs
CHANGED
|
@@ -7099,6 +7099,20 @@ async function migrate() {
|
|
|
7099
7099
|
await db.execute(`ALTER TABLE users ADD COLUMN concurrency_stripe_sub_id TEXT`);
|
|
7100
7100
|
} catch {
|
|
7101
7101
|
}
|
|
7102
|
+
await db.execute(`
|
|
7103
|
+
CREATE TABLE IF NOT EXISTS concurrency_locks (
|
|
7104
|
+
id TEXT PRIMARY KEY,
|
|
7105
|
+
user_id INTEGER NOT NULL REFERENCES users(id),
|
|
7106
|
+
operation TEXT NOT NULL,
|
|
7107
|
+
status TEXT NOT NULL DEFAULT 'active',
|
|
7108
|
+
acquired_at TEXT NOT NULL DEFAULT (datetime('now')),
|
|
7109
|
+
expires_at TEXT NOT NULL,
|
|
7110
|
+
released_at TEXT,
|
|
7111
|
+
metadata TEXT
|
|
7112
|
+
)
|
|
7113
|
+
`);
|
|
7114
|
+
await db.execute(`CREATE INDEX IF NOT EXISTS concurrency_locks_user_status ON concurrency_locks(user_id, status, expires_at)`);
|
|
7115
|
+
await db.execute(`CREATE INDEX IF NOT EXISTS concurrency_locks_operation ON concurrency_locks(operation, status, expires_at)`);
|
|
7102
7116
|
await db.execute(`
|
|
7103
7117
|
CREATE TABLE IF NOT EXISTS ledger (
|
|
7104
7118
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
@@ -8672,6 +8686,19 @@ var init_site_audit_routes = __esm({
|
|
|
8672
8686
|
function browserActiveCostMc(activeMs) {
|
|
8673
8687
|
return Math.round(activeMs * MC_PER_BROWSER_MS);
|
|
8674
8688
|
}
|
|
8689
|
+
function concurrencySlotBillingInfo() {
|
|
8690
|
+
const billingUrl = process.env.CONCURRENCY_BILLING_URL ?? "https://mcpscraper.dev/billing";
|
|
8691
|
+
return {
|
|
8692
|
+
product: "Extra concurrency slot",
|
|
8693
|
+
price_label: CONCURRENCY_SLOT_PRICE_LABEL,
|
|
8694
|
+
unit_amount_usd: CONCURRENCY_SLOT_PRICE_USD,
|
|
8695
|
+
currency: CONCURRENCY_SLOT_PRICE_CURRENCY,
|
|
8696
|
+
interval: CONCURRENCY_SLOT_PRICE_INTERVAL,
|
|
8697
|
+
billing_url: billingUrl,
|
|
8698
|
+
terminal_command: CONCURRENCY_SLOT_TERMINAL_COMMAND,
|
|
8699
|
+
terminal_command_with_api_key_env: `MCP_SCRAPER_API_KEY=sk_live_your_key ${CONCURRENCY_SLOT_TERMINAL_COMMAND}`
|
|
8700
|
+
};
|
|
8701
|
+
}
|
|
8675
8702
|
function mcToCredits(mc) {
|
|
8676
8703
|
return mc / MC_PER_CREDIT;
|
|
8677
8704
|
}
|
|
@@ -8688,7 +8715,7 @@ function insufficientBalanceResponse(balanceMc, requiredMc) {
|
|
|
8688
8715
|
topup_url: topupUrl
|
|
8689
8716
|
};
|
|
8690
8717
|
}
|
|
8691
|
-
var MC_COSTS, MC_PER_BROWSER_MS, BROWSER_OPEN_MIN_BALANCE_MC, MC_PER_CREDIT, CREDIT_COST_CATALOG, CONCURRENCY_PRICE_ID, FREE_SIGNUP_MC, FREE_MONTHLY_REFRESH_MC, BALANCE_PRICE_IDS, BALANCE_PACK_LABELS, LedgerOperation;
|
|
8718
|
+
var MC_COSTS, MC_PER_BROWSER_MS, BROWSER_OPEN_MIN_BALANCE_MC, MC_PER_CREDIT, CREDIT_COST_CATALOG, CONCURRENCY_PRICE_ID, CONCURRENCY_SLOT_PRICE_USD, CONCURRENCY_SLOT_PRICE_CURRENCY, CONCURRENCY_SLOT_PRICE_INTERVAL, CONCURRENCY_SLOT_PRICE_LABEL, CONCURRENCY_SLOT_TERMINAL_COMMAND, FREE_SIGNUP_MC, FREE_MONTHLY_REFRESH_MC, BALANCE_PRICE_IDS, BALANCE_PACK_LABELS, LedgerOperation;
|
|
8692
8719
|
var init_rates = __esm({
|
|
8693
8720
|
"src/api/rates.ts"() {
|
|
8694
8721
|
"use strict";
|
|
@@ -8815,6 +8842,11 @@ var init_rates = __esm({
|
|
|
8815
8842
|
}
|
|
8816
8843
|
];
|
|
8817
8844
|
CONCURRENCY_PRICE_ID = "price_1Ta1NRS8aAcsk3TGwsRnYbix";
|
|
8845
|
+
CONCURRENCY_SLOT_PRICE_USD = 5;
|
|
8846
|
+
CONCURRENCY_SLOT_PRICE_CURRENCY = "usd";
|
|
8847
|
+
CONCURRENCY_SLOT_PRICE_INTERVAL = "month";
|
|
8848
|
+
CONCURRENCY_SLOT_PRICE_LABEL = "$5/month";
|
|
8849
|
+
CONCURRENCY_SLOT_TERMINAL_COMMAND = "npx -y -p mcp-scraper@latest mcp-scraper-cli billing concurrency checkout";
|
|
8818
8850
|
FREE_SIGNUP_MC = 5e5;
|
|
8819
8851
|
FREE_MONTHLY_REFRESH_MC = 25e4;
|
|
8820
8852
|
BALANCE_PRICE_IDS = {
|
|
@@ -10383,6 +10415,187 @@ var init_server_schemas = __esm({
|
|
|
10383
10415
|
}
|
|
10384
10416
|
});
|
|
10385
10417
|
|
|
10418
|
+
// src/api/concurrency-gates.ts
|
|
10419
|
+
function concurrencyLimitForUser(user) {
|
|
10420
|
+
return 1 + Math.max(0, Number(user.extra_concurrency_slots ?? 0));
|
|
10421
|
+
}
|
|
10422
|
+
function concurrencyLimitExceededResponse(gate) {
|
|
10423
|
+
const upgrade = concurrencySlotBillingInfo();
|
|
10424
|
+
const message = `You have ${gate.active} active operation${gate.active !== 1 ? "s" : ""}. Your account allows ${gate.limit} concurrent operation${gate.limit !== 1 ? "s" : ""}. Wait for one to finish, or add an extra concurrency slot for ${upgrade.price_label} at ${upgrade.billing_url}. Terminal upgrade: ${upgrade.terminal_command}`;
|
|
10425
|
+
return {
|
|
10426
|
+
error: message,
|
|
10427
|
+
message,
|
|
10428
|
+
error_code: "concurrency_limit_exceeded",
|
|
10429
|
+
error_type: "concurrency_limit",
|
|
10430
|
+
retryable: true,
|
|
10431
|
+
active: gate.active,
|
|
10432
|
+
limit: gate.limit,
|
|
10433
|
+
operation: gate.operation,
|
|
10434
|
+
upgrade,
|
|
10435
|
+
upgrade_url: upgrade.billing_url,
|
|
10436
|
+
upgrade_command: upgrade.terminal_command
|
|
10437
|
+
};
|
|
10438
|
+
}
|
|
10439
|
+
function isConcurrencyLimitBypassed(user) {
|
|
10440
|
+
const raw = process.env.CONCURRENCY_LIMIT_BYPASS_EMAILS ?? process.env.HARVEST_LIMIT_BYPASS_EMAILS ?? "";
|
|
10441
|
+
return raw.split(",").map((email) => email.trim()).filter(Boolean).includes(user.email);
|
|
10442
|
+
}
|
|
10443
|
+
function lockTtlModifier(ttlSeconds) {
|
|
10444
|
+
const ttl = Number.isFinite(ttlSeconds) ? Math.max(30, Math.min(Math.trunc(Number(ttlSeconds)), MAX_LOCK_TTL_SECONDS)) : DEFAULT_LOCK_TTL_SECONDS;
|
|
10445
|
+
return `+${ttl} seconds`;
|
|
10446
|
+
}
|
|
10447
|
+
function safeMetadata(metadata) {
|
|
10448
|
+
if (!metadata) return null;
|
|
10449
|
+
try {
|
|
10450
|
+
return JSON.stringify(metadata).slice(0, 4e3);
|
|
10451
|
+
} catch {
|
|
10452
|
+
return null;
|
|
10453
|
+
}
|
|
10454
|
+
}
|
|
10455
|
+
async function ensureConcurrencyGateSchema() {
|
|
10456
|
+
if (!schemaReady) {
|
|
10457
|
+
schemaReady = (async () => {
|
|
10458
|
+
const db = getDb();
|
|
10459
|
+
await db.execute(`
|
|
10460
|
+
CREATE TABLE IF NOT EXISTS concurrency_locks (
|
|
10461
|
+
id TEXT PRIMARY KEY,
|
|
10462
|
+
user_id INTEGER NOT NULL REFERENCES users(id),
|
|
10463
|
+
operation TEXT NOT NULL,
|
|
10464
|
+
status TEXT NOT NULL DEFAULT 'active',
|
|
10465
|
+
acquired_at TEXT NOT NULL DEFAULT (datetime('now')),
|
|
10466
|
+
expires_at TEXT NOT NULL,
|
|
10467
|
+
released_at TEXT,
|
|
10468
|
+
metadata TEXT
|
|
10469
|
+
)
|
|
10470
|
+
`);
|
|
10471
|
+
await db.execute(`CREATE INDEX IF NOT EXISTS concurrency_locks_user_status ON concurrency_locks(user_id, status, expires_at)`);
|
|
10472
|
+
await db.execute(`CREATE INDEX IF NOT EXISTS concurrency_locks_operation ON concurrency_locks(operation, status, expires_at)`);
|
|
10473
|
+
})();
|
|
10474
|
+
}
|
|
10475
|
+
return schemaReady;
|
|
10476
|
+
}
|
|
10477
|
+
async function expireConcurrencyLocksForUser(userId) {
|
|
10478
|
+
await ensureConcurrencyGateSchema();
|
|
10479
|
+
await getDb().execute({
|
|
10480
|
+
sql: `UPDATE concurrency_locks
|
|
10481
|
+
SET status = 'expired', released_at = datetime('now')
|
|
10482
|
+
WHERE user_id = ? AND status = 'active' AND expires_at <= datetime('now')`,
|
|
10483
|
+
args: [userId]
|
|
10484
|
+
});
|
|
10485
|
+
}
|
|
10486
|
+
async function countActiveUsageForUser(userId) {
|
|
10487
|
+
await ensureConcurrencyGateSchema();
|
|
10488
|
+
await expireConcurrencyLocksForUser(userId);
|
|
10489
|
+
const res = await getDb().execute({
|
|
10490
|
+
sql: `SELECT
|
|
10491
|
+
(
|
|
10492
|
+
SELECT COUNT(*)
|
|
10493
|
+
FROM jobs
|
|
10494
|
+
WHERE user_id = ?
|
|
10495
|
+
AND (status = 'pending' OR (status = 'running' AND started_at > datetime('now', '-10 minutes')))
|
|
10496
|
+
) +
|
|
10497
|
+
(
|
|
10498
|
+
SELECT COUNT(*)
|
|
10499
|
+
FROM concurrency_locks
|
|
10500
|
+
WHERE user_id = ? AND status = 'active' AND expires_at > datetime('now')
|
|
10501
|
+
) AS n`,
|
|
10502
|
+
args: [userId, userId]
|
|
10503
|
+
});
|
|
10504
|
+
return Number(res.rows[0]?.n ?? 0);
|
|
10505
|
+
}
|
|
10506
|
+
async function reuseExistingConcurrencyGate(userId, lockId, ttlSeconds) {
|
|
10507
|
+
if (!/^cl_[a-f0-9]{24,32}$/.test(lockId)) return false;
|
|
10508
|
+
await ensureConcurrencyGateSchema();
|
|
10509
|
+
await expireConcurrencyLocksForUser(userId);
|
|
10510
|
+
const res = await getDb().execute({
|
|
10511
|
+
sql: `UPDATE concurrency_locks
|
|
10512
|
+
SET expires_at = datetime('now', ?)
|
|
10513
|
+
WHERE id = ?
|
|
10514
|
+
AND user_id = ?
|
|
10515
|
+
AND status = 'active'
|
|
10516
|
+
AND expires_at > datetime('now')`,
|
|
10517
|
+
args: [lockTtlModifier(ttlSeconds), lockId, userId]
|
|
10518
|
+
});
|
|
10519
|
+
return Number(res.rowsAffected ?? 0) > 0;
|
|
10520
|
+
}
|
|
10521
|
+
async function acquireConcurrencyGate(user, operation, options = {}) {
|
|
10522
|
+
await ensureConcurrencyGateSchema();
|
|
10523
|
+
const limit = concurrencyLimitForUser(user);
|
|
10524
|
+
if (isConcurrencyLimitBypassed(user)) {
|
|
10525
|
+
return { ok: true, lockId: null, active: 0, limit, operation, bypassed: true };
|
|
10526
|
+
}
|
|
10527
|
+
if (options.reuseLockId && await reuseExistingConcurrencyGate(user.id, options.reuseLockId, options.ttlSeconds)) {
|
|
10528
|
+
return { ok: true, lockId: null, active: await countActiveUsageForUser(user.id), limit, operation, reused: true };
|
|
10529
|
+
}
|
|
10530
|
+
await expireConcurrencyLocksForUser(user.id);
|
|
10531
|
+
const lockId = `cl_${(0, import_node_crypto2.randomUUID)().replace(/-/g, "").slice(0, 24)}`;
|
|
10532
|
+
const res = await getDb().execute({
|
|
10533
|
+
sql: `INSERT INTO concurrency_locks (id, user_id, operation, status, expires_at, metadata)
|
|
10534
|
+
SELECT ?, ?, ?, 'active', datetime('now', ?), ?
|
|
10535
|
+
WHERE (
|
|
10536
|
+
(
|
|
10537
|
+
SELECT COUNT(*)
|
|
10538
|
+
FROM jobs
|
|
10539
|
+
WHERE user_id = ?
|
|
10540
|
+
AND (status = 'pending' OR (status = 'running' AND started_at > datetime('now', '-10 minutes')))
|
|
10541
|
+
) +
|
|
10542
|
+
(
|
|
10543
|
+
SELECT COUNT(*)
|
|
10544
|
+
FROM concurrency_locks
|
|
10545
|
+
WHERE user_id = ? AND status = 'active' AND expires_at > datetime('now')
|
|
10546
|
+
)
|
|
10547
|
+
) < ?`,
|
|
10548
|
+
args: [
|
|
10549
|
+
lockId,
|
|
10550
|
+
user.id,
|
|
10551
|
+
operation,
|
|
10552
|
+
lockTtlModifier(options.ttlSeconds),
|
|
10553
|
+
safeMetadata(options.metadata),
|
|
10554
|
+
user.id,
|
|
10555
|
+
user.id,
|
|
10556
|
+
limit
|
|
10557
|
+
]
|
|
10558
|
+
});
|
|
10559
|
+
const active = await countActiveUsageForUser(user.id);
|
|
10560
|
+
if (Number(res.rowsAffected ?? 0) > 0) {
|
|
10561
|
+
return { ok: true, lockId, active, limit, operation };
|
|
10562
|
+
}
|
|
10563
|
+
return { ok: false, active, limit, operation, retryAfterSeconds: DEFAULT_RETRY_AFTER_SECONDS };
|
|
10564
|
+
}
|
|
10565
|
+
async function releaseConcurrencyGate(lockId) {
|
|
10566
|
+
if (!lockId) return;
|
|
10567
|
+
await ensureConcurrencyGateSchema();
|
|
10568
|
+
await getDb().execute({
|
|
10569
|
+
sql: `UPDATE concurrency_locks
|
|
10570
|
+
SET status = 'released', released_at = datetime('now')
|
|
10571
|
+
WHERE id = ? AND status = 'active'`,
|
|
10572
|
+
args: [lockId]
|
|
10573
|
+
});
|
|
10574
|
+
}
|
|
10575
|
+
async function extendConcurrencyGate(lockId, ttlSeconds = DEFAULT_LOCK_TTL_SECONDS) {
|
|
10576
|
+
if (!lockId) return;
|
|
10577
|
+
await ensureConcurrencyGateSchema();
|
|
10578
|
+
await getDb().execute({
|
|
10579
|
+
sql: `UPDATE concurrency_locks
|
|
10580
|
+
SET expires_at = datetime('now', ?)
|
|
10581
|
+
WHERE id = ? AND status = 'active' AND expires_at > datetime('now')`,
|
|
10582
|
+
args: [lockTtlModifier(ttlSeconds), lockId]
|
|
10583
|
+
});
|
|
10584
|
+
}
|
|
10585
|
+
var import_node_crypto2, DEFAULT_LOCK_TTL_SECONDS, DEFAULT_RETRY_AFTER_SECONDS, MAX_LOCK_TTL_SECONDS, schemaReady;
|
|
10586
|
+
var init_concurrency_gates = __esm({
|
|
10587
|
+
"src/api/concurrency-gates.ts"() {
|
|
10588
|
+
"use strict";
|
|
10589
|
+
import_node_crypto2 = require("crypto");
|
|
10590
|
+
init_db();
|
|
10591
|
+
init_rates();
|
|
10592
|
+
DEFAULT_LOCK_TTL_SECONDS = 15 * 60;
|
|
10593
|
+
DEFAULT_RETRY_AFTER_SECONDS = 30;
|
|
10594
|
+
MAX_LOCK_TTL_SECONDS = 24 * 60 * 60;
|
|
10595
|
+
schemaReady = null;
|
|
10596
|
+
}
|
|
10597
|
+
});
|
|
10598
|
+
|
|
10386
10599
|
// src/api/youtube-routes.ts
|
|
10387
10600
|
function buildTranscriptMarkdown(result) {
|
|
10388
10601
|
const lines = [];
|
|
@@ -10444,6 +10657,7 @@ var init_youtube_routes = __esm({
|
|
|
10444
10657
|
init_errors();
|
|
10445
10658
|
init_api_auth();
|
|
10446
10659
|
init_server_schemas();
|
|
10660
|
+
init_concurrency_gates();
|
|
10447
10661
|
youtubeApp = new import_hono2.Hono();
|
|
10448
10662
|
youtubeApp.post("/harvest", createApiKeyAuth(), async (c) => {
|
|
10449
10663
|
const raw = await c.req.json().catch(() => ({}));
|
|
@@ -10457,9 +10671,16 @@ var init_youtube_routes = __esm({
|
|
|
10457
10671
|
return c.json({ error: "channel mode requires channelHandle (@handle or UC... ID)" }, 400);
|
|
10458
10672
|
}
|
|
10459
10673
|
const user = c.get("user");
|
|
10460
|
-
const
|
|
10461
|
-
|
|
10674
|
+
const gate = await acquireConcurrencyGate(user, "youtube_harvest", {
|
|
10675
|
+
reuseLockId: c.req.header("x-mcp-scraper-concurrency-lock"),
|
|
10676
|
+
metadata: { mode, query, channelHandle }
|
|
10677
|
+
});
|
|
10678
|
+
if (!gate.ok) return c.json(concurrencyLimitExceededResponse(gate), 429, { "Retry-After": String(gate.retryAfterSeconds) });
|
|
10679
|
+
let debited = false;
|
|
10462
10680
|
try {
|
|
10681
|
+
const { ok: ytOk, balance_mc: ytBal } = await debitMc(user.id, MC_COSTS.yt_channel, LedgerOperation.YT_CHANNEL, query ?? channelHandle ?? "");
|
|
10682
|
+
if (!ytOk) return c.json(insufficientBalanceResponse(ytBal, MC_COSTS.yt_channel), 402);
|
|
10683
|
+
debited = true;
|
|
10463
10684
|
const normalizedChannelUrl = mode === "channel" ? buildYouTubeChannelVideosUrl(channelHandle.trim()) : void 0;
|
|
10464
10685
|
const result = await ytHarvest({
|
|
10465
10686
|
mode,
|
|
@@ -10484,13 +10705,15 @@ var init_youtube_routes = __esm({
|
|
|
10484
10705
|
return c.json({ error: msg }, 400);
|
|
10485
10706
|
}
|
|
10486
10707
|
if (err instanceof CaptchaError || msg.includes("CAPTCHA") || msg.includes("blocked")) {
|
|
10487
|
-
await creditMc(user.id, MC_COSTS.yt_channel, LedgerOperation.YT_CHANNEL_REFUND, "failed call");
|
|
10708
|
+
if (debited) await creditMc(user.id, MC_COSTS.yt_channel, LedgerOperation.YT_CHANNEL_REFUND, "failed call");
|
|
10488
10709
|
await logRequestEvent({ userId: user.id, source: "youtube_harvest", status: "failed", query: mode === "search" ? query?.trim() ?? "" : channelHandle?.trim() ?? "", error: msg });
|
|
10489
10710
|
return c.json({ error: msg }, 503);
|
|
10490
10711
|
}
|
|
10491
|
-
await creditMc(user.id, MC_COSTS.yt_channel, LedgerOperation.YT_CHANNEL_REFUND, "failed call");
|
|
10712
|
+
if (debited) await creditMc(user.id, MC_COSTS.yt_channel, LedgerOperation.YT_CHANNEL_REFUND, "failed call");
|
|
10492
10713
|
await logRequestEvent({ userId: user.id, source: "youtube_harvest", status: "failed", query: mode === "search" ? query?.trim() ?? "" : channelHandle?.trim() ?? "", error: msg });
|
|
10493
10714
|
return c.json({ error: msg }, 500);
|
|
10715
|
+
} finally {
|
|
10716
|
+
await releaseConcurrencyGate(gate.lockId);
|
|
10494
10717
|
}
|
|
10495
10718
|
});
|
|
10496
10719
|
youtubeApp.post("/transcribe", createApiKeyAuth(), async (c) => {
|
|
@@ -10502,9 +10725,16 @@ var init_youtube_routes = __esm({
|
|
|
10502
10725
|
const user = c.get("user");
|
|
10503
10726
|
const holdMins = 5;
|
|
10504
10727
|
const holdMc = MC_COSTS.yt_transcription * holdMins;
|
|
10505
|
-
const
|
|
10506
|
-
|
|
10728
|
+
const gate = await acquireConcurrencyGate(user, "youtube_transcribe", {
|
|
10729
|
+
reuseLockId: c.req.header("x-mcp-scraper-concurrency-lock"),
|
|
10730
|
+
metadata: { videoId: id }
|
|
10731
|
+
});
|
|
10732
|
+
if (!gate.ok) return c.json(concurrencyLimitExceededResponse(gate), 429, { "Retry-After": String(gate.retryAfterSeconds) });
|
|
10733
|
+
let debited = false;
|
|
10507
10734
|
try {
|
|
10735
|
+
const { ok: trOk, balance_mc: trBal } = await debitMc(user.id, holdMc, LedgerOperation.TRANSCRIPTION_HOLD, id);
|
|
10736
|
+
if (!trOk) return c.json(insufficientBalanceResponse(trBal, holdMc), 402);
|
|
10737
|
+
debited = true;
|
|
10508
10738
|
const result = await fetchCaptions(id);
|
|
10509
10739
|
const lastChunk = result.chunks?.[result.chunks.length - 1];
|
|
10510
10740
|
const chunkSecs = lastChunk && Number.isFinite(lastChunk.timestamp[1]) ? lastChunk.timestamp[1] : 0;
|
|
@@ -10529,10 +10759,12 @@ var init_youtube_routes = __esm({
|
|
|
10529
10759
|
html: buildTranscriptHtml(result)
|
|
10530
10760
|
});
|
|
10531
10761
|
} catch (err) {
|
|
10532
|
-
await creditMc(user.id, holdMc, LedgerOperation.TRANSCRIPTION_REFUND, "failed call");
|
|
10762
|
+
if (debited) await creditMc(user.id, holdMc, LedgerOperation.TRANSCRIPTION_REFUND, "failed call");
|
|
10533
10763
|
const msg = err instanceof Error ? err.message : String(err);
|
|
10534
10764
|
await logRequestEvent({ userId: user.id, source: "youtube_transcribe", status: "failed", query: id, error: msg });
|
|
10535
10765
|
return c.json({ error: msg }, 500);
|
|
10766
|
+
} finally {
|
|
10767
|
+
await releaseConcurrencyGate(gate.lockId);
|
|
10536
10768
|
}
|
|
10537
10769
|
});
|
|
10538
10770
|
}
|
|
@@ -10549,6 +10781,7 @@ var init_screenshot_routes = __esm({
|
|
|
10549
10781
|
init_screenshot();
|
|
10550
10782
|
init_api_auth();
|
|
10551
10783
|
init_url_utils();
|
|
10784
|
+
init_concurrency_gates();
|
|
10552
10785
|
ScreenshotBodySchema = import_zod14.z.object({
|
|
10553
10786
|
url: import_zod14.z.string().trim().min(1, "url is required"),
|
|
10554
10787
|
device: import_zod14.z.string().trim().optional(),
|
|
@@ -10564,6 +10797,7 @@ var init_screenshot_routes = __esm({
|
|
|
10564
10797
|
}
|
|
10565
10798
|
const body = parsed.data;
|
|
10566
10799
|
const urlCheck = await validatePublicHttpUrl(body.url, { field: "url", requireHttps: false });
|
|
10800
|
+
let targetUrl = urlCheck.parsed?.href;
|
|
10567
10801
|
if (urlCheck.error) {
|
|
10568
10802
|
if (!body.allowLocal) {
|
|
10569
10803
|
return c.json({ error_code: "invalid_request", message: urlCheck.error }, 400);
|
|
@@ -10577,25 +10811,17 @@ var init_screenshot_routes = __esm({
|
|
|
10577
10811
|
if (parsedFallback.protocol !== "http:" && parsedFallback.protocol !== "https:") {
|
|
10578
10812
|
return c.json({ error: "url must use http or https" }, 400);
|
|
10579
10813
|
}
|
|
10580
|
-
|
|
10581
|
-
try {
|
|
10582
|
-
const buf = await captureScreenshot(parsedFallback.href, browserServiceApiKey(), device2);
|
|
10583
|
-
return new Response(new Uint8Array(buf), {
|
|
10584
|
-
status: 200,
|
|
10585
|
-
headers: {
|
|
10586
|
-
"Content-Type": "image/png",
|
|
10587
|
-
"Content-Length": String(buf.length),
|
|
10588
|
-
"Cache-Control": "no-store"
|
|
10589
|
-
}
|
|
10590
|
-
});
|
|
10591
|
-
} catch (err) {
|
|
10592
|
-
const msg = err instanceof Error ? err.message : String(err);
|
|
10593
|
-
return c.json({ error: msg }, 502);
|
|
10594
|
-
}
|
|
10814
|
+
targetUrl = parsedFallback.href;
|
|
10595
10815
|
}
|
|
10596
10816
|
const device = body.device === "mobile" ? "mobile" : "desktop";
|
|
10817
|
+
const user = c.get("user");
|
|
10818
|
+
const gate = await acquireConcurrencyGate(user, "screenshot", {
|
|
10819
|
+
reuseLockId: c.req.header("x-mcp-scraper-concurrency-lock"),
|
|
10820
|
+
metadata: { url: targetUrl, device }
|
|
10821
|
+
});
|
|
10822
|
+
if (!gate.ok) return c.json(concurrencyLimitExceededResponse(gate), 429, { "Retry-After": String(gate.retryAfterSeconds) });
|
|
10597
10823
|
try {
|
|
10598
|
-
const buf = await captureScreenshot(
|
|
10824
|
+
const buf = await captureScreenshot(targetUrl, browserServiceApiKey(), device);
|
|
10599
10825
|
return new Response(new Uint8Array(buf), {
|
|
10600
10826
|
status: 200,
|
|
10601
10827
|
headers: {
|
|
@@ -10607,6 +10833,8 @@ var init_screenshot_routes = __esm({
|
|
|
10607
10833
|
} catch (err) {
|
|
10608
10834
|
const msg = err instanceof Error ? err.message : String(err);
|
|
10609
10835
|
return c.json({ error: msg }, 502);
|
|
10836
|
+
} finally {
|
|
10837
|
+
await releaseConcurrencyGate(gate.lockId);
|
|
10610
10838
|
}
|
|
10611
10839
|
});
|
|
10612
10840
|
}
|
|
@@ -11978,6 +12206,7 @@ var init_facebook_ad_routes = __esm({
|
|
|
11978
12206
|
import_client3 = require("@fal-ai/client");
|
|
11979
12207
|
init_api_auth();
|
|
11980
12208
|
init_url_utils();
|
|
12209
|
+
init_concurrency_gates();
|
|
11981
12210
|
FacebookAdBodySchema = import_zod15.z.object({
|
|
11982
12211
|
url: import_zod15.z.string().trim().optional(),
|
|
11983
12212
|
libraryId: import_zod15.z.string().trim().optional(),
|
|
@@ -12016,10 +12245,17 @@ var init_facebook_ad_routes = __esm({
|
|
|
12016
12245
|
const libraryId = FacebookAdExtractor.resolveLibraryId(raw2);
|
|
12017
12246
|
if (!libraryId) return c.json({ error: "Could not resolve a valid Facebook Ad Library ID from the provided input" }, 400);
|
|
12018
12247
|
const fbUser = c.get("user");
|
|
12019
|
-
const
|
|
12020
|
-
|
|
12248
|
+
const gate = await acquireConcurrencyGate(fbUser, "facebook_ad", {
|
|
12249
|
+
reuseLockId: c.req.header("x-mcp-scraper-concurrency-lock"),
|
|
12250
|
+
metadata: { libraryId }
|
|
12251
|
+
});
|
|
12252
|
+
if (!gate.ok) return c.json(concurrencyLimitExceededResponse(gate), 429, { "Retry-After": String(gate.retryAfterSeconds) });
|
|
12021
12253
|
const driver = new BrowserDriver();
|
|
12254
|
+
let debited = false;
|
|
12022
12255
|
try {
|
|
12256
|
+
const { ok: adOk, balance_mc: adBal } = await debitMc(fbUser.id, MC_COSTS.fb_ad, LedgerOperation.FB_AD, raw2);
|
|
12257
|
+
if (!adOk) return c.json(insufficientBalanceResponse(adBal, MC_COSTS.fb_ad), 402);
|
|
12258
|
+
debited = true;
|
|
12023
12259
|
await driver.launch(kernelLaunchOpts());
|
|
12024
12260
|
const extractor = new FacebookAdExtractor(driver);
|
|
12025
12261
|
const result = await extractor.extract(libraryId, { openModal: body.openModal !== false });
|
|
@@ -12033,7 +12269,7 @@ var init_facebook_ad_routes = __esm({
|
|
|
12033
12269
|
});
|
|
12034
12270
|
return c.json(result);
|
|
12035
12271
|
} catch (err) {
|
|
12036
|
-
await creditMc(fbUser.id, MC_COSTS.fb_ad, LedgerOperation.FB_AD_REFUND, "failed call");
|
|
12272
|
+
if (debited) await creditMc(fbUser.id, MC_COSTS.fb_ad, LedgerOperation.FB_AD_REFUND, "failed call");
|
|
12037
12273
|
const msg = err instanceof Error ? err.message : String(err);
|
|
12038
12274
|
await logRequestEvent({ userId: fbUser.id, source: "facebook_ad", status: "failed", query: raw2, error: msg });
|
|
12039
12275
|
if (msg.toLowerCase().includes("blocked") || msg.toLowerCase().includes("captcha")) {
|
|
@@ -12042,24 +12278,26 @@ var init_facebook_ad_routes = __esm({
|
|
|
12042
12278
|
return c.json({ error: msg }, 500);
|
|
12043
12279
|
} finally {
|
|
12044
12280
|
await driver.close();
|
|
12281
|
+
await releaseConcurrencyGate(gate.lockId);
|
|
12045
12282
|
}
|
|
12046
12283
|
});
|
|
12047
12284
|
facebookAdApp.post("/page-intel", createApiKeyAuth(), async (c) => {
|
|
12048
12285
|
const raw = await c.req.json().catch(() => ({}));
|
|
12049
12286
|
const parsed = FacebookPageIntelBodySchema.safeParse(raw);
|
|
12050
|
-
if (!parsed.success)
|
|
12051
|
-
return c.json(invalidRequest(parsed.error.issues[0]?.message ?? "Invalid request"), 400);
|
|
12052
|
-
}
|
|
12287
|
+
if (!parsed.success) return c.json(invalidRequest(parsed.error.issues[0]?.message ?? "Invalid request"), 400);
|
|
12053
12288
|
const body = parsed.data;
|
|
12054
12289
|
const maxAds = Math.min(200, Math.max(1, body.maxAds ?? 50));
|
|
12055
12290
|
const country = body.country?.trim().toUpperCase() ?? "US";
|
|
12056
12291
|
const listingUrl = buildPageIntelUrl(body, country);
|
|
12057
12292
|
const fbUser = c.get("user");
|
|
12058
|
-
const
|
|
12059
|
-
if (!
|
|
12293
|
+
const gate = await acquireConcurrencyGate(fbUser, "facebook_page_intel", { reuseLockId: c.req.header("x-mcp-scraper-concurrency-lock"), metadata: { pageId: body.pageId, query: body.query, libraryId: body.libraryId, country } });
|
|
12294
|
+
if (!gate.ok) return c.json(concurrencyLimitExceededResponse(gate), 429, { "Retry-After": String(gate.retryAfterSeconds) });
|
|
12060
12295
|
const driver = new BrowserDriver();
|
|
12061
|
-
let refunded = false;
|
|
12296
|
+
let refunded = false, debited = false;
|
|
12062
12297
|
try {
|
|
12298
|
+
const { ok: fbOk, balance_mc: fbBal } = await debitMc(fbUser.id, MC_COSTS.fb_ad, LedgerOperation.FB_AD, body.pageId ?? body.query ?? body.libraryId ?? "");
|
|
12299
|
+
if (!fbOk) return c.json(insufficientBalanceResponse(fbBal, MC_COSTS.fb_ad), 402);
|
|
12300
|
+
debited = true;
|
|
12063
12301
|
await driver.launch(await kernelLaunchOptsResidential());
|
|
12064
12302
|
await driver.navigateTo(listingUrl);
|
|
12065
12303
|
const extractor = new FacebookAdExtractor(driver);
|
|
@@ -12074,7 +12312,7 @@ var init_facebook_ad_routes = __esm({
|
|
|
12074
12312
|
return c.json(result);
|
|
12075
12313
|
} catch (err) {
|
|
12076
12314
|
const msg = err instanceof Error ? err.message : String(err);
|
|
12077
|
-
if (!refunded) await creditMc(fbUser.id, MC_COSTS.fb_ad, LedgerOperation.FB_AD_REFUND, "failed call");
|
|
12315
|
+
if (debited && !refunded) await creditMc(fbUser.id, MC_COSTS.fb_ad, LedgerOperation.FB_AD_REFUND, "failed call");
|
|
12078
12316
|
await logRequestEvent({ userId: fbUser.id, source: "facebook_page_intel", status: "failed", query: body.pageId ?? body.query ?? body.libraryId ?? "", error: msg });
|
|
12079
12317
|
if (msg.toLowerCase().includes("blocked") || msg.toLowerCase().includes("captcha")) {
|
|
12080
12318
|
return c.json({ error: msg }, 503);
|
|
@@ -12082,6 +12320,7 @@ var init_facebook_ad_routes = __esm({
|
|
|
12082
12320
|
return c.json({ error: msg }, 500);
|
|
12083
12321
|
} finally {
|
|
12084
12322
|
await driver.close();
|
|
12323
|
+
await releaseConcurrencyGate(gate.lockId);
|
|
12085
12324
|
}
|
|
12086
12325
|
});
|
|
12087
12326
|
facebookAdApp.post("/transcribe", createApiKeyAuth(), async (c) => {
|
|
@@ -12097,10 +12336,17 @@ var init_facebook_ad_routes = __esm({
|
|
|
12097
12336
|
}
|
|
12098
12337
|
const videoUrl = urlCheck.parsed.href;
|
|
12099
12338
|
const fbUser = c.get("user");
|
|
12100
|
-
const
|
|
12101
|
-
|
|
12339
|
+
const gate = await acquireConcurrencyGate(fbUser, "facebook_transcribe", {
|
|
12340
|
+
reuseLockId: c.req.header("x-mcp-scraper-concurrency-lock"),
|
|
12341
|
+
metadata: { videoUrl }
|
|
12342
|
+
});
|
|
12343
|
+
if (!gate.ok) return c.json(concurrencyLimitExceededResponse(gate), 429, { "Retry-After": String(gate.retryAfterSeconds) });
|
|
12102
12344
|
import_client3.fal.config({ credentials: process.env.FAL_KEY });
|
|
12345
|
+
let debited = false;
|
|
12103
12346
|
try {
|
|
12347
|
+
const { ok, balance_mc } = await debitMc(fbUser.id, MC_COSTS.fb_transcribe, LedgerOperation.FB_TRANSCRIBE, videoUrl);
|
|
12348
|
+
if (!ok) return c.json(insufficientBalanceResponse(balance_mc, MC_COSTS.fb_transcribe), 402);
|
|
12349
|
+
debited = true;
|
|
12104
12350
|
const startMs = Date.now();
|
|
12105
12351
|
const result = await import_client3.fal.subscribe("fal-ai/wizper", {
|
|
12106
12352
|
input: { audio_url: videoUrl, task: "transcribe", language: "en" },
|
|
@@ -12123,9 +12369,11 @@ var init_facebook_ad_routes = __esm({
|
|
|
12123
12369
|
return c.json({ text, chunks, durationMs, markdown: lines.join("\n") });
|
|
12124
12370
|
} catch (err) {
|
|
12125
12371
|
const msg = err instanceof Error ? err.message : String(err);
|
|
12126
|
-
await creditMc(fbUser.id, MC_COSTS.fb_transcribe, LedgerOperation.FB_TRANSCRIBE_REFUND, "failed call");
|
|
12372
|
+
if (debited) await creditMc(fbUser.id, MC_COSTS.fb_transcribe, LedgerOperation.FB_TRANSCRIBE_REFUND, "failed call");
|
|
12127
12373
|
await logRequestEvent({ userId: fbUser.id, source: "facebook_transcribe", status: "failed", query: videoUrl, error: msg });
|
|
12128
12374
|
return c.json({ error: msg }, 500);
|
|
12375
|
+
} finally {
|
|
12376
|
+
await releaseConcurrencyGate(gate.lockId);
|
|
12129
12377
|
}
|
|
12130
12378
|
});
|
|
12131
12379
|
facebookAdApp.post("/search", createApiKeyAuth(), async (c) => {
|
|
@@ -12139,11 +12387,18 @@ var init_facebook_ad_routes = __esm({
|
|
|
12139
12387
|
const maxResults = Math.min(20, Math.max(1, body.maxResults ?? 10));
|
|
12140
12388
|
const searchUrl = `https://www.facebook.com/ads/library/?active_status=all&ad_type=all&country=${country}&q=${encodeURIComponent(body.query.trim())}&search_type=keyword_unordered`;
|
|
12141
12389
|
const fbUser = c.get("user");
|
|
12142
|
-
const
|
|
12143
|
-
|
|
12390
|
+
const gate = await acquireConcurrencyGate(fbUser, "facebook_search", {
|
|
12391
|
+
reuseLockId: c.req.header("x-mcp-scraper-concurrency-lock"),
|
|
12392
|
+
metadata: { query: body.query.trim(), country }
|
|
12393
|
+
});
|
|
12394
|
+
if (!gate.ok) return c.json(concurrencyLimitExceededResponse(gate), 429, { "Retry-After": String(gate.retryAfterSeconds) });
|
|
12144
12395
|
const driver = new BrowserDriver();
|
|
12145
12396
|
let searchRefunded = false;
|
|
12397
|
+
let debited = false;
|
|
12146
12398
|
try {
|
|
12399
|
+
const { ok, balance_mc } = await debitMc(fbUser.id, MC_COSTS.fb_search, LedgerOperation.FB_SEARCH, body.query.trim());
|
|
12400
|
+
if (!ok) return c.json(insufficientBalanceResponse(balance_mc, MC_COSTS.fb_search), 402);
|
|
12401
|
+
debited = true;
|
|
12147
12402
|
await driver.launch(await kernelLaunchOptsResidential());
|
|
12148
12403
|
const page = driver.getPage();
|
|
12149
12404
|
const collated = await collectAdLibraryResults(page, searchUrl, Math.max(maxResults * 4, 40));
|
|
@@ -12204,7 +12459,7 @@ var init_facebook_ad_routes = __esm({
|
|
|
12204
12459
|
return c.json(searchResult);
|
|
12205
12460
|
} catch (err) {
|
|
12206
12461
|
const msg = err instanceof Error ? err.message : String(err);
|
|
12207
|
-
if (!searchRefunded) await creditMc(fbUser.id, MC_COSTS.fb_search, LedgerOperation.FB_SEARCH_REFUND, "failed call");
|
|
12462
|
+
if (debited && !searchRefunded) await creditMc(fbUser.id, MC_COSTS.fb_search, LedgerOperation.FB_SEARCH_REFUND, "failed call");
|
|
12208
12463
|
await logRequestEvent({ userId: fbUser.id, source: "facebook_search", status: "failed", query: body.query.trim(), error: msg });
|
|
12209
12464
|
if (msg.toLowerCase().includes("blocked") || msg.toLowerCase().includes("captcha")) {
|
|
12210
12465
|
return c.json({ error: msg }, 503);
|
|
@@ -12212,6 +12467,7 @@ var init_facebook_ad_routes = __esm({
|
|
|
12212
12467
|
return c.json({ error: msg }, 500);
|
|
12213
12468
|
} finally {
|
|
12214
12469
|
await driver.close();
|
|
12470
|
+
await releaseConcurrencyGate(gate.lockId);
|
|
12215
12471
|
}
|
|
12216
12472
|
});
|
|
12217
12473
|
ALLOWED_MEDIA_HOSTS = ["fbcdn.net", "cdninstagram.com", "scontent.facebook.com", "scontent.cdninstagram.com"];
|
|
@@ -13215,6 +13471,7 @@ var init_maps_routes = __esm({
|
|
|
13215
13471
|
init_errors();
|
|
13216
13472
|
init_browser_service_env();
|
|
13217
13473
|
init_maps_search_rotation();
|
|
13474
|
+
init_concurrency_gates();
|
|
13218
13475
|
mapsApp = new import_hono5.Hono();
|
|
13219
13476
|
mapsApp.post("/search", createApiKeyAuth(), async (c) => {
|
|
13220
13477
|
const user = c.get("user");
|
|
@@ -13226,14 +13483,21 @@ var init_maps_routes = __esm({
|
|
|
13226
13483
|
if (!parsed.success) {
|
|
13227
13484
|
return c.json({ error: parsed.error.issues[0]?.message ?? "Invalid request" }, 400);
|
|
13228
13485
|
}
|
|
13229
|
-
const
|
|
13230
|
-
|
|
13231
|
-
|
|
13232
|
-
|
|
13233
|
-
|
|
13234
|
-
|
|
13235
|
-
if (!ok) return c.json(insufficientBalanceResponse(balance_mc, MC_COSTS.maps_search), 402);
|
|
13486
|
+
const gate = await acquireConcurrencyGate(user, "maps_search", {
|
|
13487
|
+
reuseLockId: c.req.header("x-mcp-scraper-concurrency-lock"),
|
|
13488
|
+
metadata: { query: parsed.data.query, location: parsed.data.location }
|
|
13489
|
+
});
|
|
13490
|
+
if (!gate.ok) return c.json(concurrencyLimitExceededResponse(gate), 429, { "Retry-After": String(gate.retryAfterSeconds) });
|
|
13491
|
+
let debited = false;
|
|
13236
13492
|
try {
|
|
13493
|
+
const { ok, balance_mc } = await debitMc(
|
|
13494
|
+
user.id,
|
|
13495
|
+
MC_COSTS.maps_search,
|
|
13496
|
+
LedgerOperation.MAPS_SEARCH,
|
|
13497
|
+
[parsed.data.query, parsed.data.location].filter(Boolean).join(" ")
|
|
13498
|
+
);
|
|
13499
|
+
if (!ok) return c.json(insufficientBalanceResponse(balance_mc, MC_COSTS.maps_search), 402);
|
|
13500
|
+
debited = true;
|
|
13237
13501
|
const result = await runMapsSearchWithRotation({
|
|
13238
13502
|
...parsed.data,
|
|
13239
13503
|
configuredKernelProxyId: browserServiceProxyId()
|
|
@@ -13249,7 +13513,7 @@ var init_maps_routes = __esm({
|
|
|
13249
13513
|
});
|
|
13250
13514
|
return c.json(result);
|
|
13251
13515
|
} catch (err) {
|
|
13252
|
-
await creditMc(user.id, MC_COSTS.maps_search, LedgerOperation.REFUND, "failed maps_search call");
|
|
13516
|
+
if (debited) await creditMc(user.id, MC_COSTS.maps_search, LedgerOperation.REFUND, "failed maps_search call");
|
|
13253
13517
|
const msg = err instanceof Error ? err.message : String(err);
|
|
13254
13518
|
await logRequestEvent({
|
|
13255
13519
|
userId: user.id,
|
|
@@ -13261,6 +13525,8 @@ var init_maps_routes = __esm({
|
|
|
13261
13525
|
result: err instanceof MapsSearchRotationError ? { attempts: err.attempts } : void 0
|
|
13262
13526
|
});
|
|
13263
13527
|
return mapsErrorResponse(c, err, "maps_search_failed");
|
|
13528
|
+
} finally {
|
|
13529
|
+
await releaseConcurrencyGate(gate.lockId);
|
|
13264
13530
|
}
|
|
13265
13531
|
});
|
|
13266
13532
|
mapsApp.post("/place", createApiKeyAuth(), async (c) => {
|
|
@@ -13273,17 +13539,24 @@ var init_maps_routes = __esm({
|
|
|
13273
13539
|
if (!parsed.success) {
|
|
13274
13540
|
return c.json({ error: parsed.error.issues[0]?.message ?? "Invalid request" }, 400);
|
|
13275
13541
|
}
|
|
13276
|
-
const
|
|
13277
|
-
|
|
13278
|
-
|
|
13279
|
-
|
|
13280
|
-
|
|
13281
|
-
);
|
|
13282
|
-
if (!ok) return c.json(insufficientBalanceResponse(balance_mc, MC_COSTS.maps_place), 402);
|
|
13542
|
+
const gate = await acquireConcurrencyGate(user, "maps_place", {
|
|
13543
|
+
reuseLockId: c.req.header("x-mcp-scraper-concurrency-lock"),
|
|
13544
|
+
metadata: { businessName: parsed.data.businessName, location: parsed.data.location }
|
|
13545
|
+
});
|
|
13546
|
+
if (!gate.ok) return c.json(concurrencyLimitExceededResponse(gate), 429, { "Retry-After": String(gate.retryAfterSeconds) });
|
|
13283
13547
|
const driver = new BrowserDriver();
|
|
13284
13548
|
const extractor = new MapsExtractor(driver);
|
|
13285
13549
|
let reviewDebitMc = 0;
|
|
13550
|
+
let debited = false;
|
|
13286
13551
|
try {
|
|
13552
|
+
const { ok, balance_mc } = await debitMc(
|
|
13553
|
+
user.id,
|
|
13554
|
+
MC_COSTS.maps_place,
|
|
13555
|
+
LedgerOperation.MAPS_PLACE,
|
|
13556
|
+
`${parsed.data.businessName} ${parsed.data.location}`
|
|
13557
|
+
);
|
|
13558
|
+
if (!ok) return c.json(insufficientBalanceResponse(balance_mc, MC_COSTS.maps_place), 402);
|
|
13559
|
+
debited = true;
|
|
13287
13560
|
const result = await extractor.extract(parsed.data);
|
|
13288
13561
|
const reviewCount = Array.isArray(result.reviews) ? result.reviews.length : 0;
|
|
13289
13562
|
if (reviewCount > 0) {
|
|
@@ -13311,7 +13584,7 @@ var init_maps_routes = __esm({
|
|
|
13311
13584
|
});
|
|
13312
13585
|
return c.json(result);
|
|
13313
13586
|
} catch (err) {
|
|
13314
|
-
await creditMc(user.id, MC_COSTS.maps_place, LedgerOperation.REFUND, "failed maps_place call");
|
|
13587
|
+
if (debited) await creditMc(user.id, MC_COSTS.maps_place, LedgerOperation.REFUND, "failed maps_place call");
|
|
13315
13588
|
if (reviewDebitMc > 0) {
|
|
13316
13589
|
await creditMc(
|
|
13317
13590
|
user.id,
|
|
@@ -13332,6 +13605,7 @@ var init_maps_routes = __esm({
|
|
|
13332
13605
|
return mapsErrorResponse(c, msg, "maps_place_failed");
|
|
13333
13606
|
} finally {
|
|
13334
13607
|
await driver.close();
|
|
13608
|
+
await releaseConcurrencyGate(gate.lockId);
|
|
13335
13609
|
}
|
|
13336
13610
|
});
|
|
13337
13611
|
}
|
|
@@ -13969,6 +14243,8 @@ function formatCreditsInfo(raw, input) {
|
|
|
13969
14243
|
const costs = d.costs ?? [];
|
|
13970
14244
|
const matched = d.matched_cost;
|
|
13971
14245
|
const ledger = d.ledger ?? [];
|
|
14246
|
+
const concurrencyRaw = d.concurrency;
|
|
14247
|
+
const upgradeRaw = concurrencyRaw?.upgrade;
|
|
13972
14248
|
const costRows = costs.map((c) => {
|
|
13973
14249
|
const notes = c.notes ? ` ${c.notes}` : "";
|
|
13974
14250
|
return `| ${c.label} | ${c.credits} | ${c.unit}${notes} |`;
|
|
@@ -13984,10 +14260,20 @@ function formatCreditsInfo(raw, input) {
|
|
|
13984
14260
|
${matched.notes}` : ""}` : input.item ? `
|
|
13985
14261
|
## Matched Cost
|
|
13986
14262
|
No exact cost match found for "${input.item}". See the full cost table below.` : "";
|
|
14263
|
+
const concurrencySection = concurrencyRaw ? [
|
|
14264
|
+
`
|
|
14265
|
+
## Concurrency`,
|
|
14266
|
+
`**Current limit:** ${concurrencyRaw.current_limit ?? "unknown"} concurrent operation${concurrencyRaw.current_limit === 1 ? "" : "s"}`,
|
|
14267
|
+
`**Extra slots:** ${concurrencyRaw.current_extra_slots ?? "unknown"}`,
|
|
14268
|
+
`**Extra concurrency slot:** ${upgradeRaw?.price_label ?? "$5/month"}`,
|
|
14269
|
+
`**Upgrade in terminal:** \`${upgradeRaw?.terminal_command ?? "npx -y -p mcp-scraper@latest mcp-scraper-cli billing concurrency checkout"}\``,
|
|
14270
|
+
`**Billing URL:** ${upgradeRaw?.billing_url ?? "https://mcpscraper.dev/billing"}`
|
|
14271
|
+
].join("\n") : "";
|
|
13987
14272
|
const full = [
|
|
13988
14273
|
`# Credits`,
|
|
13989
14274
|
`**Balance:** ${balance ?? "unknown"} credits`,
|
|
13990
14275
|
matchedSection,
|
|
14276
|
+
concurrencySection,
|
|
13991
14277
|
costs.length ? `
|
|
13992
14278
|
## Cost Table
|
|
13993
14279
|
| Item | Credits | Unit |
|
|
@@ -14016,7 +14302,22 @@ ${ledgerRows}` : ""
|
|
|
14016
14302
|
operation: String(row.operation ?? ""),
|
|
14017
14303
|
credits: row.amount_mc / 1e3,
|
|
14018
14304
|
description: row.description ?? null
|
|
14019
|
-
}))
|
|
14305
|
+
})),
|
|
14306
|
+
concurrency: concurrencyRaw && upgradeRaw ? {
|
|
14307
|
+
currentExtraSlots: Number(concurrencyRaw.current_extra_slots ?? 0),
|
|
14308
|
+
currentLimit: Number(concurrencyRaw.current_limit ?? 1),
|
|
14309
|
+
hasSubscription: concurrencyRaw.has_subscription === true,
|
|
14310
|
+
upgrade: {
|
|
14311
|
+
product: String(upgradeRaw.product ?? "Extra concurrency slot"),
|
|
14312
|
+
priceLabel: String(upgradeRaw.price_label ?? "$5/month"),
|
|
14313
|
+
unitAmountUsd: Number(upgradeRaw.unit_amount_usd ?? 5),
|
|
14314
|
+
currency: String(upgradeRaw.currency ?? "usd"),
|
|
14315
|
+
interval: String(upgradeRaw.interval ?? "month"),
|
|
14316
|
+
billingUrl: String(upgradeRaw.billing_url ?? "https://mcpscraper.dev/billing"),
|
|
14317
|
+
terminalCommand: String(upgradeRaw.terminal_command ?? "npx -y -p mcp-scraper@latest mcp-scraper-cli billing concurrency checkout"),
|
|
14318
|
+
terminalCommandWithApiKeyEnv: String(upgradeRaw.terminal_command_with_api_key_env ?? "MCP_SCRAPER_API_KEY=sk_live_your_key npx -y -p mcp-scraper@latest mcp-scraper-cli billing concurrency checkout")
|
|
14319
|
+
}
|
|
14320
|
+
} : null
|
|
14020
14321
|
}
|
|
14021
14322
|
};
|
|
14022
14323
|
}
|
|
@@ -14811,6 +15112,7 @@ var init_directory_routes = __esm({
|
|
|
14811
15112
|
init_directory_workflow();
|
|
14812
15113
|
init_location_db();
|
|
14813
15114
|
init_browser_service_env();
|
|
15115
|
+
init_concurrency_gates();
|
|
14814
15116
|
directoryApp = new import_hono6.Hono();
|
|
14815
15117
|
directoryApp.post("/run", createApiKeyAuth(), async (c) => {
|
|
14816
15118
|
const user = c.get("user");
|
|
@@ -14826,18 +15128,27 @@ var init_directory_routes = __esm({
|
|
|
14826
15128
|
if (!kernelApiKey && parsed.data.proxyMode !== "none") {
|
|
14827
15129
|
return c.json({ error: "Browser service API key is required for directory workflow Maps searches unless proxyMode is none" }, 503);
|
|
14828
15130
|
}
|
|
14829
|
-
const
|
|
14830
|
-
|
|
14831
|
-
|
|
14832
|
-
|
|
14833
|
-
|
|
14834
|
-
|
|
14835
|
-
|
|
14836
|
-
|
|
14837
|
-
);
|
|
14838
|
-
if (!debit.ok) return c.json(insufficientBalanceResponse(debit.balance_mc, requiredMc), 402);
|
|
14839
|
-
}
|
|
15131
|
+
const gate = await acquireConcurrencyGate(user, "directory_workflow", {
|
|
15132
|
+
reuseLockId: c.req.header("x-mcp-scraper-concurrency-lock"),
|
|
15133
|
+
ttlSeconds: 30 * 60,
|
|
15134
|
+
metadata: { query: parsed.data.query, state: parsed.data.state }
|
|
15135
|
+
});
|
|
15136
|
+
if (!gate.ok) return c.json(concurrencyLimitExceededResponse(gate), 429, { "Retry-After": String(gate.retryAfterSeconds) });
|
|
15137
|
+
let debited = false;
|
|
15138
|
+
let requiredMc = 0;
|
|
14840
15139
|
try {
|
|
15140
|
+
const plan = await resolveDirectoryMarkets(parsed.data);
|
|
15141
|
+
requiredMc = plan.markets.length * MC_COSTS.maps_search;
|
|
15142
|
+
if (requiredMc > 0) {
|
|
15143
|
+
const debit = await debitMc(
|
|
15144
|
+
user.id,
|
|
15145
|
+
requiredMc,
|
|
15146
|
+
LedgerOperation.MAPS_SEARCH,
|
|
15147
|
+
`directory_workflow ${parsed.data.query} ${parsed.data.state} ${plan.markets.length} cities`
|
|
15148
|
+
);
|
|
15149
|
+
if (!debit.ok) return c.json(insufficientBalanceResponse(debit.balance_mc, requiredMc), 402);
|
|
15150
|
+
debited = true;
|
|
15151
|
+
}
|
|
14841
15152
|
const result = await runDirectoryWorkflowFromPlan(parsed.data, plan);
|
|
14842
15153
|
const failedCities = result.cities.filter((city) => city.status === "failed").length;
|
|
14843
15154
|
if (failedCities > 0) {
|
|
@@ -14854,7 +15165,7 @@ var init_directory_routes = __esm({
|
|
|
14854
15165
|
});
|
|
14855
15166
|
return c.json(result);
|
|
14856
15167
|
} catch (err) {
|
|
14857
|
-
if (requiredMc > 0) {
|
|
15168
|
+
if (debited && requiredMc > 0) {
|
|
14858
15169
|
await creditMc(user.id, requiredMc, LedgerOperation.REFUND, "failed directory_workflow call");
|
|
14859
15170
|
}
|
|
14860
15171
|
const message = err instanceof Error ? err.message : String(err);
|
|
@@ -14867,6 +15178,8 @@ var init_directory_routes = __esm({
|
|
|
14867
15178
|
error: message
|
|
14868
15179
|
});
|
|
14869
15180
|
return c.json({ error: message, error_code: "directory_workflow_failed", retryable: true }, 500);
|
|
15181
|
+
} finally {
|
|
15182
|
+
await releaseConcurrencyGate(gate.lockId);
|
|
14870
15183
|
}
|
|
14871
15184
|
});
|
|
14872
15185
|
}
|
|
@@ -15015,18 +15328,21 @@ var init_http_client2 = __esm({
|
|
|
15015
15328
|
"src/workflows/http-client.ts"() {
|
|
15016
15329
|
"use strict";
|
|
15017
15330
|
WorkflowHttpClient = class {
|
|
15018
|
-
constructor(apiUrl, apiKey, fetchImpl = fetch) {
|
|
15331
|
+
constructor(apiUrl, apiKey, fetchImpl = fetch, extraHeaders = {}) {
|
|
15019
15332
|
this.apiUrl = apiUrl;
|
|
15020
15333
|
this.apiKey = apiKey;
|
|
15021
15334
|
this.fetchImpl = fetchImpl;
|
|
15335
|
+
this.extraHeaders = extraHeaders;
|
|
15022
15336
|
}
|
|
15023
15337
|
apiUrl;
|
|
15024
15338
|
apiKey;
|
|
15025
15339
|
fetchImpl;
|
|
15340
|
+
extraHeaders;
|
|
15026
15341
|
async post(path6, body, timeoutMs = 18e4) {
|
|
15027
15342
|
const res = await this.fetchImpl(`${this.apiUrl.replace(/\/$/, "")}${path6}`, {
|
|
15028
15343
|
method: "POST",
|
|
15029
15344
|
headers: {
|
|
15345
|
+
...this.extraHeaders,
|
|
15030
15346
|
"Content-Type": "application/json",
|
|
15031
15347
|
"x-api-key": this.apiKey
|
|
15032
15348
|
},
|
|
@@ -16374,7 +16690,7 @@ async function runWorkflow(id, rawInput, options = {}) {
|
|
|
16374
16690
|
if (!apiKey) throw new Error("MCP_SCRAPER_API_KEY is required for workflow runs. Pass --api-key or set the environment variable.");
|
|
16375
16691
|
const apiUrl = options.apiUrl?.trim() || process.env.MCP_SCRAPER_API_URL?.trim() || "https://mcpscraper.dev";
|
|
16376
16692
|
const artifacts = await ArtifactWriter.create(definition.id, definition.title, input, options.outputDir, options.runId);
|
|
16377
|
-
const client2 = new WorkflowHttpClient(apiUrl, apiKey, options.fetchImpl);
|
|
16693
|
+
const client2 = new WorkflowHttpClient(apiUrl, apiKey, options.fetchImpl, options.headers);
|
|
16378
16694
|
try {
|
|
16379
16695
|
return await definition.run(input, {
|
|
16380
16696
|
runId: artifacts.runId,
|
|
@@ -16473,7 +16789,7 @@ async function readManifestFromSummary(summary) {
|
|
|
16473
16789
|
function webhookSignature(body, timestamp2) {
|
|
16474
16790
|
const secret2 = process.env.MCP_SCRAPER_WEBHOOK_SECRET?.trim();
|
|
16475
16791
|
if (!secret2) return null;
|
|
16476
|
-
return (0,
|
|
16792
|
+
return (0, import_node_crypto3.createHmac)("sha256", secret2).update(`${timestamp2}.${body}`).digest("hex");
|
|
16477
16793
|
}
|
|
16478
16794
|
async function deliverWorkflowWebhook(input) {
|
|
16479
16795
|
if (!input.webhookUrl) return;
|
|
@@ -16527,7 +16843,8 @@ async function executeWorkflowRun(input) {
|
|
|
16527
16843
|
apiKey: input.user.api_key,
|
|
16528
16844
|
apiUrl: input.apiUrl,
|
|
16529
16845
|
outputDir: hostedWorkflowOutputDir(),
|
|
16530
|
-
runId: input.runId
|
|
16846
|
+
runId: input.runId,
|
|
16847
|
+
headers: input.concurrencyLockId ? { "x-mcp-scraper-concurrency-lock": input.concurrencyLockId } : void 0
|
|
16531
16848
|
});
|
|
16532
16849
|
const manifest = await readManifestFromSummary(summary);
|
|
16533
16850
|
await completeWorkflowRunRecord(input.runId, summary.status, manifest ?? { summary });
|
|
@@ -16556,19 +16873,28 @@ async function dispatchDueWorkflowSchedules(apiUrl, limit = 3) {
|
|
|
16556
16873
|
continue;
|
|
16557
16874
|
}
|
|
16558
16875
|
const scheduledFor = schedule.next_run_at ?? now;
|
|
16559
|
-
const
|
|
16560
|
-
|
|
16561
|
-
scheduleId: schedule.id,
|
|
16562
|
-
workflowId: schedule.workflow_id,
|
|
16563
|
-
workflowInput: schedule.input,
|
|
16564
|
-
idempotencyKey: `${schedule.id}:${scheduledFor}`
|
|
16876
|
+
const gate = await acquireConcurrencyGate(user, "workflow_schedule_dispatch", {
|
|
16877
|
+
ttlSeconds: 60 * 60,
|
|
16878
|
+
metadata: { scheduleId: schedule.id, workflowId: schedule.workflow_id }
|
|
16565
16879
|
});
|
|
16566
|
-
|
|
16567
|
-
|
|
16568
|
-
results.push({ scheduleId: schedule.id, runId: run.id, status: run.status });
|
|
16880
|
+
if (!gate.ok) {
|
|
16881
|
+
results.push({ scheduleId: schedule.id, status: "deferred", error: concurrencyLimitExceededResponse(gate).message });
|
|
16569
16882
|
continue;
|
|
16570
16883
|
}
|
|
16884
|
+
let run = null;
|
|
16571
16885
|
try {
|
|
16886
|
+
run = await createWorkflowRun({
|
|
16887
|
+
userId: user.id,
|
|
16888
|
+
scheduleId: schedule.id,
|
|
16889
|
+
workflowId: schedule.workflow_id,
|
|
16890
|
+
workflowInput: schedule.input,
|
|
16891
|
+
idempotencyKey: `${schedule.id}:${scheduledFor}`
|
|
16892
|
+
});
|
|
16893
|
+
await markWorkflowScheduleRan(schedule.id, now, addCadence(scheduledFor, schedule.cadence));
|
|
16894
|
+
if (run.status !== "queued") {
|
|
16895
|
+
results.push({ scheduleId: schedule.id, runId: run.id, status: run.status });
|
|
16896
|
+
continue;
|
|
16897
|
+
}
|
|
16572
16898
|
const executed = await executeWorkflowRun({
|
|
16573
16899
|
runId: run.id,
|
|
16574
16900
|
user,
|
|
@@ -16576,20 +16902,23 @@ async function dispatchDueWorkflowSchedules(apiUrl, limit = 3) {
|
|
|
16576
16902
|
workflowInput: schedule.input,
|
|
16577
16903
|
apiUrl,
|
|
16578
16904
|
scheduleId: schedule.id,
|
|
16579
|
-
webhookUrl: schedule.webhook_url
|
|
16905
|
+
webhookUrl: schedule.webhook_url,
|
|
16906
|
+
concurrencyLockId: gate.lockId
|
|
16580
16907
|
});
|
|
16581
16908
|
results.push({ scheduleId: schedule.id, runId: executed.run.id, status: executed.run.status });
|
|
16582
16909
|
} catch (err) {
|
|
16583
|
-
results.push({ scheduleId: schedule.id, runId: run
|
|
16910
|
+
results.push({ scheduleId: schedule.id, runId: run?.id, status: "failed", error: err instanceof Error ? err.message : String(err) });
|
|
16911
|
+
} finally {
|
|
16912
|
+
await releaseConcurrencyGate(gate.lockId);
|
|
16584
16913
|
}
|
|
16585
16914
|
}
|
|
16586
16915
|
return { dispatched: results.length, results };
|
|
16587
16916
|
}
|
|
16588
|
-
var
|
|
16917
|
+
var import_node_crypto3, import_promises7, import_hono7, import_zod24, workflowApp, WorkflowInputSchema, WorkflowIdSchema, CadenceSchema, ScheduleStatusSchema, RunBodySchema, ScheduleCreateSchema, SchedulePatchSchema;
|
|
16589
16918
|
var init_workflow_routes = __esm({
|
|
16590
16919
|
"src/api/workflow-routes.ts"() {
|
|
16591
16920
|
"use strict";
|
|
16592
|
-
|
|
16921
|
+
import_node_crypto3 = require("crypto");
|
|
16593
16922
|
import_promises7 = require("fs/promises");
|
|
16594
16923
|
import_hono7 = require("hono");
|
|
16595
16924
|
import_zod24 = require("zod");
|
|
@@ -16598,6 +16927,7 @@ var init_workflow_routes = __esm({
|
|
|
16598
16927
|
init_api_auth();
|
|
16599
16928
|
init_db();
|
|
16600
16929
|
init_url_utils();
|
|
16930
|
+
init_concurrency_gates();
|
|
16601
16931
|
workflowApp = new import_hono7.Hono();
|
|
16602
16932
|
WorkflowInputSchema = import_zod24.z.record(import_zod24.z.unknown()).default({});
|
|
16603
16933
|
WorkflowIdSchema = import_zod24.z.string().min(1);
|
|
@@ -16641,20 +16971,29 @@ var init_workflow_routes = __esm({
|
|
|
16641
16971
|
} catch (err) {
|
|
16642
16972
|
return c.json({ error: err instanceof Error ? err.message : String(err) }, 400);
|
|
16643
16973
|
}
|
|
16644
|
-
const
|
|
16974
|
+
const gate = await acquireConcurrencyGate(user, "workflow_run", {
|
|
16975
|
+
ttlSeconds: 60 * 60,
|
|
16976
|
+
metadata: { workflowId: parsed.data.workflowId }
|
|
16977
|
+
});
|
|
16978
|
+
if (!gate.ok) return c.json(concurrencyLimitExceededResponse(gate), 429, { "Retry-After": String(gate.retryAfterSeconds) });
|
|
16979
|
+
let run = null;
|
|
16645
16980
|
try {
|
|
16981
|
+
run = await createWorkflowRun({ userId: user.id, workflowId: parsed.data.workflowId, workflowInput: input });
|
|
16646
16982
|
const executed = await executeWorkflowRun({
|
|
16647
16983
|
runId: run.id,
|
|
16648
16984
|
user,
|
|
16649
16985
|
workflowId: parsed.data.workflowId,
|
|
16650
16986
|
workflowInput: input,
|
|
16651
16987
|
apiUrl: originFromUrl(c.req.url),
|
|
16652
|
-
webhookUrl
|
|
16988
|
+
webhookUrl,
|
|
16989
|
+
concurrencyLockId: gate.lockId
|
|
16653
16990
|
});
|
|
16654
16991
|
return c.json({ run: exposeRun(c, executed.run, executed.artifacts), summary: executed.summary });
|
|
16655
16992
|
} catch (err) {
|
|
16656
|
-
const failed = await getWorkflowRun(run.id, user.id);
|
|
16993
|
+
const failed = run ? await getWorkflowRun(run.id, user.id) : null;
|
|
16657
16994
|
return c.json({ run: failed, error: err instanceof Error ? err.message : String(err) }, 500);
|
|
16995
|
+
} finally {
|
|
16996
|
+
await releaseConcurrencyGate(gate.lockId);
|
|
16658
16997
|
}
|
|
16659
16998
|
});
|
|
16660
16999
|
workflowApp.get("/runs", createApiKeyAuth(), async (c) => {
|
|
@@ -16752,8 +17091,14 @@ var init_workflow_routes = __esm({
|
|
|
16752
17091
|
const user = c.get("user");
|
|
16753
17092
|
const schedule = await getWorkflowSchedule(c.req.param("id"), user.id);
|
|
16754
17093
|
if (!schedule) return c.json({ error: "Schedule not found" }, 404);
|
|
16755
|
-
const
|
|
17094
|
+
const gate = await acquireConcurrencyGate(user, "workflow_schedule_run", {
|
|
17095
|
+
ttlSeconds: 60 * 60,
|
|
17096
|
+
metadata: { scheduleId: schedule.id, workflowId: schedule.workflow_id }
|
|
17097
|
+
});
|
|
17098
|
+
if (!gate.ok) return c.json(concurrencyLimitExceededResponse(gate), 429, { "Retry-After": String(gate.retryAfterSeconds) });
|
|
17099
|
+
let run = null;
|
|
16756
17100
|
try {
|
|
17101
|
+
run = await createWorkflowRun({ userId: user.id, scheduleId: schedule.id, workflowId: schedule.workflow_id, workflowInput: schedule.input });
|
|
16757
17102
|
const executed = await executeWorkflowRun({
|
|
16758
17103
|
runId: run.id,
|
|
16759
17104
|
user,
|
|
@@ -16761,12 +17106,15 @@ var init_workflow_routes = __esm({
|
|
|
16761
17106
|
workflowInput: schedule.input,
|
|
16762
17107
|
apiUrl: originFromUrl(c.req.url),
|
|
16763
17108
|
scheduleId: schedule.id,
|
|
16764
|
-
webhookUrl: schedule.webhook_url
|
|
17109
|
+
webhookUrl: schedule.webhook_url,
|
|
17110
|
+
concurrencyLockId: gate.lockId
|
|
16765
17111
|
});
|
|
16766
17112
|
return c.json({ run: exposeRun(c, executed.run, executed.artifacts), summary: executed.summary });
|
|
16767
17113
|
} catch (err) {
|
|
16768
|
-
const failed = await getWorkflowRun(run.id, user.id);
|
|
17114
|
+
const failed = run ? await getWorkflowRun(run.id, user.id) : null;
|
|
16769
17115
|
return c.json({ run: failed, error: err instanceof Error ? err.message : String(err) }, 500);
|
|
17116
|
+
} finally {
|
|
17117
|
+
await releaseConcurrencyGate(gate.lockId);
|
|
16770
17118
|
}
|
|
16771
17119
|
});
|
|
16772
17120
|
workflowApp.post("/cron/dispatch", async (c) => {
|
|
@@ -16782,7 +17130,7 @@ var init_workflow_routes = __esm({
|
|
|
16782
17130
|
// src/serp-intelligence/page-snapshot-extractor.ts
|
|
16783
17131
|
function sha256(value) {
|
|
16784
17132
|
if (!value) return null;
|
|
16785
|
-
return (0,
|
|
17133
|
+
return (0, import_node_crypto4.createHash)("sha256").update(value).digest("hex");
|
|
16786
17134
|
}
|
|
16787
17135
|
function countWords(markdown) {
|
|
16788
17136
|
const matches = markdown.trim().match(/\b[\p{L}\p{N}][\p{L}\p{N}'-]*\b/gu);
|
|
@@ -17082,11 +17430,11 @@ async function capturePageSnapshots(targets, options = {}) {
|
|
|
17082
17430
|
}
|
|
17083
17431
|
};
|
|
17084
17432
|
}
|
|
17085
|
-
var
|
|
17433
|
+
var import_node_crypto4, import_p_limit3, DEFAULT_TIMEOUT_MS, DEFAULT_MAX_CONCURRENCY, DEFAULT_MAX_CONTENT_CHARS;
|
|
17086
17434
|
var init_page_snapshot_extractor = __esm({
|
|
17087
17435
|
"src/serp-intelligence/page-snapshot-extractor.ts"() {
|
|
17088
17436
|
"use strict";
|
|
17089
|
-
|
|
17437
|
+
import_node_crypto4 = require("crypto");
|
|
17090
17438
|
import_p_limit3 = __toESM(require("p-limit"), 1);
|
|
17091
17439
|
init_kpo_extractor();
|
|
17092
17440
|
init_url_utils();
|
|
@@ -19382,26 +19730,6 @@ async function enforceSerpIntelligenceRateLimit(c, userId) {
|
|
|
19382
19730
|
{ "Retry-After": String(result.resetSeconds) }
|
|
19383
19731
|
);
|
|
19384
19732
|
}
|
|
19385
|
-
async function enforceSerpIntelligenceConcurrency(user) {
|
|
19386
|
-
const active = await countActiveJobsForUser(user.id);
|
|
19387
|
-
const limit = 1 + (user.extra_concurrency_slots ?? 0);
|
|
19388
|
-
if (active < limit) return null;
|
|
19389
|
-
return new Response(
|
|
19390
|
-
JSON.stringify(structuredError({
|
|
19391
|
-
error_code: "concurrency_limit_exceeded",
|
|
19392
|
-
error_type: "concurrency_limit",
|
|
19393
|
-
message: `You have ${active} active job${active !== 1 ? "s" : ""}. Your account allows ${limit} concurrent job${limit !== 1 ? "s" : ""}.`,
|
|
19394
|
-
retryable: true
|
|
19395
|
-
})),
|
|
19396
|
-
{
|
|
19397
|
-
status: 429,
|
|
19398
|
-
headers: {
|
|
19399
|
-
"Content-Type": "application/json",
|
|
19400
|
-
"Retry-After": "30"
|
|
19401
|
-
}
|
|
19402
|
-
}
|
|
19403
|
-
);
|
|
19404
|
-
}
|
|
19405
19733
|
function pageSnapshotTargetsFromBody(body) {
|
|
19406
19734
|
return body.targets?.length ? body.targets : body.urls.map((url) => ({
|
|
19407
19735
|
url,
|
|
@@ -19421,6 +19749,7 @@ var init_serp_intelligence_routes = __esm({
|
|
|
19421
19749
|
init_api_auth();
|
|
19422
19750
|
init_db();
|
|
19423
19751
|
init_rates();
|
|
19752
|
+
init_concurrency_gates();
|
|
19424
19753
|
SERP_INTELLIGENCE_RATE_LIMIT = 60;
|
|
19425
19754
|
SERP_INTELLIGENCE_RATE_WINDOW_SECONDS = 60;
|
|
19426
19755
|
POST_CAPTURE_ROUTE_LABEL = "POST /capture";
|
|
@@ -19435,17 +19764,22 @@ var init_serp_intelligence_routes = __esm({
|
|
|
19435
19764
|
if (!parsed.success) return c.json(formatZodError(parsed.error), 400);
|
|
19436
19765
|
const limited = await enforceSerpIntelligenceRateLimit(c, user.id);
|
|
19437
19766
|
if (limited) return limited;
|
|
19438
|
-
const
|
|
19439
|
-
|
|
19767
|
+
const gate = await acquireConcurrencyGate(user, "serp_intelligence_capture", {
|
|
19768
|
+
reuseLockId: c.req.header("x-mcp-scraper-concurrency-lock"),
|
|
19769
|
+
metadata: { query: parsed.data.query, location: parsed.data.location }
|
|
19770
|
+
});
|
|
19771
|
+
if (!gate.ok) return c.json(concurrencyLimitExceededResponse(gate), 429, { "Retry-After": String(gate.retryAfterSeconds) });
|
|
19440
19772
|
const cost = MC_COSTS.serp;
|
|
19441
|
-
|
|
19442
|
-
user.id,
|
|
19443
|
-
cost,
|
|
19444
|
-
LedgerOperation.SERP,
|
|
19445
|
-
parsed.data.query
|
|
19446
|
-
);
|
|
19447
|
-
if (!ok) return c.json(insufficientBalanceResponse(balance_mc, cost), 402);
|
|
19773
|
+
let debited = false;
|
|
19448
19774
|
try {
|
|
19775
|
+
const { ok, balance_mc } = await debitMc(
|
|
19776
|
+
user.id,
|
|
19777
|
+
cost,
|
|
19778
|
+
LedgerOperation.SERP,
|
|
19779
|
+
parsed.data.query
|
|
19780
|
+
);
|
|
19781
|
+
if (!ok) return c.json(insufficientBalanceResponse(balance_mc, cost), 402);
|
|
19782
|
+
debited = true;
|
|
19449
19783
|
const result = await captureSerpIntelligenceSnapshot(parsed.data, {
|
|
19450
19784
|
kernelApiKey: browserServiceApiKey(),
|
|
19451
19785
|
kernelProxyId: browserServiceProxyId(),
|
|
@@ -19463,7 +19797,7 @@ var init_serp_intelligence_routes = __esm({
|
|
|
19463
19797
|
});
|
|
19464
19798
|
return c.json(result);
|
|
19465
19799
|
} catch (error) {
|
|
19466
|
-
await creditMc(user.id, cost, LedgerOperation.REFUND, "failed serp intelligence capture");
|
|
19800
|
+
if (debited) await creditMc(user.id, cost, LedgerOperation.REFUND, "failed serp intelligence capture");
|
|
19467
19801
|
const body = error instanceof SerpIntelligenceCaptureError ? error.toJSON() : structuredError({
|
|
19468
19802
|
error_code: "capture_failed",
|
|
19469
19803
|
error_type: "capture_error",
|
|
@@ -19480,6 +19814,8 @@ var init_serp_intelligence_routes = __esm({
|
|
|
19480
19814
|
});
|
|
19481
19815
|
const status = error instanceof SerpIntelligenceCaptureError ? error.httpStatus : 500;
|
|
19482
19816
|
return c.json(body, status);
|
|
19817
|
+
} finally {
|
|
19818
|
+
await releaseConcurrencyGate(gate.lockId);
|
|
19483
19819
|
}
|
|
19484
19820
|
});
|
|
19485
19821
|
serpIntelligenceApp.post("/page-snapshots", async (c) => {
|
|
@@ -19490,18 +19826,23 @@ var init_serp_intelligence_routes = __esm({
|
|
|
19490
19826
|
if (!parsed.success) return c.json(formatZodError(parsed.error), 400);
|
|
19491
19827
|
const limited = await enforceSerpIntelligenceRateLimit(c, user.id);
|
|
19492
19828
|
if (limited) return limited;
|
|
19493
|
-
const concurrencyLimited = await enforceSerpIntelligenceConcurrency(user);
|
|
19494
|
-
if (concurrencyLimited) return concurrencyLimited;
|
|
19495
19829
|
const targets = pageSnapshotTargetsFromBody(parsed.data);
|
|
19830
|
+
const gate = await acquireConcurrencyGate(user, "serp_intelligence_page_snapshots", {
|
|
19831
|
+
reuseLockId: c.req.header("x-mcp-scraper-concurrency-lock"),
|
|
19832
|
+
metadata: { targets: targets.length }
|
|
19833
|
+
});
|
|
19834
|
+
if (!gate.ok) return c.json(concurrencyLimitExceededResponse(gate), 429, { "Retry-After": String(gate.retryAfterSeconds) });
|
|
19496
19835
|
const cost = targets.length * MC_COSTS.page_scrape;
|
|
19497
|
-
|
|
19498
|
-
user.id,
|
|
19499
|
-
cost,
|
|
19500
|
-
LedgerOperation.EXTRACT_URL,
|
|
19501
|
-
`serp intelligence page snapshots: ${targets.length}`
|
|
19502
|
-
);
|
|
19503
|
-
if (!ok) return c.json(insufficientBalanceResponse(balance_mc, cost), 402);
|
|
19836
|
+
let debited = false;
|
|
19504
19837
|
try {
|
|
19838
|
+
const { ok, balance_mc } = await debitMc(
|
|
19839
|
+
user.id,
|
|
19840
|
+
cost,
|
|
19841
|
+
LedgerOperation.EXTRACT_URL,
|
|
19842
|
+
`serp intelligence page snapshots: ${targets.length}`
|
|
19843
|
+
);
|
|
19844
|
+
if (!ok) return c.json(insufficientBalanceResponse(balance_mc, cost), 402);
|
|
19845
|
+
debited = true;
|
|
19505
19846
|
const result = await capturePageSnapshots(targets, {
|
|
19506
19847
|
kernelApiKey: browserServiceApiKey(),
|
|
19507
19848
|
timeoutMs: parsed.data.timeoutMs,
|
|
@@ -19518,7 +19859,7 @@ var init_serp_intelligence_routes = __esm({
|
|
|
19518
19859
|
});
|
|
19519
19860
|
return c.json(result);
|
|
19520
19861
|
} catch (error) {
|
|
19521
|
-
await creditMc(user.id, cost, LedgerOperation.REFUND, "failed serp intelligence page snapshots");
|
|
19862
|
+
if (debited) await creditMc(user.id, cost, LedgerOperation.REFUND, "failed serp intelligence page snapshots");
|
|
19522
19863
|
const body = structuredError({
|
|
19523
19864
|
error_code: "page_snapshot_failed",
|
|
19524
19865
|
error_type: "capture_error",
|
|
@@ -19533,6 +19874,8 @@ var init_serp_intelligence_routes = __esm({
|
|
|
19533
19874
|
error: body.message
|
|
19534
19875
|
});
|
|
19535
19876
|
return c.json(body, 500);
|
|
19877
|
+
} finally {
|
|
19878
|
+
await releaseConcurrencyGate(gate.lockId);
|
|
19536
19879
|
}
|
|
19537
19880
|
});
|
|
19538
19881
|
}
|
|
@@ -19543,7 +19886,7 @@ var PACKAGE_VERSION;
|
|
|
19543
19886
|
var init_version = __esm({
|
|
19544
19887
|
"src/version.ts"() {
|
|
19545
19888
|
"use strict";
|
|
19546
|
-
PACKAGE_VERSION = "0.2.
|
|
19889
|
+
PACKAGE_VERSION = "0.2.14";
|
|
19547
19890
|
}
|
|
19548
19891
|
});
|
|
19549
19892
|
|
|
@@ -19898,7 +20241,22 @@ var init_mcp_tool_schemas = __esm({
|
|
|
19898
20241
|
operation: import_zod26.z.string(),
|
|
19899
20242
|
credits: import_zod26.z.number(),
|
|
19900
20243
|
description: NullableString
|
|
19901
|
-
}))
|
|
20244
|
+
})),
|
|
20245
|
+
concurrency: import_zod26.z.object({
|
|
20246
|
+
currentExtraSlots: import_zod26.z.number().int().min(0),
|
|
20247
|
+
currentLimit: import_zod26.z.number().int().min(1),
|
|
20248
|
+
hasSubscription: import_zod26.z.boolean(),
|
|
20249
|
+
upgrade: import_zod26.z.object({
|
|
20250
|
+
product: import_zod26.z.string(),
|
|
20251
|
+
priceLabel: import_zod26.z.string(),
|
|
20252
|
+
unitAmountUsd: import_zod26.z.number(),
|
|
20253
|
+
currency: import_zod26.z.string(),
|
|
20254
|
+
interval: import_zod26.z.string(),
|
|
20255
|
+
billingUrl: import_zod26.z.string().url(),
|
|
20256
|
+
terminalCommand: import_zod26.z.string(),
|
|
20257
|
+
terminalCommandWithApiKeyEnv: import_zod26.z.string()
|
|
20258
|
+
})
|
|
20259
|
+
}).nullable()
|
|
19902
20260
|
};
|
|
19903
20261
|
MapSiteUrlsOutputSchema = {
|
|
19904
20262
|
startUrl: import_zod26.z.string(),
|
|
@@ -19956,7 +20314,7 @@ var init_mcp_tool_schemas = __esm({
|
|
|
19956
20314
|
}))
|
|
19957
20315
|
};
|
|
19958
20316
|
CreditsInfoInputSchema = {
|
|
19959
|
-
item: import_zod26.z.string().optional().describe('Optional tool, action, or feature to look up, e.g. "maps reviews", "extract_url",
|
|
20317
|
+
item: import_zod26.z.string().optional().describe('Optional tool, action, or feature to look up, e.g. "maps reviews", "extract_url", "YouTube transcription", or "concurrency"'),
|
|
19960
20318
|
includeLedger: import_zod26.z.boolean().default(false).describe("Whether to include recent credit ledger entries")
|
|
19961
20319
|
};
|
|
19962
20320
|
SearchSerpInputSchema = {
|
|
@@ -20492,7 +20850,7 @@ function registerPaaExtractorMcpTools(server, executor, options = {}) {
|
|
|
20492
20850
|
}, async (input) => buildRankTrackerBlueprint(input));
|
|
20493
20851
|
server.registerTool("credits_info", {
|
|
20494
20852
|
title: "MCP Scraper Credits & Costs",
|
|
20495
|
-
description: "Answer questions about MCP Scraper credits: current credit balance, what a specific tool/action costs, the full cost table, and
|
|
20853
|
+
description: "Answer questions about MCP Scraper credits, usage limits, and concurrency upgrades: current credit balance, what a specific tool/action costs, the full cost table, current concurrency limit, the extra-slot price, billing URL, and the terminal checkout command. Does not expose payment methods or credit card information.",
|
|
20496
20854
|
inputSchema: CreditsInfoInputSchema,
|
|
20497
20855
|
outputSchema: CreditsInfoOutputSchema,
|
|
20498
20856
|
annotations: {
|
|
@@ -20725,6 +21083,7 @@ async function migrateBrowserAgent() {
|
|
|
20725
21083
|
status TEXT NOT NULL DEFAULT 'open',
|
|
20726
21084
|
label TEXT,
|
|
20727
21085
|
user_id INTEGER,
|
|
21086
|
+
concurrency_lock_id TEXT,
|
|
20728
21087
|
created_at TEXT NOT NULL DEFAULT (datetime('now')),
|
|
20729
21088
|
closed_at TEXT,
|
|
20730
21089
|
last_action_at TEXT,
|
|
@@ -20734,6 +21093,11 @@ async function migrateBrowserAgent() {
|
|
|
20734
21093
|
`);
|
|
20735
21094
|
await db.execute(`CREATE INDEX IF NOT EXISTS browser_agent_sessions_status ON browser_agent_sessions(status)`);
|
|
20736
21095
|
await db.execute(`CREATE INDEX IF NOT EXISTS browser_agent_sessions_user ON browser_agent_sessions(user_id)`);
|
|
21096
|
+
try {
|
|
21097
|
+
await db.execute(`ALTER TABLE browser_agent_sessions ADD COLUMN concurrency_lock_id TEXT`);
|
|
21098
|
+
} catch {
|
|
21099
|
+
}
|
|
21100
|
+
await db.execute(`CREATE INDEX IF NOT EXISTS browser_agent_sessions_concurrency_lock ON browser_agent_sessions(concurrency_lock_id)`);
|
|
20737
21101
|
await db.execute(`
|
|
20738
21102
|
CREATE TABLE IF NOT EXISTS browser_agent_actions (
|
|
20739
21103
|
id TEXT PRIMARY KEY,
|
|
@@ -20761,11 +21125,11 @@ async function migrateBrowserAgent() {
|
|
|
20761
21125
|
}
|
|
20762
21126
|
async function createSessionRow(input) {
|
|
20763
21127
|
const db = getDb();
|
|
20764
|
-
const id = `bas_${(0,
|
|
21128
|
+
const id = `bas_${(0, import_node_crypto5.randomUUID)().replace(/-/g, "").slice(0, 20)}`;
|
|
20765
21129
|
await db.execute({
|
|
20766
|
-
sql: `INSERT INTO browser_agent_sessions (id, runtime_session_id, live_view_url, cdp_ws_url, status, label, user_id, last_action_at)
|
|
20767
|
-
VALUES (?, ?, ?, ?, 'open', ?, ?, datetime('now'))`,
|
|
20768
|
-
args: [id, input.runtimeSessionId, input.liveViewUrl, input.cdpWsUrl, input.label, input.userId]
|
|
21130
|
+
sql: `INSERT INTO browser_agent_sessions (id, runtime_session_id, live_view_url, cdp_ws_url, status, label, user_id, concurrency_lock_id, last_action_at)
|
|
21131
|
+
VALUES (?, ?, ?, ?, 'open', ?, ?, ?, datetime('now'))`,
|
|
21132
|
+
args: [id, input.runtimeSessionId, input.liveViewUrl, input.cdpWsUrl, input.label, input.userId, input.concurrencyLockId ?? null]
|
|
20769
21133
|
});
|
|
20770
21134
|
const row = await getSessionRow(id);
|
|
20771
21135
|
if (!row) throw new Error("session insert failed");
|
|
@@ -20822,7 +21186,7 @@ async function recordAction(input) {
|
|
|
20822
21186
|
sql: `INSERT INTO browser_agent_actions (id, session_id, type, params_json, ok, error)
|
|
20823
21187
|
VALUES (?, ?, ?, ?, ?, ?)`,
|
|
20824
21188
|
args: [
|
|
20825
|
-
`baa_${(0,
|
|
21189
|
+
`baa_${(0, import_node_crypto5.randomUUID)().replace(/-/g, "").slice(0, 20)}`,
|
|
20826
21190
|
input.sessionId,
|
|
20827
21191
|
input.type,
|
|
20828
21192
|
input.params == null ? null : JSON.stringify(input.params),
|
|
@@ -20862,11 +21226,11 @@ async function listReplayRows(sessionId) {
|
|
|
20862
21226
|
});
|
|
20863
21227
|
return res.rows;
|
|
20864
21228
|
}
|
|
20865
|
-
var
|
|
21229
|
+
var import_node_crypto5, _ready;
|
|
20866
21230
|
var init_browser_agent_db = __esm({
|
|
20867
21231
|
"src/api/browser-agent-db.ts"() {
|
|
20868
21232
|
"use strict";
|
|
20869
|
-
|
|
21233
|
+
import_node_crypto5 = require("crypto");
|
|
20870
21234
|
init_db();
|
|
20871
21235
|
_ready = false;
|
|
20872
21236
|
}
|
|
@@ -21186,6 +21550,12 @@ var init_browser_agent_service = __esm({
|
|
|
21186
21550
|
});
|
|
21187
21551
|
|
|
21188
21552
|
// src/api/browser-agent-routes.ts
|
|
21553
|
+
function browserSessionLockTtlSeconds(timeoutSeconds) {
|
|
21554
|
+
if (typeof timeoutSeconds === "number" && Number.isFinite(timeoutSeconds) && timeoutSeconds > 0) {
|
|
21555
|
+
return Math.max(30 * 60, Math.min(Math.trunc(timeoutSeconds) + 5 * 60, 24 * 60 * 60));
|
|
21556
|
+
}
|
|
21557
|
+
return DEFAULT_BROWSER_SESSION_LOCK_TTL_SECONDS;
|
|
21558
|
+
}
|
|
21189
21559
|
async function charge(sessionId, userId, startedAtMs) {
|
|
21190
21560
|
const elapsedMs = Date.now() - startedAtMs;
|
|
21191
21561
|
const { active_ms, billed_mc } = await addActiveMs(sessionId, elapsedMs);
|
|
@@ -21241,6 +21611,8 @@ async function loadOpenSession(id, userId) {
|
|
|
21241
21611
|
const row = await getSessionRow(id);
|
|
21242
21612
|
if (!row) return null;
|
|
21243
21613
|
if (row.user_id != null && row.user_id !== userId) return null;
|
|
21614
|
+
if (row.status !== "open") return null;
|
|
21615
|
+
await extendConcurrencyGate(row.concurrency_lock_id, DEFAULT_BROWSER_SESSION_LOCK_TTL_SECONDS);
|
|
21244
21616
|
return row;
|
|
21245
21617
|
}
|
|
21246
21618
|
function buildBrowserAgentRoutes() {
|
|
@@ -21256,23 +21628,39 @@ function buildBrowserAgentRoutes() {
|
|
|
21256
21628
|
return c.json(insufficientBalanceResponse(Number(user.balance_mc ?? 0), BROWSER_OPEN_MIN_BALANCE_MC), 402);
|
|
21257
21629
|
}
|
|
21258
21630
|
const body = await c.req.json().catch(() => ({}));
|
|
21631
|
+
const timeoutSeconds = typeof body.timeout_seconds === "number" ? body.timeout_seconds : void 0;
|
|
21632
|
+
const gate = await acquireConcurrencyGate(user, "browser_agent_session", {
|
|
21633
|
+
ttlSeconds: browserSessionLockTtlSeconds(timeoutSeconds),
|
|
21634
|
+
metadata: { label: typeof body.label === "string" ? body.label : null }
|
|
21635
|
+
});
|
|
21636
|
+
if (!gate.ok) return c.json(concurrencyLimitExceededResponse(gate), 429, { "Retry-After": String(gate.retryAfterSeconds) });
|
|
21637
|
+
let runtimeSessionId = null;
|
|
21259
21638
|
try {
|
|
21260
21639
|
const created = await createSession({
|
|
21261
|
-
timeoutSeconds
|
|
21640
|
+
timeoutSeconds,
|
|
21262
21641
|
proxyId: typeof body.proxy_id === "string" ? body.proxy_id : void 0,
|
|
21263
21642
|
profileName: typeof body.profile === "string" ? body.profile : void 0,
|
|
21264
21643
|
disableDefaultProxy: body.disable_default_proxy === true,
|
|
21265
21644
|
viewport: body.viewport && typeof body.viewport === "object" ? body.viewport : void 0
|
|
21266
21645
|
});
|
|
21646
|
+
runtimeSessionId = created.runtimeSessionId;
|
|
21267
21647
|
const row = await createSessionRow({
|
|
21268
21648
|
runtimeSessionId: created.runtimeSessionId,
|
|
21269
21649
|
liveViewUrl: created.liveViewUrl,
|
|
21270
21650
|
cdpWsUrl: created.cdpWsUrl,
|
|
21271
21651
|
label: typeof body.label === "string" ? body.label : null,
|
|
21272
|
-
userId: user.id
|
|
21652
|
+
userId: user.id,
|
|
21653
|
+
concurrencyLockId: gate.lockId
|
|
21273
21654
|
});
|
|
21274
21655
|
return c.json({ ...publicSession(row), watch_url: `/console/${row.id}` });
|
|
21275
21656
|
} catch (err) {
|
|
21657
|
+
if (runtimeSessionId) {
|
|
21658
|
+
try {
|
|
21659
|
+
await closeSession(runtimeSessionId);
|
|
21660
|
+
} catch {
|
|
21661
|
+
}
|
|
21662
|
+
}
|
|
21663
|
+
await releaseConcurrencyGate(gate.lockId);
|
|
21276
21664
|
return c.json(failure(err), 502);
|
|
21277
21665
|
}
|
|
21278
21666
|
});
|
|
@@ -21303,6 +21691,7 @@ function buildBrowserAgentRoutes() {
|
|
|
21303
21691
|
} catch {
|
|
21304
21692
|
}
|
|
21305
21693
|
await markSessionClosed(row.id);
|
|
21694
|
+
await releaseConcurrencyGate(row.concurrency_lock_id);
|
|
21306
21695
|
return c.json({ ok: true });
|
|
21307
21696
|
});
|
|
21308
21697
|
app2.post("/sessions/:id/goto", async (c) => {
|
|
@@ -21560,7 +21949,7 @@ function buildBrowserAgentRoutes() {
|
|
|
21560
21949
|
});
|
|
21561
21950
|
return app2;
|
|
21562
21951
|
}
|
|
21563
|
-
var import_hono10, auth;
|
|
21952
|
+
var import_hono10, auth, DEFAULT_BROWSER_SESSION_LOCK_TTL_SECONDS;
|
|
21564
21953
|
var init_browser_agent_routes = __esm({
|
|
21565
21954
|
"src/api/browser-agent-routes.ts"() {
|
|
21566
21955
|
"use strict";
|
|
@@ -21571,7 +21960,9 @@ var init_browser_agent_routes = __esm({
|
|
|
21571
21960
|
init_rates();
|
|
21572
21961
|
init_browser_agent_db();
|
|
21573
21962
|
init_browser_agent_service();
|
|
21963
|
+
init_concurrency_gates();
|
|
21574
21964
|
auth = createApiKeyAuth();
|
|
21965
|
+
DEFAULT_BROWSER_SESSION_LOCK_TTL_SECONDS = 2 * 60 * 60;
|
|
21575
21966
|
}
|
|
21576
21967
|
});
|
|
21577
21968
|
|
|
@@ -22073,14 +22464,14 @@ function getSessionSecret() {
|
|
|
22073
22464
|
function safeEqualHex(a, b) {
|
|
22074
22465
|
if (a.length !== b.length) return false;
|
|
22075
22466
|
try {
|
|
22076
|
-
return (0,
|
|
22467
|
+
return (0, import_node_crypto6.timingSafeEqual)(Buffer.from(a, "hex"), Buffer.from(b, "hex"));
|
|
22077
22468
|
} catch {
|
|
22078
22469
|
return false;
|
|
22079
22470
|
}
|
|
22080
22471
|
}
|
|
22081
22472
|
function signSession(userId) {
|
|
22082
22473
|
const payload = String(userId);
|
|
22083
|
-
const sig = (0,
|
|
22474
|
+
const sig = (0, import_node_crypto6.createHmac)("sha256", secret()).update(payload).digest("hex");
|
|
22084
22475
|
return `${payload}.${sig}`;
|
|
22085
22476
|
}
|
|
22086
22477
|
function verifySession(token) {
|
|
@@ -22088,16 +22479,16 @@ function verifySession(token) {
|
|
|
22088
22479
|
if (dot === -1) return null;
|
|
22089
22480
|
const payload = token.slice(0, dot);
|
|
22090
22481
|
const sig = token.slice(dot + 1);
|
|
22091
|
-
const expected = (0,
|
|
22482
|
+
const expected = (0, import_node_crypto6.createHmac)("sha256", secret()).update(payload).digest("hex");
|
|
22092
22483
|
if (!safeEqualHex(sig, expected)) return null;
|
|
22093
22484
|
const id = parseInt(payload);
|
|
22094
22485
|
return isNaN(id) ? null : id;
|
|
22095
22486
|
}
|
|
22096
|
-
var
|
|
22487
|
+
var import_node_crypto6, isProduction, secret;
|
|
22097
22488
|
var init_session = __esm({
|
|
22098
22489
|
"src/api/session.ts"() {
|
|
22099
22490
|
"use strict";
|
|
22100
|
-
|
|
22491
|
+
import_node_crypto6 = require("crypto");
|
|
22101
22492
|
isProduction = () => process.env.NODE_ENV === "production" || process.env.VERCEL === "1";
|
|
22102
22493
|
secret = () => getSessionSecret();
|
|
22103
22494
|
}
|
|
@@ -22306,14 +22697,23 @@ function countPaaQuestions2(result) {
|
|
|
22306
22697
|
function paaCostForQuestionCount2(questionCount) {
|
|
22307
22698
|
return Math.max(1, questionCount) * MC_COSTS.paa;
|
|
22308
22699
|
}
|
|
22309
|
-
async function checkHarvestLimits(
|
|
22310
|
-
if (
|
|
22311
|
-
|
|
22312
|
-
const
|
|
22313
|
-
|
|
22700
|
+
async function checkHarvestLimits(user, reuseLockId) {
|
|
22701
|
+
if (reuseLockId && await reuseExistingConcurrencyGate(user.id, reuseLockId, 30 * 60)) return null;
|
|
22702
|
+
if (isConcurrencyLimitBypassed(user)) return null;
|
|
22703
|
+
const limit = concurrencyLimitForUser(user);
|
|
22704
|
+
const active = await countActiveUsageForUser(user.id);
|
|
22705
|
+
if (active >= limit) {
|
|
22706
|
+
return concurrencyLimitExceededResponse({
|
|
22707
|
+
ok: false,
|
|
22708
|
+
active,
|
|
22709
|
+
limit,
|
|
22710
|
+
operation: "harvest",
|
|
22711
|
+
retryAfterSeconds: 30
|
|
22712
|
+
});
|
|
22713
|
+
}
|
|
22314
22714
|
return null;
|
|
22315
22715
|
}
|
|
22316
|
-
var import_resend, import_hono12, import_hono13, import_factory6, import_cookie, import_stripe2, secureCookies, isProduction2, sessionCookieOptions, requireAllowedOrigin, auth2, adminAuth, sessionAuth, app, STRIPE_API_VERSION,
|
|
22716
|
+
var import_resend, import_hono12, import_hono13, import_factory6, import_cookie, import_stripe2, secureCookies, isProduction2, sessionCookieOptions, requireAllowedOrigin, auth2, adminAuth, sessionAuth, app, STRIPE_API_VERSION, SYNC_HARVEST_TIMEOUT_OVERRIDE_MS;
|
|
22317
22717
|
var init_server = __esm({
|
|
22318
22718
|
"src/api/server.ts"() {
|
|
22319
22719
|
"use strict";
|
|
@@ -22359,6 +22759,7 @@ var init_server = __esm({
|
|
|
22359
22759
|
init_harvest_problems();
|
|
22360
22760
|
init_harvest_attempt_events();
|
|
22361
22761
|
init_session();
|
|
22762
|
+
init_concurrency_gates();
|
|
22362
22763
|
secureCookies = process.env.NODE_ENV === "production" || process.env.VERCEL === "1";
|
|
22363
22764
|
isProduction2 = secureCookies;
|
|
22364
22765
|
sessionCookieOptions = {
|
|
@@ -22555,9 +22956,6 @@ var init_server = __esm({
|
|
|
22555
22956
|
await revokeApiKey(user.id);
|
|
22556
22957
|
return c.json({ ok: true });
|
|
22557
22958
|
});
|
|
22558
|
-
BYPASS_EMAILS = new Set(
|
|
22559
|
-
(process.env.HARVEST_LIMIT_BYPASS_EMAILS ?? "").split(",").map((e) => e.trim()).filter(Boolean)
|
|
22560
|
-
);
|
|
22561
22959
|
SYNC_HARVEST_TIMEOUT_OVERRIDE_MS = (() => {
|
|
22562
22960
|
const raw = process.env.SYNC_HARVEST_TIMEOUT_MS;
|
|
22563
22961
|
const parsed = raw === void 0 ? NaN : Number(raw);
|
|
@@ -22575,8 +22973,8 @@ var init_server = __esm({
|
|
|
22575
22973
|
if (checked.error) return c.json({ error: checked.error }, 400);
|
|
22576
22974
|
body.callback_url = checked.parsed?.href;
|
|
22577
22975
|
}
|
|
22578
|
-
const limitErr = await checkHarvestLimits(user
|
|
22579
|
-
if (limitErr) return c.json(limitErr, 429);
|
|
22976
|
+
const limitErr = await checkHarvestLimits(user, c.req.header("x-mcp-scraper-concurrency-lock"));
|
|
22977
|
+
if (limitErr) return c.json(limitErr, 429, { "Retry-After": "30" });
|
|
22580
22978
|
const options = {
|
|
22581
22979
|
query: body.query.trim(),
|
|
22582
22980
|
location: body.location,
|
|
@@ -22611,8 +23009,8 @@ var init_server = __esm({
|
|
|
22611
23009
|
const bodyResult = HarvestBodySchema.safeParse(raw);
|
|
22612
23010
|
if (!bodyResult.success) return c.json({ error: bodyResult.error.issues[0]?.message ?? "Invalid request" }, 400);
|
|
22613
23011
|
const body = bodyResult.data;
|
|
22614
|
-
const limitErr = await checkHarvestLimits(user
|
|
22615
|
-
if (limitErr) return c.json(limitErr, 429);
|
|
23012
|
+
const limitErr = await checkHarvestLimits(user, c.req.header("x-mcp-scraper-concurrency-lock"));
|
|
23013
|
+
if (limitErr) return c.json(limitErr, 429, { "Retry-After": "30" });
|
|
22616
23014
|
const options = {
|
|
22617
23015
|
query: body.query.trim(),
|
|
22618
23016
|
location: body.location,
|
|
@@ -22771,9 +23169,16 @@ var init_server = __esm({
|
|
|
22771
23169
|
}
|
|
22772
23170
|
})();
|
|
22773
23171
|
const user = c.get("user");
|
|
22774
|
-
const
|
|
22775
|
-
|
|
23172
|
+
const gate = await acquireConcurrencyGate(user, "extract_url", {
|
|
23173
|
+
reuseLockId: c.req.header("x-mcp-scraper-concurrency-lock"),
|
|
23174
|
+
metadata: { url: canonicalUrl }
|
|
23175
|
+
});
|
|
23176
|
+
if (!gate.ok) return c.json(concurrencyLimitExceededResponse(gate), 429, { "Retry-After": String(gate.retryAfterSeconds) });
|
|
23177
|
+
let debited = false;
|
|
22776
23178
|
try {
|
|
23179
|
+
const { ok: euOk, balance_mc: euBal } = await debitMc(user.id, MC_COSTS.page_scrape, LedgerOperation.EXTRACT_URL, new URL(canonicalUrl).hostname);
|
|
23180
|
+
if (!euOk) return c.json(insufficientBalanceResponse(euBal, MC_COSTS.page_scrape), 402);
|
|
23181
|
+
debited = true;
|
|
22777
23182
|
const kernelApiKey = browserServiceApiKey();
|
|
22778
23183
|
const device = screenshotDevice === "mobile" ? "mobile" : "desktop";
|
|
22779
23184
|
const [result, pageData] = await Promise.all([
|
|
@@ -22791,9 +23196,11 @@ var init_server = __esm({
|
|
|
22791
23196
|
return c.json({ ...result, screenshot: screenshotMeta, branding: brandingData, media: mediaMeta });
|
|
22792
23197
|
} catch (err) {
|
|
22793
23198
|
const msg = err instanceof Error ? err.message : String(err);
|
|
22794
|
-
await creditMc(user.id, MC_COSTS.page_scrape, LedgerOperation.EXTRACT_URL, "failed call");
|
|
23199
|
+
if (debited) await creditMc(user.id, MC_COSTS.page_scrape, LedgerOperation.EXTRACT_URL, "failed call");
|
|
22795
23200
|
await logRequestEvent({ userId: user.id, source: "extract_url", status: "failed", query: canonicalUrl, error: msg });
|
|
22796
23201
|
return c.json({ error: msg }, 500);
|
|
23202
|
+
} finally {
|
|
23203
|
+
await releaseConcurrencyGate(gate.lockId);
|
|
22797
23204
|
}
|
|
22798
23205
|
});
|
|
22799
23206
|
app.post("/map-urls", auth2, async (c) => {
|
|
@@ -22805,9 +23212,16 @@ var init_server = __esm({
|
|
|
22805
23212
|
if (checked.error || !checked.parsed) return c.json({ error: checked.error ?? "Invalid URL" }, 400);
|
|
22806
23213
|
const parsed = checked.parsed;
|
|
22807
23214
|
const user = c.get("user");
|
|
22808
|
-
const
|
|
22809
|
-
|
|
23215
|
+
const gate = await acquireConcurrencyGate(user, "map_urls", {
|
|
23216
|
+
reuseLockId: c.req.header("x-mcp-scraper-concurrency-lock"),
|
|
23217
|
+
metadata: { url: parsed.href }
|
|
23218
|
+
});
|
|
23219
|
+
if (!gate.ok) return c.json(concurrencyLimitExceededResponse(gate), 429, { "Retry-After": String(gate.retryAfterSeconds) });
|
|
23220
|
+
let debited = false;
|
|
22810
23221
|
try {
|
|
23222
|
+
const { ok: mapOk, balance_mc: mapBal } = await debitMc(user.id, MC_COSTS.url_map, LedgerOperation.URL_MAP, parsed.hostname);
|
|
23223
|
+
if (!mapOk) return c.json(insufficientBalanceResponse(mapBal, MC_COSTS.url_map), 402);
|
|
23224
|
+
debited = true;
|
|
22811
23225
|
const result = await spiderSite({
|
|
22812
23226
|
startUrl: parsed.href,
|
|
22813
23227
|
maxUrls: Math.min(2e3, Math.max(1, body.maxUrls ?? 500)),
|
|
@@ -22825,7 +23239,7 @@ var init_server = __esm({
|
|
|
22825
23239
|
return c.json(result);
|
|
22826
23240
|
} catch (err) {
|
|
22827
23241
|
const msg = err instanceof Error ? err.message : String(err);
|
|
22828
|
-
await creditMc(user.id, MC_COSTS.url_map, LedgerOperation.URL_MAP_REFUND, "failed call");
|
|
23242
|
+
if (debited) await creditMc(user.id, MC_COSTS.url_map, LedgerOperation.URL_MAP_REFUND, "failed call");
|
|
22829
23243
|
await logRequestEvent({
|
|
22830
23244
|
userId: user.id,
|
|
22831
23245
|
source: "map_urls",
|
|
@@ -22834,6 +23248,8 @@ var init_server = __esm({
|
|
|
22834
23248
|
error: msg
|
|
22835
23249
|
});
|
|
22836
23250
|
return c.json({ error: msg }, 500);
|
|
23251
|
+
} finally {
|
|
23252
|
+
await releaseConcurrencyGate(gate.lockId);
|
|
22837
23253
|
}
|
|
22838
23254
|
});
|
|
22839
23255
|
app.post("/extract-site", auth2, async (c) => {
|
|
@@ -22846,9 +23262,16 @@ var init_server = __esm({
|
|
|
22846
23262
|
const parsed = checked.parsed;
|
|
22847
23263
|
const user = c.get("user");
|
|
22848
23264
|
const siteHoldMc = MC_COSTS.page_scrape * 10;
|
|
22849
|
-
const
|
|
22850
|
-
|
|
23265
|
+
const gate = await acquireConcurrencyGate(user, "extract_site", {
|
|
23266
|
+
reuseLockId: c.req.header("x-mcp-scraper-concurrency-lock"),
|
|
23267
|
+
metadata: { url: parsed.href }
|
|
23268
|
+
});
|
|
23269
|
+
if (!gate.ok) return c.json(concurrencyLimitExceededResponse(gate), 429, { "Retry-After": String(gate.retryAfterSeconds) });
|
|
23270
|
+
let debited = false;
|
|
22851
23271
|
try {
|
|
23272
|
+
const { ok: siteOk, balance_mc: siteBal } = await debitMc(user.id, siteHoldMc, LedgerOperation.EXTRACT_SITE_HOLD, parsed.hostname);
|
|
23273
|
+
if (!siteOk) return c.json(insufficientBalanceResponse(siteBal, siteHoldMc), 402);
|
|
23274
|
+
debited = true;
|
|
22852
23275
|
const result = await extractSite({
|
|
22853
23276
|
startUrl: parsed.href,
|
|
22854
23277
|
maxPages: Math.min(200, Math.max(1, body.maxPages ?? 100)),
|
|
@@ -22869,7 +23292,7 @@ var init_server = __esm({
|
|
|
22869
23292
|
});
|
|
22870
23293
|
return c.json(result);
|
|
22871
23294
|
} catch (err) {
|
|
22872
|
-
await creditMc(user.id, siteHoldMc, LedgerOperation.EXTRACT_SITE_REFUND, "failed call");
|
|
23295
|
+
if (debited) await creditMc(user.id, siteHoldMc, LedgerOperation.EXTRACT_SITE_REFUND, "failed call");
|
|
22873
23296
|
const msg = err instanceof Error ? err.message : String(err);
|
|
22874
23297
|
await logRequestEvent({
|
|
22875
23298
|
userId: user.id,
|
|
@@ -22879,6 +23302,8 @@ var init_server = __esm({
|
|
|
22879
23302
|
error: msg
|
|
22880
23303
|
});
|
|
22881
23304
|
return c.json({ error: msg }, 500);
|
|
23305
|
+
} finally {
|
|
23306
|
+
await releaseConcurrencyGate(gate.lockId);
|
|
22882
23307
|
}
|
|
22883
23308
|
});
|
|
22884
23309
|
app.post("/billing/checkout", requireAllowedOrigin, sessionAuth, async (c) => {
|
|
@@ -22947,6 +23372,60 @@ var init_server = __esm({
|
|
|
22947
23372
|
return c.json({ error: message }, 500);
|
|
22948
23373
|
}
|
|
22949
23374
|
});
|
|
23375
|
+
app.post("/billing/concurrency/terminal-checkout", auth2, async (c) => {
|
|
23376
|
+
try {
|
|
23377
|
+
const user = c.get("user");
|
|
23378
|
+
const upgrade = concurrencySlotBillingInfo();
|
|
23379
|
+
if (user.concurrency_stripe_sub_id) {
|
|
23380
|
+
return c.json({
|
|
23381
|
+
error: "Already subscribed. Your account already has an active extra concurrency slot subscription.",
|
|
23382
|
+
error_code: "concurrency_subscription_exists",
|
|
23383
|
+
concurrency: {
|
|
23384
|
+
extra_slots: user.extra_concurrency_slots,
|
|
23385
|
+
limit: concurrencyLimitForUser(user),
|
|
23386
|
+
has_subscription: true,
|
|
23387
|
+
upgrade
|
|
23388
|
+
}
|
|
23389
|
+
}, 409);
|
|
23390
|
+
}
|
|
23391
|
+
const secret2 = process.env.STRIPE_SECRET_KEY?.trim();
|
|
23392
|
+
if (!secret2) return c.json({ error: "Stripe is not configured." }, 503);
|
|
23393
|
+
const stripeClient = new import_stripe2.default(secret2, { apiVersion: STRIPE_API_VERSION });
|
|
23394
|
+
let customerId = user.stripe_customer_id;
|
|
23395
|
+
if (!customerId) {
|
|
23396
|
+
const customer = await stripeClient.customers.create({ email: user.email });
|
|
23397
|
+
customerId = customer.id;
|
|
23398
|
+
await setStripeCustomerId(user.id, customerId);
|
|
23399
|
+
}
|
|
23400
|
+
const session = await stripeClient.checkout.sessions.create({
|
|
23401
|
+
customer: customerId,
|
|
23402
|
+
mode: "subscription",
|
|
23403
|
+
line_items: [{ price: CONCURRENCY_PRICE_ID, quantity: 1 }],
|
|
23404
|
+
automatic_tax: { enabled: true },
|
|
23405
|
+
billing_address_collection: "required",
|
|
23406
|
+
customer_update: { address: "auto", name: "auto" },
|
|
23407
|
+
tax_id_collection: { enabled: true },
|
|
23408
|
+
success_url: `${appOrigin()}/billing?slot_added=1&source=terminal`,
|
|
23409
|
+
cancel_url: `${appOrigin()}/billing?slot_cancelled=1&source=terminal`
|
|
23410
|
+
});
|
|
23411
|
+
if (!session.url) return c.json({ error: "Stripe did not return a checkout URL." }, 502);
|
|
23412
|
+
return c.json({
|
|
23413
|
+
checkout_url: session.url,
|
|
23414
|
+
price: upgrade,
|
|
23415
|
+
concurrency: {
|
|
23416
|
+
current_extra_slots: user.extra_concurrency_slots,
|
|
23417
|
+
current_limit: concurrencyLimitForUser(user),
|
|
23418
|
+
after_checkout_extra_slots: user.extra_concurrency_slots + 1,
|
|
23419
|
+
after_checkout_limit: concurrencyLimitForUser({ extra_concurrency_slots: user.extra_concurrency_slots + 1 })
|
|
23420
|
+
},
|
|
23421
|
+
next_step: "Open checkout_url in a browser, complete checkout, then restart or retry the MCP request."
|
|
23422
|
+
});
|
|
23423
|
+
} catch (err) {
|
|
23424
|
+
const message = err instanceof Error ? err.message : "Unable to start terminal checkout.";
|
|
23425
|
+
console.error("[billing/concurrency/terminal-checkout]", message);
|
|
23426
|
+
return c.json({ error: message }, 500);
|
|
23427
|
+
}
|
|
23428
|
+
});
|
|
22950
23429
|
app.post("/billing/concurrency/cancel", requireAllowedOrigin, sessionAuth, async (c) => {
|
|
22951
23430
|
const user = c.get("sessionUser");
|
|
22952
23431
|
if (!user.concurrency_stripe_sub_id) return c.json({ error: "No active concurrency subscription." }, 404);
|
|
@@ -22954,9 +23433,10 @@ var init_server = __esm({
|
|
|
22954
23433
|
if (!stripeSecret) return c.json({ error: "Stripe is not configured." }, 503);
|
|
22955
23434
|
const stripeClient = new import_stripe2.default(stripeSecret, { apiVersion: STRIPE_API_VERSION });
|
|
22956
23435
|
await stripeClient.subscriptions.cancel(user.concurrency_stripe_sub_id);
|
|
22957
|
-
|
|
23436
|
+
const nextSlots = Math.max(0, user.extra_concurrency_slots - 1);
|
|
23437
|
+
await setExtraConcurrencySlots(user.id, nextSlots);
|
|
22958
23438
|
await setConcurrencySubId(user.id, null);
|
|
22959
|
-
return c.json({ ok: true, concurrency_limit:
|
|
23439
|
+
return c.json({ ok: true, extra_concurrency_slots: nextSlots, concurrency_limit: 1 + nextSlots });
|
|
22960
23440
|
});
|
|
22961
23441
|
app.get("/billing/balance", auth2, async (c) => {
|
|
22962
23442
|
const user = c.get("user");
|
|
@@ -22991,6 +23471,12 @@ var init_server = __esm({
|
|
|
22991
23471
|
item: body.item ?? null,
|
|
22992
23472
|
matched_cost: matchedCost ? (({ aliases, ...cost }) => cost)(matchedCost) : null,
|
|
22993
23473
|
costs,
|
|
23474
|
+
concurrency: {
|
|
23475
|
+
current_extra_slots: user.extra_concurrency_slots,
|
|
23476
|
+
current_limit: concurrencyLimitForUser(user),
|
|
23477
|
+
has_subscription: !!user.concurrency_stripe_sub_id,
|
|
23478
|
+
upgrade: concurrencySlotBillingInfo()
|
|
23479
|
+
},
|
|
22994
23480
|
ledger
|
|
22995
23481
|
});
|
|
22996
23482
|
});
|