mcp-scraper 0.2.12 → 0.2.14
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +15 -2
- package/dist/bin/api-server.cjs +1836 -633
- package/dist/bin/api-server.cjs.map +1 -1
- package/dist/bin/api-server.js +3 -3
- package/dist/bin/browser-agent-stdio-server.cjs +1 -1
- package/dist/bin/browser-agent-stdio-server.cjs.map +1 -1
- package/dist/bin/browser-agent-stdio-server.js +2 -2
- package/dist/bin/mcp-scraper-cli.cjs +799 -12
- package/dist/bin/mcp-scraper-cli.cjs.map +1 -1
- package/dist/bin/mcp-scraper-cli.js +90 -9
- package/dist/bin/mcp-scraper-cli.js.map +1 -1
- package/dist/bin/mcp-scraper-combined-stdio-server.cjs +47 -5
- package/dist/bin/mcp-scraper-combined-stdio-server.cjs.map +1 -1
- package/dist/bin/mcp-scraper-combined-stdio-server.js +3 -3
- package/dist/bin/mcp-scraper-install.cjs +4 -3
- package/dist/bin/mcp-scraper-install.cjs.map +1 -1
- package/dist/bin/mcp-scraper-install.js +4 -3
- package/dist/bin/mcp-scraper-install.js.map +1 -1
- package/dist/bin/mcp-stdio-server.cjs +47 -5
- package/dist/bin/mcp-stdio-server.cjs.map +1 -1
- package/dist/bin/mcp-stdio-server.js +2 -2
- package/dist/{chunk-ROS67BNV.js → chunk-BSYPATSM.js} +21 -2
- package/dist/chunk-BSYPATSM.js.map +1 -0
- package/dist/chunk-F44RBOJ5.js +1575 -0
- package/dist/chunk-F44RBOJ5.js.map +1 -0
- package/dist/{chunk-P5RYXUV7.js → chunk-IWQTNY2E.js} +48 -6
- package/dist/chunk-IWQTNY2E.js.map +1 -0
- package/dist/chunk-ROCXJOVY.js +7 -0
- package/dist/chunk-ROCXJOVY.js.map +1 -0
- package/dist/{chunk-7GCCOT3M.js → chunk-TL7YTFLH.js} +15 -1
- package/dist/chunk-TL7YTFLH.js.map +1 -0
- package/dist/{chunk-PB7ENA3G.js → chunk-ZV2XXYR7.js} +2 -2
- package/dist/{db-BVHYI57K.js → db-P5X6UQ3E.js} +2 -2
- package/dist/{server-I2H5WG5I.js → server-7QP6HQJJ.js} +556 -164
- package/dist/server-7QP6HQJJ.js.map +1 -0
- package/dist/{worker-TDJQ6TH3.js → worker-OZSWIS3F.js} +3 -3
- package/docs/specs/cli-agent-wiring-spec.md +4 -1
- package/docs/specs/seo-cli-growth-roadmap-spec.md +4 -2
- package/package.json +1 -1
- package/dist/chunk-7GCCOT3M.js.map +0 -1
- package/dist/chunk-L6IS63WS.js +0 -869
- package/dist/chunk-L6IS63WS.js.map +0 -1
- package/dist/chunk-LK27MYGD.js +0 -7
- package/dist/chunk-LK27MYGD.js.map +0 -1
- package/dist/chunk-P5RYXUV7.js.map +0 -1
- package/dist/chunk-ROS67BNV.js.map +0 -1
- package/dist/server-I2H5WG5I.js.map +0 -1
- /package/dist/{chunk-PB7ENA3G.js.map → chunk-ZV2XXYR7.js.map} +0 -0
- /package/dist/{db-BVHYI57K.js.map → db-P5X6UQ3E.js.map} +0 -0
- /package/dist/{worker-TDJQ6TH3.js.map → worker-OZSWIS3F.js.map} +0 -0
package/dist/bin/api-server.cjs
CHANGED
|
@@ -7099,6 +7099,20 @@ async function migrate() {
|
|
|
7099
7099
|
await db.execute(`ALTER TABLE users ADD COLUMN concurrency_stripe_sub_id TEXT`);
|
|
7100
7100
|
} catch {
|
|
7101
7101
|
}
|
|
7102
|
+
await db.execute(`
|
|
7103
|
+
CREATE TABLE IF NOT EXISTS concurrency_locks (
|
|
7104
|
+
id TEXT PRIMARY KEY,
|
|
7105
|
+
user_id INTEGER NOT NULL REFERENCES users(id),
|
|
7106
|
+
operation TEXT NOT NULL,
|
|
7107
|
+
status TEXT NOT NULL DEFAULT 'active',
|
|
7108
|
+
acquired_at TEXT NOT NULL DEFAULT (datetime('now')),
|
|
7109
|
+
expires_at TEXT NOT NULL,
|
|
7110
|
+
released_at TEXT,
|
|
7111
|
+
metadata TEXT
|
|
7112
|
+
)
|
|
7113
|
+
`);
|
|
7114
|
+
await db.execute(`CREATE INDEX IF NOT EXISTS concurrency_locks_user_status ON concurrency_locks(user_id, status, expires_at)`);
|
|
7115
|
+
await db.execute(`CREATE INDEX IF NOT EXISTS concurrency_locks_operation ON concurrency_locks(operation, status, expires_at)`);
|
|
7102
7116
|
await db.execute(`
|
|
7103
7117
|
CREATE TABLE IF NOT EXISTS ledger (
|
|
7104
7118
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
@@ -8672,6 +8686,19 @@ var init_site_audit_routes = __esm({
|
|
|
8672
8686
|
function browserActiveCostMc(activeMs) {
|
|
8673
8687
|
return Math.round(activeMs * MC_PER_BROWSER_MS);
|
|
8674
8688
|
}
|
|
8689
|
+
function concurrencySlotBillingInfo() {
|
|
8690
|
+
const billingUrl = process.env.CONCURRENCY_BILLING_URL ?? "https://mcpscraper.dev/billing";
|
|
8691
|
+
return {
|
|
8692
|
+
product: "Extra concurrency slot",
|
|
8693
|
+
price_label: CONCURRENCY_SLOT_PRICE_LABEL,
|
|
8694
|
+
unit_amount_usd: CONCURRENCY_SLOT_PRICE_USD,
|
|
8695
|
+
currency: CONCURRENCY_SLOT_PRICE_CURRENCY,
|
|
8696
|
+
interval: CONCURRENCY_SLOT_PRICE_INTERVAL,
|
|
8697
|
+
billing_url: billingUrl,
|
|
8698
|
+
terminal_command: CONCURRENCY_SLOT_TERMINAL_COMMAND,
|
|
8699
|
+
terminal_command_with_api_key_env: `MCP_SCRAPER_API_KEY=sk_live_your_key ${CONCURRENCY_SLOT_TERMINAL_COMMAND}`
|
|
8700
|
+
};
|
|
8701
|
+
}
|
|
8675
8702
|
function mcToCredits(mc) {
|
|
8676
8703
|
return mc / MC_PER_CREDIT;
|
|
8677
8704
|
}
|
|
@@ -8688,7 +8715,7 @@ function insufficientBalanceResponse(balanceMc, requiredMc) {
|
|
|
8688
8715
|
topup_url: topupUrl
|
|
8689
8716
|
};
|
|
8690
8717
|
}
|
|
8691
|
-
var MC_COSTS, MC_PER_BROWSER_MS, BROWSER_OPEN_MIN_BALANCE_MC, MC_PER_CREDIT, CREDIT_COST_CATALOG, CONCURRENCY_PRICE_ID, FREE_SIGNUP_MC, FREE_MONTHLY_REFRESH_MC, BALANCE_PRICE_IDS, BALANCE_PACK_LABELS, LedgerOperation;
|
|
8718
|
+
var MC_COSTS, MC_PER_BROWSER_MS, BROWSER_OPEN_MIN_BALANCE_MC, MC_PER_CREDIT, CREDIT_COST_CATALOG, CONCURRENCY_PRICE_ID, CONCURRENCY_SLOT_PRICE_USD, CONCURRENCY_SLOT_PRICE_CURRENCY, CONCURRENCY_SLOT_PRICE_INTERVAL, CONCURRENCY_SLOT_PRICE_LABEL, CONCURRENCY_SLOT_TERMINAL_COMMAND, FREE_SIGNUP_MC, FREE_MONTHLY_REFRESH_MC, BALANCE_PRICE_IDS, BALANCE_PACK_LABELS, LedgerOperation;
|
|
8692
8719
|
var init_rates = __esm({
|
|
8693
8720
|
"src/api/rates.ts"() {
|
|
8694
8721
|
"use strict";
|
|
@@ -8815,6 +8842,11 @@ var init_rates = __esm({
|
|
|
8815
8842
|
}
|
|
8816
8843
|
];
|
|
8817
8844
|
CONCURRENCY_PRICE_ID = "price_1Ta1NRS8aAcsk3TGwsRnYbix";
|
|
8845
|
+
CONCURRENCY_SLOT_PRICE_USD = 5;
|
|
8846
|
+
CONCURRENCY_SLOT_PRICE_CURRENCY = "usd";
|
|
8847
|
+
CONCURRENCY_SLOT_PRICE_INTERVAL = "month";
|
|
8848
|
+
CONCURRENCY_SLOT_PRICE_LABEL = "$5/month";
|
|
8849
|
+
CONCURRENCY_SLOT_TERMINAL_COMMAND = "npx -y -p mcp-scraper@latest mcp-scraper-cli billing concurrency checkout";
|
|
8818
8850
|
FREE_SIGNUP_MC = 5e5;
|
|
8819
8851
|
FREE_MONTHLY_REFRESH_MC = 25e4;
|
|
8820
8852
|
BALANCE_PRICE_IDS = {
|
|
@@ -10383,6 +10415,187 @@ var init_server_schemas = __esm({
|
|
|
10383
10415
|
}
|
|
10384
10416
|
});
|
|
10385
10417
|
|
|
10418
|
+
// src/api/concurrency-gates.ts
|
|
10419
|
+
function concurrencyLimitForUser(user) {
|
|
10420
|
+
return 1 + Math.max(0, Number(user.extra_concurrency_slots ?? 0));
|
|
10421
|
+
}
|
|
10422
|
+
function concurrencyLimitExceededResponse(gate) {
|
|
10423
|
+
const upgrade = concurrencySlotBillingInfo();
|
|
10424
|
+
const message = `You have ${gate.active} active operation${gate.active !== 1 ? "s" : ""}. Your account allows ${gate.limit} concurrent operation${gate.limit !== 1 ? "s" : ""}. Wait for one to finish, or add an extra concurrency slot for ${upgrade.price_label} at ${upgrade.billing_url}. Terminal upgrade: ${upgrade.terminal_command}`;
|
|
10425
|
+
return {
|
|
10426
|
+
error: message,
|
|
10427
|
+
message,
|
|
10428
|
+
error_code: "concurrency_limit_exceeded",
|
|
10429
|
+
error_type: "concurrency_limit",
|
|
10430
|
+
retryable: true,
|
|
10431
|
+
active: gate.active,
|
|
10432
|
+
limit: gate.limit,
|
|
10433
|
+
operation: gate.operation,
|
|
10434
|
+
upgrade,
|
|
10435
|
+
upgrade_url: upgrade.billing_url,
|
|
10436
|
+
upgrade_command: upgrade.terminal_command
|
|
10437
|
+
};
|
|
10438
|
+
}
|
|
10439
|
+
function isConcurrencyLimitBypassed(user) {
|
|
10440
|
+
const raw = process.env.CONCURRENCY_LIMIT_BYPASS_EMAILS ?? process.env.HARVEST_LIMIT_BYPASS_EMAILS ?? "";
|
|
10441
|
+
return raw.split(",").map((email) => email.trim()).filter(Boolean).includes(user.email);
|
|
10442
|
+
}
|
|
10443
|
+
function lockTtlModifier(ttlSeconds) {
|
|
10444
|
+
const ttl = Number.isFinite(ttlSeconds) ? Math.max(30, Math.min(Math.trunc(Number(ttlSeconds)), MAX_LOCK_TTL_SECONDS)) : DEFAULT_LOCK_TTL_SECONDS;
|
|
10445
|
+
return `+${ttl} seconds`;
|
|
10446
|
+
}
|
|
10447
|
+
function safeMetadata(metadata) {
|
|
10448
|
+
if (!metadata) return null;
|
|
10449
|
+
try {
|
|
10450
|
+
return JSON.stringify(metadata).slice(0, 4e3);
|
|
10451
|
+
} catch {
|
|
10452
|
+
return null;
|
|
10453
|
+
}
|
|
10454
|
+
}
|
|
10455
|
+
async function ensureConcurrencyGateSchema() {
|
|
10456
|
+
if (!schemaReady) {
|
|
10457
|
+
schemaReady = (async () => {
|
|
10458
|
+
const db = getDb();
|
|
10459
|
+
await db.execute(`
|
|
10460
|
+
CREATE TABLE IF NOT EXISTS concurrency_locks (
|
|
10461
|
+
id TEXT PRIMARY KEY,
|
|
10462
|
+
user_id INTEGER NOT NULL REFERENCES users(id),
|
|
10463
|
+
operation TEXT NOT NULL,
|
|
10464
|
+
status TEXT NOT NULL DEFAULT 'active',
|
|
10465
|
+
acquired_at TEXT NOT NULL DEFAULT (datetime('now')),
|
|
10466
|
+
expires_at TEXT NOT NULL,
|
|
10467
|
+
released_at TEXT,
|
|
10468
|
+
metadata TEXT
|
|
10469
|
+
)
|
|
10470
|
+
`);
|
|
10471
|
+
await db.execute(`CREATE INDEX IF NOT EXISTS concurrency_locks_user_status ON concurrency_locks(user_id, status, expires_at)`);
|
|
10472
|
+
await db.execute(`CREATE INDEX IF NOT EXISTS concurrency_locks_operation ON concurrency_locks(operation, status, expires_at)`);
|
|
10473
|
+
})();
|
|
10474
|
+
}
|
|
10475
|
+
return schemaReady;
|
|
10476
|
+
}
|
|
10477
|
+
async function expireConcurrencyLocksForUser(userId) {
|
|
10478
|
+
await ensureConcurrencyGateSchema();
|
|
10479
|
+
await getDb().execute({
|
|
10480
|
+
sql: `UPDATE concurrency_locks
|
|
10481
|
+
SET status = 'expired', released_at = datetime('now')
|
|
10482
|
+
WHERE user_id = ? AND status = 'active' AND expires_at <= datetime('now')`,
|
|
10483
|
+
args: [userId]
|
|
10484
|
+
});
|
|
10485
|
+
}
|
|
10486
|
+
async function countActiveUsageForUser(userId) {
|
|
10487
|
+
await ensureConcurrencyGateSchema();
|
|
10488
|
+
await expireConcurrencyLocksForUser(userId);
|
|
10489
|
+
const res = await getDb().execute({
|
|
10490
|
+
sql: `SELECT
|
|
10491
|
+
(
|
|
10492
|
+
SELECT COUNT(*)
|
|
10493
|
+
FROM jobs
|
|
10494
|
+
WHERE user_id = ?
|
|
10495
|
+
AND (status = 'pending' OR (status = 'running' AND started_at > datetime('now', '-10 minutes')))
|
|
10496
|
+
) +
|
|
10497
|
+
(
|
|
10498
|
+
SELECT COUNT(*)
|
|
10499
|
+
FROM concurrency_locks
|
|
10500
|
+
WHERE user_id = ? AND status = 'active' AND expires_at > datetime('now')
|
|
10501
|
+
) AS n`,
|
|
10502
|
+
args: [userId, userId]
|
|
10503
|
+
});
|
|
10504
|
+
return Number(res.rows[0]?.n ?? 0);
|
|
10505
|
+
}
|
|
10506
|
+
async function reuseExistingConcurrencyGate(userId, lockId, ttlSeconds) {
|
|
10507
|
+
if (!/^cl_[a-f0-9]{24,32}$/.test(lockId)) return false;
|
|
10508
|
+
await ensureConcurrencyGateSchema();
|
|
10509
|
+
await expireConcurrencyLocksForUser(userId);
|
|
10510
|
+
const res = await getDb().execute({
|
|
10511
|
+
sql: `UPDATE concurrency_locks
|
|
10512
|
+
SET expires_at = datetime('now', ?)
|
|
10513
|
+
WHERE id = ?
|
|
10514
|
+
AND user_id = ?
|
|
10515
|
+
AND status = 'active'
|
|
10516
|
+
AND expires_at > datetime('now')`,
|
|
10517
|
+
args: [lockTtlModifier(ttlSeconds), lockId, userId]
|
|
10518
|
+
});
|
|
10519
|
+
return Number(res.rowsAffected ?? 0) > 0;
|
|
10520
|
+
}
|
|
10521
|
+
async function acquireConcurrencyGate(user, operation, options = {}) {
|
|
10522
|
+
await ensureConcurrencyGateSchema();
|
|
10523
|
+
const limit = concurrencyLimitForUser(user);
|
|
10524
|
+
if (isConcurrencyLimitBypassed(user)) {
|
|
10525
|
+
return { ok: true, lockId: null, active: 0, limit, operation, bypassed: true };
|
|
10526
|
+
}
|
|
10527
|
+
if (options.reuseLockId && await reuseExistingConcurrencyGate(user.id, options.reuseLockId, options.ttlSeconds)) {
|
|
10528
|
+
return { ok: true, lockId: null, active: await countActiveUsageForUser(user.id), limit, operation, reused: true };
|
|
10529
|
+
}
|
|
10530
|
+
await expireConcurrencyLocksForUser(user.id);
|
|
10531
|
+
const lockId = `cl_${(0, import_node_crypto2.randomUUID)().replace(/-/g, "").slice(0, 24)}`;
|
|
10532
|
+
const res = await getDb().execute({
|
|
10533
|
+
sql: `INSERT INTO concurrency_locks (id, user_id, operation, status, expires_at, metadata)
|
|
10534
|
+
SELECT ?, ?, ?, 'active', datetime('now', ?), ?
|
|
10535
|
+
WHERE (
|
|
10536
|
+
(
|
|
10537
|
+
SELECT COUNT(*)
|
|
10538
|
+
FROM jobs
|
|
10539
|
+
WHERE user_id = ?
|
|
10540
|
+
AND (status = 'pending' OR (status = 'running' AND started_at > datetime('now', '-10 minutes')))
|
|
10541
|
+
) +
|
|
10542
|
+
(
|
|
10543
|
+
SELECT COUNT(*)
|
|
10544
|
+
FROM concurrency_locks
|
|
10545
|
+
WHERE user_id = ? AND status = 'active' AND expires_at > datetime('now')
|
|
10546
|
+
)
|
|
10547
|
+
) < ?`,
|
|
10548
|
+
args: [
|
|
10549
|
+
lockId,
|
|
10550
|
+
user.id,
|
|
10551
|
+
operation,
|
|
10552
|
+
lockTtlModifier(options.ttlSeconds),
|
|
10553
|
+
safeMetadata(options.metadata),
|
|
10554
|
+
user.id,
|
|
10555
|
+
user.id,
|
|
10556
|
+
limit
|
|
10557
|
+
]
|
|
10558
|
+
});
|
|
10559
|
+
const active = await countActiveUsageForUser(user.id);
|
|
10560
|
+
if (Number(res.rowsAffected ?? 0) > 0) {
|
|
10561
|
+
return { ok: true, lockId, active, limit, operation };
|
|
10562
|
+
}
|
|
10563
|
+
return { ok: false, active, limit, operation, retryAfterSeconds: DEFAULT_RETRY_AFTER_SECONDS };
|
|
10564
|
+
}
|
|
10565
|
+
async function releaseConcurrencyGate(lockId) {
|
|
10566
|
+
if (!lockId) return;
|
|
10567
|
+
await ensureConcurrencyGateSchema();
|
|
10568
|
+
await getDb().execute({
|
|
10569
|
+
sql: `UPDATE concurrency_locks
|
|
10570
|
+
SET status = 'released', released_at = datetime('now')
|
|
10571
|
+
WHERE id = ? AND status = 'active'`,
|
|
10572
|
+
args: [lockId]
|
|
10573
|
+
});
|
|
10574
|
+
}
|
|
10575
|
+
async function extendConcurrencyGate(lockId, ttlSeconds = DEFAULT_LOCK_TTL_SECONDS) {
|
|
10576
|
+
if (!lockId) return;
|
|
10577
|
+
await ensureConcurrencyGateSchema();
|
|
10578
|
+
await getDb().execute({
|
|
10579
|
+
sql: `UPDATE concurrency_locks
|
|
10580
|
+
SET expires_at = datetime('now', ?)
|
|
10581
|
+
WHERE id = ? AND status = 'active' AND expires_at > datetime('now')`,
|
|
10582
|
+
args: [lockTtlModifier(ttlSeconds), lockId]
|
|
10583
|
+
});
|
|
10584
|
+
}
|
|
10585
|
+
var import_node_crypto2, DEFAULT_LOCK_TTL_SECONDS, DEFAULT_RETRY_AFTER_SECONDS, MAX_LOCK_TTL_SECONDS, schemaReady;
|
|
10586
|
+
var init_concurrency_gates = __esm({
|
|
10587
|
+
"src/api/concurrency-gates.ts"() {
|
|
10588
|
+
"use strict";
|
|
10589
|
+
import_node_crypto2 = require("crypto");
|
|
10590
|
+
init_db();
|
|
10591
|
+
init_rates();
|
|
10592
|
+
DEFAULT_LOCK_TTL_SECONDS = 15 * 60;
|
|
10593
|
+
DEFAULT_RETRY_AFTER_SECONDS = 30;
|
|
10594
|
+
MAX_LOCK_TTL_SECONDS = 24 * 60 * 60;
|
|
10595
|
+
schemaReady = null;
|
|
10596
|
+
}
|
|
10597
|
+
});
|
|
10598
|
+
|
|
10386
10599
|
// src/api/youtube-routes.ts
|
|
10387
10600
|
function buildTranscriptMarkdown(result) {
|
|
10388
10601
|
const lines = [];
|
|
@@ -10444,6 +10657,7 @@ var init_youtube_routes = __esm({
|
|
|
10444
10657
|
init_errors();
|
|
10445
10658
|
init_api_auth();
|
|
10446
10659
|
init_server_schemas();
|
|
10660
|
+
init_concurrency_gates();
|
|
10447
10661
|
youtubeApp = new import_hono2.Hono();
|
|
10448
10662
|
youtubeApp.post("/harvest", createApiKeyAuth(), async (c) => {
|
|
10449
10663
|
const raw = await c.req.json().catch(() => ({}));
|
|
@@ -10457,9 +10671,16 @@ var init_youtube_routes = __esm({
|
|
|
10457
10671
|
return c.json({ error: "channel mode requires channelHandle (@handle or UC... ID)" }, 400);
|
|
10458
10672
|
}
|
|
10459
10673
|
const user = c.get("user");
|
|
10460
|
-
const
|
|
10461
|
-
|
|
10674
|
+
const gate = await acquireConcurrencyGate(user, "youtube_harvest", {
|
|
10675
|
+
reuseLockId: c.req.header("x-mcp-scraper-concurrency-lock"),
|
|
10676
|
+
metadata: { mode, query, channelHandle }
|
|
10677
|
+
});
|
|
10678
|
+
if (!gate.ok) return c.json(concurrencyLimitExceededResponse(gate), 429, { "Retry-After": String(gate.retryAfterSeconds) });
|
|
10679
|
+
let debited = false;
|
|
10462
10680
|
try {
|
|
10681
|
+
const { ok: ytOk, balance_mc: ytBal } = await debitMc(user.id, MC_COSTS.yt_channel, LedgerOperation.YT_CHANNEL, query ?? channelHandle ?? "");
|
|
10682
|
+
if (!ytOk) return c.json(insufficientBalanceResponse(ytBal, MC_COSTS.yt_channel), 402);
|
|
10683
|
+
debited = true;
|
|
10463
10684
|
const normalizedChannelUrl = mode === "channel" ? buildYouTubeChannelVideosUrl(channelHandle.trim()) : void 0;
|
|
10464
10685
|
const result = await ytHarvest({
|
|
10465
10686
|
mode,
|
|
@@ -10484,13 +10705,15 @@ var init_youtube_routes = __esm({
|
|
|
10484
10705
|
return c.json({ error: msg }, 400);
|
|
10485
10706
|
}
|
|
10486
10707
|
if (err instanceof CaptchaError || msg.includes("CAPTCHA") || msg.includes("blocked")) {
|
|
10487
|
-
await creditMc(user.id, MC_COSTS.yt_channel, LedgerOperation.YT_CHANNEL_REFUND, "failed call");
|
|
10708
|
+
if (debited) await creditMc(user.id, MC_COSTS.yt_channel, LedgerOperation.YT_CHANNEL_REFUND, "failed call");
|
|
10488
10709
|
await logRequestEvent({ userId: user.id, source: "youtube_harvest", status: "failed", query: mode === "search" ? query?.trim() ?? "" : channelHandle?.trim() ?? "", error: msg });
|
|
10489
10710
|
return c.json({ error: msg }, 503);
|
|
10490
10711
|
}
|
|
10491
|
-
await creditMc(user.id, MC_COSTS.yt_channel, LedgerOperation.YT_CHANNEL_REFUND, "failed call");
|
|
10712
|
+
if (debited) await creditMc(user.id, MC_COSTS.yt_channel, LedgerOperation.YT_CHANNEL_REFUND, "failed call");
|
|
10492
10713
|
await logRequestEvent({ userId: user.id, source: "youtube_harvest", status: "failed", query: mode === "search" ? query?.trim() ?? "" : channelHandle?.trim() ?? "", error: msg });
|
|
10493
10714
|
return c.json({ error: msg }, 500);
|
|
10715
|
+
} finally {
|
|
10716
|
+
await releaseConcurrencyGate(gate.lockId);
|
|
10494
10717
|
}
|
|
10495
10718
|
});
|
|
10496
10719
|
youtubeApp.post("/transcribe", createApiKeyAuth(), async (c) => {
|
|
@@ -10502,9 +10725,16 @@ var init_youtube_routes = __esm({
|
|
|
10502
10725
|
const user = c.get("user");
|
|
10503
10726
|
const holdMins = 5;
|
|
10504
10727
|
const holdMc = MC_COSTS.yt_transcription * holdMins;
|
|
10505
|
-
const
|
|
10506
|
-
|
|
10728
|
+
const gate = await acquireConcurrencyGate(user, "youtube_transcribe", {
|
|
10729
|
+
reuseLockId: c.req.header("x-mcp-scraper-concurrency-lock"),
|
|
10730
|
+
metadata: { videoId: id }
|
|
10731
|
+
});
|
|
10732
|
+
if (!gate.ok) return c.json(concurrencyLimitExceededResponse(gate), 429, { "Retry-After": String(gate.retryAfterSeconds) });
|
|
10733
|
+
let debited = false;
|
|
10507
10734
|
try {
|
|
10735
|
+
const { ok: trOk, balance_mc: trBal } = await debitMc(user.id, holdMc, LedgerOperation.TRANSCRIPTION_HOLD, id);
|
|
10736
|
+
if (!trOk) return c.json(insufficientBalanceResponse(trBal, holdMc), 402);
|
|
10737
|
+
debited = true;
|
|
10508
10738
|
const result = await fetchCaptions(id);
|
|
10509
10739
|
const lastChunk = result.chunks?.[result.chunks.length - 1];
|
|
10510
10740
|
const chunkSecs = lastChunk && Number.isFinite(lastChunk.timestamp[1]) ? lastChunk.timestamp[1] : 0;
|
|
@@ -10529,10 +10759,12 @@ var init_youtube_routes = __esm({
|
|
|
10529
10759
|
html: buildTranscriptHtml(result)
|
|
10530
10760
|
});
|
|
10531
10761
|
} catch (err) {
|
|
10532
|
-
await creditMc(user.id, holdMc, LedgerOperation.TRANSCRIPTION_REFUND, "failed call");
|
|
10762
|
+
if (debited) await creditMc(user.id, holdMc, LedgerOperation.TRANSCRIPTION_REFUND, "failed call");
|
|
10533
10763
|
const msg = err instanceof Error ? err.message : String(err);
|
|
10534
10764
|
await logRequestEvent({ userId: user.id, source: "youtube_transcribe", status: "failed", query: id, error: msg });
|
|
10535
10765
|
return c.json({ error: msg }, 500);
|
|
10766
|
+
} finally {
|
|
10767
|
+
await releaseConcurrencyGate(gate.lockId);
|
|
10536
10768
|
}
|
|
10537
10769
|
});
|
|
10538
10770
|
}
|
|
@@ -10549,6 +10781,7 @@ var init_screenshot_routes = __esm({
|
|
|
10549
10781
|
init_screenshot();
|
|
10550
10782
|
init_api_auth();
|
|
10551
10783
|
init_url_utils();
|
|
10784
|
+
init_concurrency_gates();
|
|
10552
10785
|
ScreenshotBodySchema = import_zod14.z.object({
|
|
10553
10786
|
url: import_zod14.z.string().trim().min(1, "url is required"),
|
|
10554
10787
|
device: import_zod14.z.string().trim().optional(),
|
|
@@ -10564,6 +10797,7 @@ var init_screenshot_routes = __esm({
|
|
|
10564
10797
|
}
|
|
10565
10798
|
const body = parsed.data;
|
|
10566
10799
|
const urlCheck = await validatePublicHttpUrl(body.url, { field: "url", requireHttps: false });
|
|
10800
|
+
let targetUrl = urlCheck.parsed?.href;
|
|
10567
10801
|
if (urlCheck.error) {
|
|
10568
10802
|
if (!body.allowLocal) {
|
|
10569
10803
|
return c.json({ error_code: "invalid_request", message: urlCheck.error }, 400);
|
|
@@ -10577,25 +10811,17 @@ var init_screenshot_routes = __esm({
|
|
|
10577
10811
|
if (parsedFallback.protocol !== "http:" && parsedFallback.protocol !== "https:") {
|
|
10578
10812
|
return c.json({ error: "url must use http or https" }, 400);
|
|
10579
10813
|
}
|
|
10580
|
-
|
|
10581
|
-
try {
|
|
10582
|
-
const buf = await captureScreenshot(parsedFallback.href, browserServiceApiKey(), device2);
|
|
10583
|
-
return new Response(new Uint8Array(buf), {
|
|
10584
|
-
status: 200,
|
|
10585
|
-
headers: {
|
|
10586
|
-
"Content-Type": "image/png",
|
|
10587
|
-
"Content-Length": String(buf.length),
|
|
10588
|
-
"Cache-Control": "no-store"
|
|
10589
|
-
}
|
|
10590
|
-
});
|
|
10591
|
-
} catch (err) {
|
|
10592
|
-
const msg = err instanceof Error ? err.message : String(err);
|
|
10593
|
-
return c.json({ error: msg }, 502);
|
|
10594
|
-
}
|
|
10814
|
+
targetUrl = parsedFallback.href;
|
|
10595
10815
|
}
|
|
10596
10816
|
const device = body.device === "mobile" ? "mobile" : "desktop";
|
|
10817
|
+
const user = c.get("user");
|
|
10818
|
+
const gate = await acquireConcurrencyGate(user, "screenshot", {
|
|
10819
|
+
reuseLockId: c.req.header("x-mcp-scraper-concurrency-lock"),
|
|
10820
|
+
metadata: { url: targetUrl, device }
|
|
10821
|
+
});
|
|
10822
|
+
if (!gate.ok) return c.json(concurrencyLimitExceededResponse(gate), 429, { "Retry-After": String(gate.retryAfterSeconds) });
|
|
10597
10823
|
try {
|
|
10598
|
-
const buf = await captureScreenshot(
|
|
10824
|
+
const buf = await captureScreenshot(targetUrl, browserServiceApiKey(), device);
|
|
10599
10825
|
return new Response(new Uint8Array(buf), {
|
|
10600
10826
|
status: 200,
|
|
10601
10827
|
headers: {
|
|
@@ -10607,6 +10833,8 @@ var init_screenshot_routes = __esm({
|
|
|
10607
10833
|
} catch (err) {
|
|
10608
10834
|
const msg = err instanceof Error ? err.message : String(err);
|
|
10609
10835
|
return c.json({ error: msg }, 502);
|
|
10836
|
+
} finally {
|
|
10837
|
+
await releaseConcurrencyGate(gate.lockId);
|
|
10610
10838
|
}
|
|
10611
10839
|
});
|
|
10612
10840
|
}
|
|
@@ -11978,6 +12206,7 @@ var init_facebook_ad_routes = __esm({
|
|
|
11978
12206
|
import_client3 = require("@fal-ai/client");
|
|
11979
12207
|
init_api_auth();
|
|
11980
12208
|
init_url_utils();
|
|
12209
|
+
init_concurrency_gates();
|
|
11981
12210
|
FacebookAdBodySchema = import_zod15.z.object({
|
|
11982
12211
|
url: import_zod15.z.string().trim().optional(),
|
|
11983
12212
|
libraryId: import_zod15.z.string().trim().optional(),
|
|
@@ -12016,10 +12245,17 @@ var init_facebook_ad_routes = __esm({
|
|
|
12016
12245
|
const libraryId = FacebookAdExtractor.resolveLibraryId(raw2);
|
|
12017
12246
|
if (!libraryId) return c.json({ error: "Could not resolve a valid Facebook Ad Library ID from the provided input" }, 400);
|
|
12018
12247
|
const fbUser = c.get("user");
|
|
12019
|
-
const
|
|
12020
|
-
|
|
12248
|
+
const gate = await acquireConcurrencyGate(fbUser, "facebook_ad", {
|
|
12249
|
+
reuseLockId: c.req.header("x-mcp-scraper-concurrency-lock"),
|
|
12250
|
+
metadata: { libraryId }
|
|
12251
|
+
});
|
|
12252
|
+
if (!gate.ok) return c.json(concurrencyLimitExceededResponse(gate), 429, { "Retry-After": String(gate.retryAfterSeconds) });
|
|
12021
12253
|
const driver = new BrowserDriver();
|
|
12254
|
+
let debited = false;
|
|
12022
12255
|
try {
|
|
12256
|
+
const { ok: adOk, balance_mc: adBal } = await debitMc(fbUser.id, MC_COSTS.fb_ad, LedgerOperation.FB_AD, raw2);
|
|
12257
|
+
if (!adOk) return c.json(insufficientBalanceResponse(adBal, MC_COSTS.fb_ad), 402);
|
|
12258
|
+
debited = true;
|
|
12023
12259
|
await driver.launch(kernelLaunchOpts());
|
|
12024
12260
|
const extractor = new FacebookAdExtractor(driver);
|
|
12025
12261
|
const result = await extractor.extract(libraryId, { openModal: body.openModal !== false });
|
|
@@ -12033,7 +12269,7 @@ var init_facebook_ad_routes = __esm({
|
|
|
12033
12269
|
});
|
|
12034
12270
|
return c.json(result);
|
|
12035
12271
|
} catch (err) {
|
|
12036
|
-
await creditMc(fbUser.id, MC_COSTS.fb_ad, LedgerOperation.FB_AD_REFUND, "failed call");
|
|
12272
|
+
if (debited) await creditMc(fbUser.id, MC_COSTS.fb_ad, LedgerOperation.FB_AD_REFUND, "failed call");
|
|
12037
12273
|
const msg = err instanceof Error ? err.message : String(err);
|
|
12038
12274
|
await logRequestEvent({ userId: fbUser.id, source: "facebook_ad", status: "failed", query: raw2, error: msg });
|
|
12039
12275
|
if (msg.toLowerCase().includes("blocked") || msg.toLowerCase().includes("captcha")) {
|
|
@@ -12042,24 +12278,26 @@ var init_facebook_ad_routes = __esm({
|
|
|
12042
12278
|
return c.json({ error: msg }, 500);
|
|
12043
12279
|
} finally {
|
|
12044
12280
|
await driver.close();
|
|
12281
|
+
await releaseConcurrencyGate(gate.lockId);
|
|
12045
12282
|
}
|
|
12046
12283
|
});
|
|
12047
12284
|
facebookAdApp.post("/page-intel", createApiKeyAuth(), async (c) => {
|
|
12048
12285
|
const raw = await c.req.json().catch(() => ({}));
|
|
12049
12286
|
const parsed = FacebookPageIntelBodySchema.safeParse(raw);
|
|
12050
|
-
if (!parsed.success)
|
|
12051
|
-
return c.json(invalidRequest(parsed.error.issues[0]?.message ?? "Invalid request"), 400);
|
|
12052
|
-
}
|
|
12287
|
+
if (!parsed.success) return c.json(invalidRequest(parsed.error.issues[0]?.message ?? "Invalid request"), 400);
|
|
12053
12288
|
const body = parsed.data;
|
|
12054
12289
|
const maxAds = Math.min(200, Math.max(1, body.maxAds ?? 50));
|
|
12055
12290
|
const country = body.country?.trim().toUpperCase() ?? "US";
|
|
12056
12291
|
const listingUrl = buildPageIntelUrl(body, country);
|
|
12057
12292
|
const fbUser = c.get("user");
|
|
12058
|
-
const
|
|
12059
|
-
if (!
|
|
12293
|
+
const gate = await acquireConcurrencyGate(fbUser, "facebook_page_intel", { reuseLockId: c.req.header("x-mcp-scraper-concurrency-lock"), metadata: { pageId: body.pageId, query: body.query, libraryId: body.libraryId, country } });
|
|
12294
|
+
if (!gate.ok) return c.json(concurrencyLimitExceededResponse(gate), 429, { "Retry-After": String(gate.retryAfterSeconds) });
|
|
12060
12295
|
const driver = new BrowserDriver();
|
|
12061
|
-
let refunded = false;
|
|
12296
|
+
let refunded = false, debited = false;
|
|
12062
12297
|
try {
|
|
12298
|
+
const { ok: fbOk, balance_mc: fbBal } = await debitMc(fbUser.id, MC_COSTS.fb_ad, LedgerOperation.FB_AD, body.pageId ?? body.query ?? body.libraryId ?? "");
|
|
12299
|
+
if (!fbOk) return c.json(insufficientBalanceResponse(fbBal, MC_COSTS.fb_ad), 402);
|
|
12300
|
+
debited = true;
|
|
12063
12301
|
await driver.launch(await kernelLaunchOptsResidential());
|
|
12064
12302
|
await driver.navigateTo(listingUrl);
|
|
12065
12303
|
const extractor = new FacebookAdExtractor(driver);
|
|
@@ -12074,7 +12312,7 @@ var init_facebook_ad_routes = __esm({
|
|
|
12074
12312
|
return c.json(result);
|
|
12075
12313
|
} catch (err) {
|
|
12076
12314
|
const msg = err instanceof Error ? err.message : String(err);
|
|
12077
|
-
if (!refunded) await creditMc(fbUser.id, MC_COSTS.fb_ad, LedgerOperation.FB_AD_REFUND, "failed call");
|
|
12315
|
+
if (debited && !refunded) await creditMc(fbUser.id, MC_COSTS.fb_ad, LedgerOperation.FB_AD_REFUND, "failed call");
|
|
12078
12316
|
await logRequestEvent({ userId: fbUser.id, source: "facebook_page_intel", status: "failed", query: body.pageId ?? body.query ?? body.libraryId ?? "", error: msg });
|
|
12079
12317
|
if (msg.toLowerCase().includes("blocked") || msg.toLowerCase().includes("captcha")) {
|
|
12080
12318
|
return c.json({ error: msg }, 503);
|
|
@@ -12082,6 +12320,7 @@ var init_facebook_ad_routes = __esm({
|
|
|
12082
12320
|
return c.json({ error: msg }, 500);
|
|
12083
12321
|
} finally {
|
|
12084
12322
|
await driver.close();
|
|
12323
|
+
await releaseConcurrencyGate(gate.lockId);
|
|
12085
12324
|
}
|
|
12086
12325
|
});
|
|
12087
12326
|
facebookAdApp.post("/transcribe", createApiKeyAuth(), async (c) => {
|
|
@@ -12097,10 +12336,17 @@ var init_facebook_ad_routes = __esm({
|
|
|
12097
12336
|
}
|
|
12098
12337
|
const videoUrl = urlCheck.parsed.href;
|
|
12099
12338
|
const fbUser = c.get("user");
|
|
12100
|
-
const
|
|
12101
|
-
|
|
12339
|
+
const gate = await acquireConcurrencyGate(fbUser, "facebook_transcribe", {
|
|
12340
|
+
reuseLockId: c.req.header("x-mcp-scraper-concurrency-lock"),
|
|
12341
|
+
metadata: { videoUrl }
|
|
12342
|
+
});
|
|
12343
|
+
if (!gate.ok) return c.json(concurrencyLimitExceededResponse(gate), 429, { "Retry-After": String(gate.retryAfterSeconds) });
|
|
12102
12344
|
import_client3.fal.config({ credentials: process.env.FAL_KEY });
|
|
12345
|
+
let debited = false;
|
|
12103
12346
|
try {
|
|
12347
|
+
const { ok, balance_mc } = await debitMc(fbUser.id, MC_COSTS.fb_transcribe, LedgerOperation.FB_TRANSCRIBE, videoUrl);
|
|
12348
|
+
if (!ok) return c.json(insufficientBalanceResponse(balance_mc, MC_COSTS.fb_transcribe), 402);
|
|
12349
|
+
debited = true;
|
|
12104
12350
|
const startMs = Date.now();
|
|
12105
12351
|
const result = await import_client3.fal.subscribe("fal-ai/wizper", {
|
|
12106
12352
|
input: { audio_url: videoUrl, task: "transcribe", language: "en" },
|
|
@@ -12123,9 +12369,11 @@ var init_facebook_ad_routes = __esm({
|
|
|
12123
12369
|
return c.json({ text, chunks, durationMs, markdown: lines.join("\n") });
|
|
12124
12370
|
} catch (err) {
|
|
12125
12371
|
const msg = err instanceof Error ? err.message : String(err);
|
|
12126
|
-
await creditMc(fbUser.id, MC_COSTS.fb_transcribe, LedgerOperation.FB_TRANSCRIBE_REFUND, "failed call");
|
|
12372
|
+
if (debited) await creditMc(fbUser.id, MC_COSTS.fb_transcribe, LedgerOperation.FB_TRANSCRIBE_REFUND, "failed call");
|
|
12127
12373
|
await logRequestEvent({ userId: fbUser.id, source: "facebook_transcribe", status: "failed", query: videoUrl, error: msg });
|
|
12128
12374
|
return c.json({ error: msg }, 500);
|
|
12375
|
+
} finally {
|
|
12376
|
+
await releaseConcurrencyGate(gate.lockId);
|
|
12129
12377
|
}
|
|
12130
12378
|
});
|
|
12131
12379
|
facebookAdApp.post("/search", createApiKeyAuth(), async (c) => {
|
|
@@ -12139,11 +12387,18 @@ var init_facebook_ad_routes = __esm({
|
|
|
12139
12387
|
const maxResults = Math.min(20, Math.max(1, body.maxResults ?? 10));
|
|
12140
12388
|
const searchUrl = `https://www.facebook.com/ads/library/?active_status=all&ad_type=all&country=${country}&q=${encodeURIComponent(body.query.trim())}&search_type=keyword_unordered`;
|
|
12141
12389
|
const fbUser = c.get("user");
|
|
12142
|
-
const
|
|
12143
|
-
|
|
12390
|
+
const gate = await acquireConcurrencyGate(fbUser, "facebook_search", {
|
|
12391
|
+
reuseLockId: c.req.header("x-mcp-scraper-concurrency-lock"),
|
|
12392
|
+
metadata: { query: body.query.trim(), country }
|
|
12393
|
+
});
|
|
12394
|
+
if (!gate.ok) return c.json(concurrencyLimitExceededResponse(gate), 429, { "Retry-After": String(gate.retryAfterSeconds) });
|
|
12144
12395
|
const driver = new BrowserDriver();
|
|
12145
12396
|
let searchRefunded = false;
|
|
12397
|
+
let debited = false;
|
|
12146
12398
|
try {
|
|
12399
|
+
const { ok, balance_mc } = await debitMc(fbUser.id, MC_COSTS.fb_search, LedgerOperation.FB_SEARCH, body.query.trim());
|
|
12400
|
+
if (!ok) return c.json(insufficientBalanceResponse(balance_mc, MC_COSTS.fb_search), 402);
|
|
12401
|
+
debited = true;
|
|
12147
12402
|
await driver.launch(await kernelLaunchOptsResidential());
|
|
12148
12403
|
const page = driver.getPage();
|
|
12149
12404
|
const collated = await collectAdLibraryResults(page, searchUrl, Math.max(maxResults * 4, 40));
|
|
@@ -12204,7 +12459,7 @@ var init_facebook_ad_routes = __esm({
|
|
|
12204
12459
|
return c.json(searchResult);
|
|
12205
12460
|
} catch (err) {
|
|
12206
12461
|
const msg = err instanceof Error ? err.message : String(err);
|
|
12207
|
-
if (!searchRefunded) await creditMc(fbUser.id, MC_COSTS.fb_search, LedgerOperation.FB_SEARCH_REFUND, "failed call");
|
|
12462
|
+
if (debited && !searchRefunded) await creditMc(fbUser.id, MC_COSTS.fb_search, LedgerOperation.FB_SEARCH_REFUND, "failed call");
|
|
12208
12463
|
await logRequestEvent({ userId: fbUser.id, source: "facebook_search", status: "failed", query: body.query.trim(), error: msg });
|
|
12209
12464
|
if (msg.toLowerCase().includes("blocked") || msg.toLowerCase().includes("captcha")) {
|
|
12210
12465
|
return c.json({ error: msg }, 503);
|
|
@@ -12212,6 +12467,7 @@ var init_facebook_ad_routes = __esm({
|
|
|
12212
12467
|
return c.json({ error: msg }, 500);
|
|
12213
12468
|
} finally {
|
|
12214
12469
|
await driver.close();
|
|
12470
|
+
await releaseConcurrencyGate(gate.lockId);
|
|
12215
12471
|
}
|
|
12216
12472
|
});
|
|
12217
12473
|
ALLOWED_MEDIA_HOSTS = ["fbcdn.net", "cdninstagram.com", "scontent.facebook.com", "scontent.cdninstagram.com"];
|
|
@@ -13215,6 +13471,7 @@ var init_maps_routes = __esm({
|
|
|
13215
13471
|
init_errors();
|
|
13216
13472
|
init_browser_service_env();
|
|
13217
13473
|
init_maps_search_rotation();
|
|
13474
|
+
init_concurrency_gates();
|
|
13218
13475
|
mapsApp = new import_hono5.Hono();
|
|
13219
13476
|
mapsApp.post("/search", createApiKeyAuth(), async (c) => {
|
|
13220
13477
|
const user = c.get("user");
|
|
@@ -13226,14 +13483,21 @@ var init_maps_routes = __esm({
|
|
|
13226
13483
|
if (!parsed.success) {
|
|
13227
13484
|
return c.json({ error: parsed.error.issues[0]?.message ?? "Invalid request" }, 400);
|
|
13228
13485
|
}
|
|
13229
|
-
const
|
|
13230
|
-
|
|
13231
|
-
|
|
13232
|
-
|
|
13233
|
-
|
|
13234
|
-
|
|
13235
|
-
if (!ok) return c.json(insufficientBalanceResponse(balance_mc, MC_COSTS.maps_search), 402);
|
|
13486
|
+
const gate = await acquireConcurrencyGate(user, "maps_search", {
|
|
13487
|
+
reuseLockId: c.req.header("x-mcp-scraper-concurrency-lock"),
|
|
13488
|
+
metadata: { query: parsed.data.query, location: parsed.data.location }
|
|
13489
|
+
});
|
|
13490
|
+
if (!gate.ok) return c.json(concurrencyLimitExceededResponse(gate), 429, { "Retry-After": String(gate.retryAfterSeconds) });
|
|
13491
|
+
let debited = false;
|
|
13236
13492
|
try {
|
|
13493
|
+
const { ok, balance_mc } = await debitMc(
|
|
13494
|
+
user.id,
|
|
13495
|
+
MC_COSTS.maps_search,
|
|
13496
|
+
LedgerOperation.MAPS_SEARCH,
|
|
13497
|
+
[parsed.data.query, parsed.data.location].filter(Boolean).join(" ")
|
|
13498
|
+
);
|
|
13499
|
+
if (!ok) return c.json(insufficientBalanceResponse(balance_mc, MC_COSTS.maps_search), 402);
|
|
13500
|
+
debited = true;
|
|
13237
13501
|
const result = await runMapsSearchWithRotation({
|
|
13238
13502
|
...parsed.data,
|
|
13239
13503
|
configuredKernelProxyId: browserServiceProxyId()
|
|
@@ -13249,7 +13513,7 @@ var init_maps_routes = __esm({
|
|
|
13249
13513
|
});
|
|
13250
13514
|
return c.json(result);
|
|
13251
13515
|
} catch (err) {
|
|
13252
|
-
await creditMc(user.id, MC_COSTS.maps_search, LedgerOperation.REFUND, "failed maps_search call");
|
|
13516
|
+
if (debited) await creditMc(user.id, MC_COSTS.maps_search, LedgerOperation.REFUND, "failed maps_search call");
|
|
13253
13517
|
const msg = err instanceof Error ? err.message : String(err);
|
|
13254
13518
|
await logRequestEvent({
|
|
13255
13519
|
userId: user.id,
|
|
@@ -13261,6 +13525,8 @@ var init_maps_routes = __esm({
|
|
|
13261
13525
|
result: err instanceof MapsSearchRotationError ? { attempts: err.attempts } : void 0
|
|
13262
13526
|
});
|
|
13263
13527
|
return mapsErrorResponse(c, err, "maps_search_failed");
|
|
13528
|
+
} finally {
|
|
13529
|
+
await releaseConcurrencyGate(gate.lockId);
|
|
13264
13530
|
}
|
|
13265
13531
|
});
|
|
13266
13532
|
mapsApp.post("/place", createApiKeyAuth(), async (c) => {
|
|
@@ -13273,17 +13539,24 @@ var init_maps_routes = __esm({
|
|
|
13273
13539
|
if (!parsed.success) {
|
|
13274
13540
|
return c.json({ error: parsed.error.issues[0]?.message ?? "Invalid request" }, 400);
|
|
13275
13541
|
}
|
|
13276
|
-
const
|
|
13277
|
-
|
|
13278
|
-
|
|
13279
|
-
|
|
13280
|
-
|
|
13281
|
-
);
|
|
13282
|
-
if (!ok) return c.json(insufficientBalanceResponse(balance_mc, MC_COSTS.maps_place), 402);
|
|
13542
|
+
const gate = await acquireConcurrencyGate(user, "maps_place", {
|
|
13543
|
+
reuseLockId: c.req.header("x-mcp-scraper-concurrency-lock"),
|
|
13544
|
+
metadata: { businessName: parsed.data.businessName, location: parsed.data.location }
|
|
13545
|
+
});
|
|
13546
|
+
if (!gate.ok) return c.json(concurrencyLimitExceededResponse(gate), 429, { "Retry-After": String(gate.retryAfterSeconds) });
|
|
13283
13547
|
const driver = new BrowserDriver();
|
|
13284
13548
|
const extractor = new MapsExtractor(driver);
|
|
13285
13549
|
let reviewDebitMc = 0;
|
|
13550
|
+
let debited = false;
|
|
13286
13551
|
try {
|
|
13552
|
+
const { ok, balance_mc } = await debitMc(
|
|
13553
|
+
user.id,
|
|
13554
|
+
MC_COSTS.maps_place,
|
|
13555
|
+
LedgerOperation.MAPS_PLACE,
|
|
13556
|
+
`${parsed.data.businessName} ${parsed.data.location}`
|
|
13557
|
+
);
|
|
13558
|
+
if (!ok) return c.json(insufficientBalanceResponse(balance_mc, MC_COSTS.maps_place), 402);
|
|
13559
|
+
debited = true;
|
|
13287
13560
|
const result = await extractor.extract(parsed.data);
|
|
13288
13561
|
const reviewCount = Array.isArray(result.reviews) ? result.reviews.length : 0;
|
|
13289
13562
|
if (reviewCount > 0) {
|
|
@@ -13311,7 +13584,7 @@ var init_maps_routes = __esm({
|
|
|
13311
13584
|
});
|
|
13312
13585
|
return c.json(result);
|
|
13313
13586
|
} catch (err) {
|
|
13314
|
-
await creditMc(user.id, MC_COSTS.maps_place, LedgerOperation.REFUND, "failed maps_place call");
|
|
13587
|
+
if (debited) await creditMc(user.id, MC_COSTS.maps_place, LedgerOperation.REFUND, "failed maps_place call");
|
|
13315
13588
|
if (reviewDebitMc > 0) {
|
|
13316
13589
|
await creditMc(
|
|
13317
13590
|
user.id,
|
|
@@ -13332,6 +13605,7 @@ var init_maps_routes = __esm({
|
|
|
13332
13605
|
return mapsErrorResponse(c, msg, "maps_place_failed");
|
|
13333
13606
|
} finally {
|
|
13334
13607
|
await driver.close();
|
|
13608
|
+
await releaseConcurrencyGate(gate.lockId);
|
|
13335
13609
|
}
|
|
13336
13610
|
});
|
|
13337
13611
|
}
|
|
@@ -13969,6 +14243,8 @@ function formatCreditsInfo(raw, input) {
|
|
|
13969
14243
|
const costs = d.costs ?? [];
|
|
13970
14244
|
const matched = d.matched_cost;
|
|
13971
14245
|
const ledger = d.ledger ?? [];
|
|
14246
|
+
const concurrencyRaw = d.concurrency;
|
|
14247
|
+
const upgradeRaw = concurrencyRaw?.upgrade;
|
|
13972
14248
|
const costRows = costs.map((c) => {
|
|
13973
14249
|
const notes = c.notes ? ` ${c.notes}` : "";
|
|
13974
14250
|
return `| ${c.label} | ${c.credits} | ${c.unit}${notes} |`;
|
|
@@ -13984,10 +14260,20 @@ function formatCreditsInfo(raw, input) {
|
|
|
13984
14260
|
${matched.notes}` : ""}` : input.item ? `
|
|
13985
14261
|
## Matched Cost
|
|
13986
14262
|
No exact cost match found for "${input.item}". See the full cost table below.` : "";
|
|
14263
|
+
const concurrencySection = concurrencyRaw ? [
|
|
14264
|
+
`
|
|
14265
|
+
## Concurrency`,
|
|
14266
|
+
`**Current limit:** ${concurrencyRaw.current_limit ?? "unknown"} concurrent operation${concurrencyRaw.current_limit === 1 ? "" : "s"}`,
|
|
14267
|
+
`**Extra slots:** ${concurrencyRaw.current_extra_slots ?? "unknown"}`,
|
|
14268
|
+
`**Extra concurrency slot:** ${upgradeRaw?.price_label ?? "$5/month"}`,
|
|
14269
|
+
`**Upgrade in terminal:** \`${upgradeRaw?.terminal_command ?? "npx -y -p mcp-scraper@latest mcp-scraper-cli billing concurrency checkout"}\``,
|
|
14270
|
+
`**Billing URL:** ${upgradeRaw?.billing_url ?? "https://mcpscraper.dev/billing"}`
|
|
14271
|
+
].join("\n") : "";
|
|
13987
14272
|
const full = [
|
|
13988
14273
|
`# Credits`,
|
|
13989
14274
|
`**Balance:** ${balance ?? "unknown"} credits`,
|
|
13990
14275
|
matchedSection,
|
|
14276
|
+
concurrencySection,
|
|
13991
14277
|
costs.length ? `
|
|
13992
14278
|
## Cost Table
|
|
13993
14279
|
| Item | Credits | Unit |
|
|
@@ -14016,7 +14302,22 @@ ${ledgerRows}` : ""
|
|
|
14016
14302
|
operation: String(row.operation ?? ""),
|
|
14017
14303
|
credits: row.amount_mc / 1e3,
|
|
14018
14304
|
description: row.description ?? null
|
|
14019
|
-
}))
|
|
14305
|
+
})),
|
|
14306
|
+
concurrency: concurrencyRaw && upgradeRaw ? {
|
|
14307
|
+
currentExtraSlots: Number(concurrencyRaw.current_extra_slots ?? 0),
|
|
14308
|
+
currentLimit: Number(concurrencyRaw.current_limit ?? 1),
|
|
14309
|
+
hasSubscription: concurrencyRaw.has_subscription === true,
|
|
14310
|
+
upgrade: {
|
|
14311
|
+
product: String(upgradeRaw.product ?? "Extra concurrency slot"),
|
|
14312
|
+
priceLabel: String(upgradeRaw.price_label ?? "$5/month"),
|
|
14313
|
+
unitAmountUsd: Number(upgradeRaw.unit_amount_usd ?? 5),
|
|
14314
|
+
currency: String(upgradeRaw.currency ?? "usd"),
|
|
14315
|
+
interval: String(upgradeRaw.interval ?? "month"),
|
|
14316
|
+
billingUrl: String(upgradeRaw.billing_url ?? "https://mcpscraper.dev/billing"),
|
|
14317
|
+
terminalCommand: String(upgradeRaw.terminal_command ?? "npx -y -p mcp-scraper@latest mcp-scraper-cli billing concurrency checkout"),
|
|
14318
|
+
terminalCommandWithApiKeyEnv: String(upgradeRaw.terminal_command_with_api_key_env ?? "MCP_SCRAPER_API_KEY=sk_live_your_key npx -y -p mcp-scraper@latest mcp-scraper-cli billing concurrency checkout")
|
|
14319
|
+
}
|
|
14320
|
+
} : null
|
|
14020
14321
|
}
|
|
14021
14322
|
};
|
|
14022
14323
|
}
|
|
@@ -14095,7 +14396,7 @@ function formatDirectoryWorkflow(raw, input) {
|
|
|
14095
14396
|
const csvPath = d.csvPath ?? null;
|
|
14096
14397
|
const totalResultCount = d.totalResultCount ?? cities.reduce((sum, city) => sum + city.resultCount, 0);
|
|
14097
14398
|
const durationMs = d.durationMs;
|
|
14098
|
-
const
|
|
14399
|
+
const marketRows2 = cities.map((city) => {
|
|
14099
14400
|
const zips = city.zips?.length ? city.zips.slice(0, 8).join(" ") + (city.zips.length > 8 ? ` +${city.zips.length - 8}` : "") : "\u2014";
|
|
14100
14401
|
return `| ${cell(city.city)} | ${city.population.toLocaleString()} | ${city.zips?.length ?? 0} | ${city.resultCount} | ${city.status} | ${cell(zips)} |`;
|
|
14101
14402
|
}).join("\n");
|
|
@@ -14116,7 +14417,7 @@ ${warnings.map((w) => `- ${w}`).join("\n")}` : "";
|
|
|
14116
14417
|
## Markets
|
|
14117
14418
|
| City | Population | ZIPs | Maps Results | Status | ZIP Sample |
|
|
14118
14419
|
|---|---:|---:|---:|---|---|
|
|
14119
|
-
${
|
|
14420
|
+
${marketRows2}`,
|
|
14120
14421
|
businessRows ? `
|
|
14121
14422
|
## Top Candidates By City
|
|
14122
14423
|
| City | # | Name | Category | Rating | Website | Maps |
|
|
@@ -14811,6 +15112,7 @@ var init_directory_routes = __esm({
|
|
|
14811
15112
|
init_directory_workflow();
|
|
14812
15113
|
init_location_db();
|
|
14813
15114
|
init_browser_service_env();
|
|
15115
|
+
init_concurrency_gates();
|
|
14814
15116
|
directoryApp = new import_hono6.Hono();
|
|
14815
15117
|
directoryApp.post("/run", createApiKeyAuth(), async (c) => {
|
|
14816
15118
|
const user = c.get("user");
|
|
@@ -14826,18 +15128,27 @@ var init_directory_routes = __esm({
|
|
|
14826
15128
|
if (!kernelApiKey && parsed.data.proxyMode !== "none") {
|
|
14827
15129
|
return c.json({ error: "Browser service API key is required for directory workflow Maps searches unless proxyMode is none" }, 503);
|
|
14828
15130
|
}
|
|
14829
|
-
const
|
|
14830
|
-
|
|
14831
|
-
|
|
14832
|
-
|
|
14833
|
-
|
|
14834
|
-
|
|
14835
|
-
|
|
14836
|
-
|
|
14837
|
-
);
|
|
14838
|
-
if (!debit.ok) return c.json(insufficientBalanceResponse(debit.balance_mc, requiredMc), 402);
|
|
14839
|
-
}
|
|
15131
|
+
const gate = await acquireConcurrencyGate(user, "directory_workflow", {
|
|
15132
|
+
reuseLockId: c.req.header("x-mcp-scraper-concurrency-lock"),
|
|
15133
|
+
ttlSeconds: 30 * 60,
|
|
15134
|
+
metadata: { query: parsed.data.query, state: parsed.data.state }
|
|
15135
|
+
});
|
|
15136
|
+
if (!gate.ok) return c.json(concurrencyLimitExceededResponse(gate), 429, { "Retry-After": String(gate.retryAfterSeconds) });
|
|
15137
|
+
let debited = false;
|
|
15138
|
+
let requiredMc = 0;
|
|
14840
15139
|
try {
|
|
15140
|
+
const plan = await resolveDirectoryMarkets(parsed.data);
|
|
15141
|
+
requiredMc = plan.markets.length * MC_COSTS.maps_search;
|
|
15142
|
+
if (requiredMc > 0) {
|
|
15143
|
+
const debit = await debitMc(
|
|
15144
|
+
user.id,
|
|
15145
|
+
requiredMc,
|
|
15146
|
+
LedgerOperation.MAPS_SEARCH,
|
|
15147
|
+
`directory_workflow ${parsed.data.query} ${parsed.data.state} ${plan.markets.length} cities`
|
|
15148
|
+
);
|
|
15149
|
+
if (!debit.ok) return c.json(insufficientBalanceResponse(debit.balance_mc, requiredMc), 402);
|
|
15150
|
+
debited = true;
|
|
15151
|
+
}
|
|
14841
15152
|
const result = await runDirectoryWorkflowFromPlan(parsed.data, plan);
|
|
14842
15153
|
const failedCities = result.cities.filter((city) => city.status === "failed").length;
|
|
14843
15154
|
if (failedCities > 0) {
|
|
@@ -14854,7 +15165,7 @@ var init_directory_routes = __esm({
|
|
|
14854
15165
|
});
|
|
14855
15166
|
return c.json(result);
|
|
14856
15167
|
} catch (err) {
|
|
14857
|
-
if (requiredMc > 0) {
|
|
15168
|
+
if (debited && requiredMc > 0) {
|
|
14858
15169
|
await creditMc(user.id, requiredMc, LedgerOperation.REFUND, "failed directory_workflow call");
|
|
14859
15170
|
}
|
|
14860
15171
|
const message = err instanceof Error ? err.message : String(err);
|
|
@@ -14867,6 +15178,8 @@ var init_directory_routes = __esm({
|
|
|
14867
15178
|
error: message
|
|
14868
15179
|
});
|
|
14869
15180
|
return c.json({ error: message, error_code: "directory_workflow_failed", retryable: true }, 500);
|
|
15181
|
+
} finally {
|
|
15182
|
+
await releaseConcurrencyGate(gate.lockId);
|
|
14870
15183
|
}
|
|
14871
15184
|
});
|
|
14872
15185
|
}
|
|
@@ -15015,18 +15328,21 @@ var init_http_client2 = __esm({
|
|
|
15015
15328
|
"src/workflows/http-client.ts"() {
|
|
15016
15329
|
"use strict";
|
|
15017
15330
|
WorkflowHttpClient = class {
|
|
15018
|
-
constructor(apiUrl, apiKey, fetchImpl = fetch) {
|
|
15331
|
+
constructor(apiUrl, apiKey, fetchImpl = fetch, extraHeaders = {}) {
|
|
15019
15332
|
this.apiUrl = apiUrl;
|
|
15020
15333
|
this.apiKey = apiKey;
|
|
15021
15334
|
this.fetchImpl = fetchImpl;
|
|
15335
|
+
this.extraHeaders = extraHeaders;
|
|
15022
15336
|
}
|
|
15023
15337
|
apiUrl;
|
|
15024
15338
|
apiKey;
|
|
15025
15339
|
fetchImpl;
|
|
15340
|
+
extraHeaders;
|
|
15026
15341
|
async post(path6, body, timeoutMs = 18e4) {
|
|
15027
15342
|
const res = await this.fetchImpl(`${this.apiUrl.replace(/\/$/, "")}${path6}`, {
|
|
15028
15343
|
method: "POST",
|
|
15029
15344
|
headers: {
|
|
15345
|
+
...this.extraHeaders,
|
|
15030
15346
|
"Content-Type": "application/json",
|
|
15031
15347
|
"x-api-key": this.apiKey
|
|
15032
15348
|
},
|
|
@@ -15646,6 +15962,718 @@ ${summary}
|
|
|
15646
15962
|
}
|
|
15647
15963
|
});
|
|
15648
15964
|
|
|
15965
|
+
// src/workflows/workflows/seo-workflow-utils.ts
|
|
15966
|
+
function normalizeDomain2(value) {
|
|
15967
|
+
if (!value) return null;
|
|
15968
|
+
try {
|
|
15969
|
+
const url = new URL(value.includes("://") ? value : `https://${value}`);
|
|
15970
|
+
return url.hostname.replace(/^www\./, "").toLowerCase();
|
|
15971
|
+
} catch {
|
|
15972
|
+
return value.replace(/^https?:\/\//, "").replace(/^www\./, "").split("/")[0]?.toLowerCase() || null;
|
|
15973
|
+
}
|
|
15974
|
+
}
|
|
15975
|
+
function domainFromUrl2(url) {
|
|
15976
|
+
return normalizeDomain2(url) ?? "";
|
|
15977
|
+
}
|
|
15978
|
+
function numberFrom2(value) {
|
|
15979
|
+
if (value === null || value === void 0 || value === "") return null;
|
|
15980
|
+
const parsed = Number(String(value).replace(/[^\d.]/g, ""));
|
|
15981
|
+
return Number.isFinite(parsed) ? parsed : null;
|
|
15982
|
+
}
|
|
15983
|
+
function median2(values) {
|
|
15984
|
+
const nums = values.filter((v) => v !== null).sort((a, b) => a - b);
|
|
15985
|
+
if (!nums.length) return null;
|
|
15986
|
+
return nums[Math.floor(nums.length / 2)] ?? null;
|
|
15987
|
+
}
|
|
15988
|
+
function textTerms(text, limit = 12) {
|
|
15989
|
+
const counts = /* @__PURE__ */ new Map();
|
|
15990
|
+
for (const token of text.toLowerCase().replace(/[^a-z0-9\s]/g, " ").split(/\s+/)) {
|
|
15991
|
+
if (token.length < 4 || STOP_WORDS.has(token)) continue;
|
|
15992
|
+
counts.set(token, (counts.get(token) ?? 0) + 1);
|
|
15993
|
+
}
|
|
15994
|
+
return [...counts.entries()].sort((a, b) => b[1] - a[1] || a[0].localeCompare(b[0])).slice(0, limit).map(([term, count]) => `${term} (${count})`).join("; ");
|
|
15995
|
+
}
|
|
15996
|
+
function classifyQuestion(question) {
|
|
15997
|
+
const q = question.toLowerCase();
|
|
15998
|
+
if (/\b(cost|price|pricing|charge|expensive|cheap)\b/.test(q)) return "cost";
|
|
15999
|
+
if (/\b(best|top|recommended|reviews?|compare|versus|vs)\b/.test(q)) return "comparison";
|
|
16000
|
+
if (/\b(how|steps?|process|way to)\b/.test(q)) return "process";
|
|
16001
|
+
if (/\b(why|worth|important|benefit)\b/.test(q)) return "why";
|
|
16002
|
+
if (/\b(near me|city|local|nearby)\b/.test(q)) return "local";
|
|
16003
|
+
if (/\b(can|does|do|is|are|should|will)\b/.test(q)) return "decision";
|
|
16004
|
+
return "definition";
|
|
16005
|
+
}
|
|
16006
|
+
function questionRows(paa) {
|
|
16007
|
+
return (paa?.flat ?? []).filter((row) => row.question).map((row, index) => {
|
|
16008
|
+
const url = row.source_cite ?? "";
|
|
16009
|
+
const domain = normalizeDomain2(row.source_site ?? "") ?? domainFromUrl2(url);
|
|
16010
|
+
return {
|
|
16011
|
+
position: index + 1,
|
|
16012
|
+
intent: classifyQuestion(row.question ?? ""),
|
|
16013
|
+
question: row.question ?? "",
|
|
16014
|
+
answer_excerpt: (row.answer ?? "").slice(0, 320),
|
|
16015
|
+
source_title: row.source_title ?? "",
|
|
16016
|
+
source_domain: domain,
|
|
16017
|
+
source_url: url
|
|
16018
|
+
};
|
|
16019
|
+
});
|
|
16020
|
+
}
|
|
16021
|
+
function sourceDomainRows(rows) {
|
|
16022
|
+
const byDomain = /* @__PURE__ */ new Map();
|
|
16023
|
+
for (const row of rows) {
|
|
16024
|
+
const domain = String(row.source_domain ?? row.domain ?? "");
|
|
16025
|
+
if (!domain) continue;
|
|
16026
|
+
const entry = byDomain.get(domain) ?? { questions: 0, urls: /* @__PURE__ */ new Set(), intents: /* @__PURE__ */ new Map() };
|
|
16027
|
+
entry.questions += row.question ? 1 : 0;
|
|
16028
|
+
if (row.source_url || row.url) entry.urls.add(String(row.source_url ?? row.url));
|
|
16029
|
+
if (row.intent) entry.intents.set(String(row.intent), (entry.intents.get(String(row.intent)) ?? 0) + 1);
|
|
16030
|
+
byDomain.set(domain, entry);
|
|
16031
|
+
}
|
|
16032
|
+
return [...byDomain.entries()].map(([domain, entry]) => ({
|
|
16033
|
+
domain,
|
|
16034
|
+
question_mentions: entry.questions,
|
|
16035
|
+
source_url_count: entry.urls.size,
|
|
16036
|
+
top_intents: [...entry.intents.entries()].sort((a, b) => b[1] - a[1]).map(([intent, count]) => `${intent} (${count})`).join("; ")
|
|
16037
|
+
})).sort((a, b) => Number(b.question_mentions) - Number(a.question_mentions) || String(a.domain).localeCompare(String(b.domain)));
|
|
16038
|
+
}
|
|
16039
|
+
function splitSentences(text) {
|
|
16040
|
+
return (text ?? "").replace(/\s+/g, " ").split(/(?<=[.!?])\s+/).map((sentence) => sentence.trim()).filter((sentence) => sentence.length > 20).slice(0, 20);
|
|
16041
|
+
}
|
|
16042
|
+
function classifySentence(sentence) {
|
|
16043
|
+
const s = sentence.toLowerCase();
|
|
16044
|
+
if (/\bis\b|\bare\b|\bmeans\b|\brefers to\b/.test(s)) return "definition";
|
|
16045
|
+
if (/\binclude\b|\bconsider\b|\bfactors?\b|\bcriteria\b/.test(s)) return "criteria";
|
|
16046
|
+
if (/\bfirst\b|\bthen\b|\bsteps?\b|\bprocess\b/.test(s)) return "process";
|
|
16047
|
+
if (/\bvs\b|\bthan\b|\bcompare\b|\bdifference\b/.test(s)) return "comparison";
|
|
16048
|
+
if (/\bcost\b|\bprice\b|\baverage\b|\brange\b/.test(s)) return "cost";
|
|
16049
|
+
return "claim";
|
|
16050
|
+
}
|
|
16051
|
+
function pageSummaryRow(page, source) {
|
|
16052
|
+
return {
|
|
16053
|
+
position: source.position ?? "",
|
|
16054
|
+
domain: source.domain,
|
|
16055
|
+
url: source.url,
|
|
16056
|
+
serp_title: source.title ?? "",
|
|
16057
|
+
page_title: page.title ?? "",
|
|
16058
|
+
h1: page.h1 ?? "",
|
|
16059
|
+
meta_description: page.metaDescription ?? "",
|
|
16060
|
+
word_count: page.wordCount ?? "",
|
|
16061
|
+
heading_count: page.headings?.length ?? 0,
|
|
16062
|
+
schema_types: (page.schemaTypes ?? []).join("; ")
|
|
16063
|
+
};
|
|
16064
|
+
}
|
|
16065
|
+
async function mapLimit3(items, limit, fn) {
|
|
16066
|
+
const out = new Array(items.length);
|
|
16067
|
+
let next = 0;
|
|
16068
|
+
async function worker() {
|
|
16069
|
+
while (next < items.length) {
|
|
16070
|
+
const index = next;
|
|
16071
|
+
next += 1;
|
|
16072
|
+
out[index] = await fn(items[index], index);
|
|
16073
|
+
}
|
|
16074
|
+
}
|
|
16075
|
+
await Promise.all(Array.from({ length: Math.min(Math.max(1, limit), items.length) }, () => worker()));
|
|
16076
|
+
return out;
|
|
16077
|
+
}
|
|
16078
|
+
var STOP_WORDS;
|
|
16079
|
+
var init_seo_workflow_utils = __esm({
|
|
16080
|
+
"src/workflows/workflows/seo-workflow-utils.ts"() {
|
|
16081
|
+
"use strict";
|
|
16082
|
+
STOP_WORDS = /* @__PURE__ */ new Set([
|
|
16083
|
+
"about",
|
|
16084
|
+
"after",
|
|
16085
|
+
"also",
|
|
16086
|
+
"because",
|
|
16087
|
+
"been",
|
|
16088
|
+
"best",
|
|
16089
|
+
"both",
|
|
16090
|
+
"from",
|
|
16091
|
+
"have",
|
|
16092
|
+
"into",
|
|
16093
|
+
"more",
|
|
16094
|
+
"most",
|
|
16095
|
+
"near",
|
|
16096
|
+
"only",
|
|
16097
|
+
"over",
|
|
16098
|
+
"than",
|
|
16099
|
+
"that",
|
|
16100
|
+
"their",
|
|
16101
|
+
"them",
|
|
16102
|
+
"then",
|
|
16103
|
+
"there",
|
|
16104
|
+
"these",
|
|
16105
|
+
"they",
|
|
16106
|
+
"this",
|
|
16107
|
+
"what",
|
|
16108
|
+
"when",
|
|
16109
|
+
"where",
|
|
16110
|
+
"which",
|
|
16111
|
+
"while",
|
|
16112
|
+
"with",
|
|
16113
|
+
"your",
|
|
16114
|
+
"will",
|
|
16115
|
+
"would",
|
|
16116
|
+
"should",
|
|
16117
|
+
"could",
|
|
16118
|
+
"does",
|
|
16119
|
+
"were",
|
|
16120
|
+
"cost",
|
|
16121
|
+
"costs"
|
|
16122
|
+
]);
|
|
16123
|
+
}
|
|
16124
|
+
});
|
|
16125
|
+
|
|
16126
|
+
// src/workflows/workflows/comparison-briefs.ts
|
|
16127
|
+
function businessRowsFromMaps(location2, query, results) {
|
|
16128
|
+
return results.map((result) => ({
|
|
16129
|
+
source_query: query,
|
|
16130
|
+
source_location: location2,
|
|
16131
|
+
city: location2.split(",")[0]?.trim() ?? location2,
|
|
16132
|
+
state: location2.split(",")[1]?.trim() ?? "",
|
|
16133
|
+
population: "",
|
|
16134
|
+
result_position: result.position,
|
|
16135
|
+
business_name: result.name,
|
|
16136
|
+
review_stars: result.rating ?? "",
|
|
16137
|
+
review_count: result.reviewCount ?? "",
|
|
16138
|
+
category: result.category ?? "",
|
|
16139
|
+
address: result.address ?? "",
|
|
16140
|
+
phone: result.phone ?? "",
|
|
16141
|
+
website_url: result.websiteUrl ?? "",
|
|
16142
|
+
place_url: result.placeUrl ?? "",
|
|
16143
|
+
cid: result.cid ?? "",
|
|
16144
|
+
cid_decimal: result.cidDecimal ?? "",
|
|
16145
|
+
result_status: "ok",
|
|
16146
|
+
error: ""
|
|
16147
|
+
}));
|
|
16148
|
+
}
|
|
16149
|
+
function marketRows(rows) {
|
|
16150
|
+
const byLocation = /* @__PURE__ */ new Map();
|
|
16151
|
+
for (const row of rows) {
|
|
16152
|
+
const key = String(row.source_location ?? row.location ?? "");
|
|
16153
|
+
if (!key) continue;
|
|
16154
|
+
const list = byLocation.get(key) ?? [];
|
|
16155
|
+
list.push(row);
|
|
16156
|
+
byLocation.set(key, list);
|
|
16157
|
+
}
|
|
16158
|
+
return [...byLocation.entries()].map(([location2, list]) => {
|
|
16159
|
+
const counts = list.map((row) => numberFrom2(row.review_count));
|
|
16160
|
+
const ratings = list.map((row) => numberFrom2(row.review_stars));
|
|
16161
|
+
const topThree = list.slice(0, 3).map((row) => numberFrom2(row.review_count)).filter((v) => v !== null);
|
|
16162
|
+
const categories = /* @__PURE__ */ new Map();
|
|
16163
|
+
for (const row of list) {
|
|
16164
|
+
const category = String(row.category ?? "");
|
|
16165
|
+
if (category) categories.set(category, (categories.get(category) ?? 0) + 1);
|
|
16166
|
+
}
|
|
16167
|
+
return {
|
|
16168
|
+
source_location: location2,
|
|
16169
|
+
result_count: list.filter((row) => row.business_name).length,
|
|
16170
|
+
median_review_count: median2(counts) ?? "",
|
|
16171
|
+
median_rating: median2(ratings) ?? "",
|
|
16172
|
+
top_three_average_review_count: topThree.length ? Math.round(topThree.reduce((a, b) => a + b, 0) / topThree.length) : "",
|
|
16173
|
+
top_categories: [...categories.entries()].sort((a, b) => b[1] - a[1]).slice(0, 5).map(([cat, count]) => `${cat} (${count})`).join("; "),
|
|
16174
|
+
websites_present: list.filter((row) => row.website_url).length
|
|
16175
|
+
};
|
|
16176
|
+
});
|
|
16177
|
+
}
|
|
16178
|
+
function comparisonRows(rows) {
|
|
16179
|
+
const benchmarkByLocation = /* @__PURE__ */ new Map();
|
|
16180
|
+
for (const market of marketRows(rows)) {
|
|
16181
|
+
benchmarkByLocation.set(String(market.source_location), numberFrom2(market.top_three_average_review_count) ?? 0);
|
|
16182
|
+
}
|
|
16183
|
+
return rows.filter((row) => row.business_name).map((row) => {
|
|
16184
|
+
const reviews = numberFrom2(row.review_count) ?? 0;
|
|
16185
|
+
const benchmark = benchmarkByLocation.get(String(row.source_location)) ?? 0;
|
|
16186
|
+
const websiteMissing = !row.website_url;
|
|
16187
|
+
const rank = numberFrom2(row.result_position) ?? 999;
|
|
16188
|
+
return {
|
|
16189
|
+
source_location: row.source_location,
|
|
16190
|
+
result_position: row.result_position,
|
|
16191
|
+
business_name: row.business_name,
|
|
16192
|
+
category: row.category,
|
|
16193
|
+
review_stars: row.review_stars,
|
|
16194
|
+
review_count: row.review_count,
|
|
16195
|
+
review_gap_to_top3_average: benchmark ? Math.max(0, benchmark - reviews) : "",
|
|
16196
|
+
website_url: row.website_url,
|
|
16197
|
+
place_url: row.place_url,
|
|
16198
|
+
comparison_note: rank <= 3 ? "visible leader" : websiteMissing ? "ranking without website" : reviews < benchmark ? "review-light competitor" : "visible competitor"
|
|
16199
|
+
};
|
|
16200
|
+
});
|
|
16201
|
+
}
|
|
16202
|
+
function organicRows(serp, targetDomain) {
|
|
16203
|
+
return (serp?.organicResults ?? []).map((result) => {
|
|
16204
|
+
const url = result.url ?? "";
|
|
16205
|
+
const domain = normalizeDomain2(result.domain ?? "") ?? domainFromUrl2(url);
|
|
16206
|
+
return {
|
|
16207
|
+
position: result.position ?? "",
|
|
16208
|
+
title: result.title ?? "",
|
|
16209
|
+
url,
|
|
16210
|
+
domain,
|
|
16211
|
+
snippet: result.snippet ?? "",
|
|
16212
|
+
is_target: targetDomain ? domain === targetDomain : false
|
|
16213
|
+
};
|
|
16214
|
+
});
|
|
16215
|
+
}
|
|
16216
|
+
function pageGapRows(targetPage, competitorPages) {
|
|
16217
|
+
const targetHeadingText = new Set((targetPage?.headings ?? []).map((h) => h.text.toLowerCase().replace(/[^a-z0-9\s]/g, " ").trim()));
|
|
16218
|
+
const targetTerms = new Set((targetPage?.headings ?? []).flatMap((h) => h.text.toLowerCase().replace(/[^a-z0-9\s]/g, " ").split(/\s+/).filter(Boolean)));
|
|
16219
|
+
const rows = [];
|
|
16220
|
+
for (const { source, page } of competitorPages) {
|
|
16221
|
+
for (const heading of page.headings ?? []) {
|
|
16222
|
+
if (heading.level > 3 || !heading.text) continue;
|
|
16223
|
+
const normalized = heading.text.toLowerCase().replace(/[^a-z0-9\s]/g, " ").trim();
|
|
16224
|
+
const terms = normalized.split(/\s+/).filter((term) => term.length > 3);
|
|
16225
|
+
const overlap = terms.filter((term) => targetTerms.has(term)).length;
|
|
16226
|
+
const covered = targetHeadingText.has(normalized) || overlap >= Math.max(2, Math.ceil(terms.length / 2));
|
|
16227
|
+
if (covered && targetPage) continue;
|
|
16228
|
+
rows.push({
|
|
16229
|
+
source_position: source.position ?? "",
|
|
16230
|
+
source_domain: normalizeDomain2(source.domain ?? "") ?? domainFromUrl2(source.url),
|
|
16231
|
+
source_url: source.url ?? "",
|
|
16232
|
+
heading_level: heading.level,
|
|
16233
|
+
competitor_heading: heading.text,
|
|
16234
|
+
target_coverage: targetPage ? "not found in target headings" : "no target page extracted",
|
|
16235
|
+
terms: textTerms(heading.text, 6)
|
|
16236
|
+
});
|
|
16237
|
+
}
|
|
16238
|
+
}
|
|
16239
|
+
return rows.slice(0, 150);
|
|
16240
|
+
}
|
|
16241
|
+
async function extractPages(ctx, sources, warnings, labelPrefix) {
|
|
16242
|
+
return mapLimit3(sources.filter((source) => source.url), 2, async (source, index) => {
|
|
16243
|
+
try {
|
|
16244
|
+
const page = await ctx.client.post("/extract-url", { url: source.url }, 18e4);
|
|
16245
|
+
await ctx.artifacts.writeJson(`${labelPrefix} ${index + 1}`, `raw/extract-url/${labelPrefix.toLowerCase()}-${index + 1}.json`, page);
|
|
16246
|
+
return { source, page };
|
|
16247
|
+
} catch (err) {
|
|
16248
|
+
warnings.push(`Page extraction failed for ${source.url}: ${err instanceof Error ? err.message : String(err)}`);
|
|
16249
|
+
return null;
|
|
16250
|
+
}
|
|
16251
|
+
}).then((items) => items.filter((item) => item !== null));
|
|
16252
|
+
}
|
|
16253
|
+
var import_zod22, ProxyModeSchema, MapComparisonInputSchema, SerpComparisonInputSchema, PaaExpansionBriefInputSchema, AiOverviewLanguageInputSchema, mapComparisonWorkflowDefinition, serpComparisonWorkflowDefinition, paaExpansionBriefWorkflowDefinition, aiOverviewLanguageWorkflowDefinition;
|
|
16254
|
+
var init_comparison_briefs = __esm({
|
|
16255
|
+
"src/workflows/workflows/comparison-briefs.ts"() {
|
|
16256
|
+
"use strict";
|
|
16257
|
+
import_zod22 = require("zod");
|
|
16258
|
+
init_report_renderer();
|
|
16259
|
+
init_directory();
|
|
16260
|
+
init_seo_workflow_utils();
|
|
16261
|
+
ProxyModeSchema = import_zod22.z.enum(["location", "configured", "none"]);
|
|
16262
|
+
MapComparisonInputSchema = import_zod22.z.object({
|
|
16263
|
+
query: import_zod22.z.string().min(1),
|
|
16264
|
+
location: import_zod22.z.string().optional(),
|
|
16265
|
+
state: import_zod22.z.string().optional(),
|
|
16266
|
+
minPopulation: import_zod22.z.number().int().min(0).default(1e5),
|
|
16267
|
+
maxCities: import_zod22.z.number().int().min(1).max(100).default(5),
|
|
16268
|
+
maxResultsPerCity: import_zod22.z.number().int().min(1).max(50).default(20),
|
|
16269
|
+
hydrateTop: import_zod22.z.number().int().min(0).max(10).default(5),
|
|
16270
|
+
maxReviews: import_zod22.z.number().int().min(0).max(500).default(25),
|
|
16271
|
+
concurrency: import_zod22.z.number().int().min(1).max(5).default(5),
|
|
16272
|
+
proxyMode: ProxyModeSchema.default("location"),
|
|
16273
|
+
returnPartial: import_zod22.z.boolean().default(true)
|
|
16274
|
+
}).refine((input) => input.location || input.state, {
|
|
16275
|
+
message: "Either location or state is required for map-comparison"
|
|
16276
|
+
});
|
|
16277
|
+
SerpComparisonInputSchema = import_zod22.z.object({
|
|
16278
|
+
keyword: import_zod22.z.string().min(1),
|
|
16279
|
+
domain: import_zod22.z.string().optional(),
|
|
16280
|
+
url: import_zod22.z.string().url().optional(),
|
|
16281
|
+
location: import_zod22.z.string().optional(),
|
|
16282
|
+
maxResults: import_zod22.z.number().int().min(1).max(20).default(10),
|
|
16283
|
+
maxQuestions: import_zod22.z.number().int().min(1).max(200).default(40),
|
|
16284
|
+
extractTop: import_zod22.z.number().int().min(0).max(10).default(5),
|
|
16285
|
+
includePaa: import_zod22.z.boolean().default(true),
|
|
16286
|
+
includeAiOverview: import_zod22.z.boolean().default(true),
|
|
16287
|
+
returnPartial: import_zod22.z.boolean().default(true)
|
|
16288
|
+
});
|
|
16289
|
+
PaaExpansionBriefInputSchema = import_zod22.z.object({
|
|
16290
|
+
keyword: import_zod22.z.string().min(1),
|
|
16291
|
+
location: import_zod22.z.string().optional(),
|
|
16292
|
+
maxQuestions: import_zod22.z.number().int().min(1).max(300).default(80),
|
|
16293
|
+
depth: import_zod22.z.number().int().min(1).max(6).default(3),
|
|
16294
|
+
returnPartial: import_zod22.z.boolean().default(true)
|
|
16295
|
+
});
|
|
16296
|
+
AiOverviewLanguageInputSchema = import_zod22.z.object({
|
|
16297
|
+
keyword: import_zod22.z.string().min(1),
|
|
16298
|
+
domain: import_zod22.z.string().optional(),
|
|
16299
|
+
url: import_zod22.z.string().url().optional(),
|
|
16300
|
+
location: import_zod22.z.string().optional(),
|
|
16301
|
+
maxQuestions: import_zod22.z.number().int().min(1).max(200).default(40),
|
|
16302
|
+
extractTop: import_zod22.z.number().int().min(0).max(8).default(3),
|
|
16303
|
+
returnPartial: import_zod22.z.boolean().default(true)
|
|
16304
|
+
});
|
|
16305
|
+
mapComparisonWorkflowDefinition = {
|
|
16306
|
+
id: "map-comparison",
|
|
16307
|
+
title: "Maps Comparison",
|
|
16308
|
+
description: "Compare Google Maps competitors by rank, reviews, stars, categories, websites, and profile/review signals.",
|
|
16309
|
+
inputSchema: MapComparisonInputSchema,
|
|
16310
|
+
async run(input, ctx) {
|
|
16311
|
+
await ctx.artifacts.writeManifest("running", {}, [], []);
|
|
16312
|
+
const warnings = [];
|
|
16313
|
+
let rows;
|
|
16314
|
+
let directory = null;
|
|
16315
|
+
let mapsSearch = null;
|
|
16316
|
+
if (input.location) {
|
|
16317
|
+
mapsSearch = await ctx.client.post("/maps/search", {
|
|
16318
|
+
query: input.query,
|
|
16319
|
+
location: input.location,
|
|
16320
|
+
maxResults: input.maxResultsPerCity,
|
|
16321
|
+
proxyMode: input.proxyMode
|
|
16322
|
+
}, 24e4);
|
|
16323
|
+
rows = businessRowsFromMaps(input.location, input.query, mapsSearch.results);
|
|
16324
|
+
await ctx.artifacts.writeJson("Maps search raw JSON", "raw/maps-search.json", mapsSearch);
|
|
16325
|
+
} else {
|
|
16326
|
+
directory = await ctx.client.post("/directory/run", {
|
|
16327
|
+
query: input.query,
|
|
16328
|
+
state: input.state,
|
|
16329
|
+
minPopulation: input.minPopulation,
|
|
16330
|
+
maxCities: input.maxCities,
|
|
16331
|
+
maxResultsPerCity: input.maxResultsPerCity,
|
|
16332
|
+
concurrency: input.concurrency,
|
|
16333
|
+
proxyMode: input.proxyMode,
|
|
16334
|
+
saveCsv: true
|
|
16335
|
+
}, 9e5);
|
|
16336
|
+
rows = directoryRows(directory);
|
|
16337
|
+
await ctx.artifacts.writeJson("Directory raw JSON", "raw/directory-workflow.json", directory);
|
|
16338
|
+
await ctx.artifacts.writeCsv("Directory CSV", "exports/directory.csv", DIRECTORY_CSV_HEADERS, rows);
|
|
16339
|
+
warnings.push(...directory.warnings);
|
|
16340
|
+
}
|
|
16341
|
+
const compareRows = comparisonRows(rows);
|
|
16342
|
+
const selected = compareRows.slice(0, input.hydrateTop * Math.max(1, input.location ? 1 : input.maxCities));
|
|
16343
|
+
const hydrated = await mapLimit3(selected, 3, async (row, index) => {
|
|
16344
|
+
try {
|
|
16345
|
+
const detail = await ctx.client.post("/maps/place", {
|
|
16346
|
+
businessName: row.business_name,
|
|
16347
|
+
location: row.source_location,
|
|
16348
|
+
includeReviews: input.maxReviews > 0,
|
|
16349
|
+
maxReviews: Math.max(1, input.maxReviews)
|
|
16350
|
+
}, 18e4);
|
|
16351
|
+
await ctx.artifacts.writeJson(`${row.business_name} profile`, `raw/maps-place-intel/${index + 1}-${String(row.business_name).toLowerCase().replace(/[^a-z0-9]+/g, "-")}.json`, detail);
|
|
16352
|
+
return { row, detail, error: "" };
|
|
16353
|
+
} catch (err) {
|
|
16354
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
16355
|
+
warnings.push(`Profile hydration failed for ${row.business_name}: ${message}`);
|
|
16356
|
+
return { row, detail: null, error: message };
|
|
16357
|
+
}
|
|
16358
|
+
});
|
|
16359
|
+
const profileRows = hydrated.map(({ row, detail, error }) => ({
|
|
16360
|
+
source_location: row.source_location,
|
|
16361
|
+
result_position: row.result_position,
|
|
16362
|
+
business_name: row.business_name,
|
|
16363
|
+
category: detail?.category ?? row.category,
|
|
16364
|
+
review_stars: detail?.rating ?? row.review_stars,
|
|
16365
|
+
review_count: detail?.reviewCount ?? row.review_count,
|
|
16366
|
+
website_url: detail?.website ?? row.website_url,
|
|
16367
|
+
review_topics: (detail?.reviewTopics ?? []).map((t) => `${t.label} (${t.count})`).join("; "),
|
|
16368
|
+
about_attributes: (detail?.aboutAttributes ?? []).map((a) => `${a.section}: ${a.attribute}`).join("; "),
|
|
16369
|
+
reviews_status: detail?.reviewsStatus ?? "",
|
|
16370
|
+
error
|
|
16371
|
+
}));
|
|
16372
|
+
const markets = marketRows(rows);
|
|
16373
|
+
await ctx.artifacts.writeCsv("Maps results CSV", "maps-results.csv", ["source_query", "source_location", "city", "state", "population", "result_position", "business_name", "review_stars", "review_count", "category", "address", "phone", "website_url", "place_url", "cid", "cid_decimal", "result_status", "error"], rows);
|
|
16374
|
+
await ctx.artifacts.writeCsv("Comparison CSV", "map-comparison.csv", ["source_location", "result_position", "business_name", "category", "review_stars", "review_count", "review_gap_to_top3_average", "website_url", "place_url", "comparison_note"], compareRows);
|
|
16375
|
+
await ctx.artifacts.writeCsv("Profile insights CSV", "profile-insights.csv", ["source_location", "result_position", "business_name", "category", "review_stars", "review_count", "website_url", "review_topics", "about_attributes", "reviews_status", "error"], profileRows);
|
|
16376
|
+
await ctx.artifacts.writeJson("Maps comparison evidence", "evidence.json", { input, directory, mapsSearch, rows, compareRows, profileRows, markets, warnings });
|
|
16377
|
+
await ctx.artifacts.writeText("Brief", "brief.md", [
|
|
16378
|
+
`# Maps Comparison: ${input.query}`,
|
|
16379
|
+
"",
|
|
16380
|
+
`Markets: ${markets.map((row) => row.source_location).join(", ")}`,
|
|
16381
|
+
"",
|
|
16382
|
+
"## How to Use",
|
|
16383
|
+
"- Compare rank position against review count and category patterns.",
|
|
16384
|
+
"- Treat review gaps and missing websites as opportunity signals, not guarantees.",
|
|
16385
|
+
"- Use profile topics and attributes as evidence for local content and GBP improvements."
|
|
16386
|
+
].join("\n"));
|
|
16387
|
+
const summary = `${compareRows.length} Maps competitors compared across ${markets.length} market(s); ${profileRows.filter((row) => !row.error).length} profiles hydrated.`;
|
|
16388
|
+
const reportPath = await ctx.artifacts.writeHtml("HTML report", "report.html", renderWorkflowReport({
|
|
16389
|
+
title: "Maps Comparison",
|
|
16390
|
+
subtitle: `${input.query}${input.location ? ` \xB7 ${input.location}` : input.state ? ` \xB7 ${input.state}` : ""}`,
|
|
16391
|
+
summary,
|
|
16392
|
+
warnings,
|
|
16393
|
+
tables: [
|
|
16394
|
+
{ title: "Market Benchmarks", columns: ["source_location", "result_count", "median_review_count", "median_rating", "top_three_average_review_count", "top_categories", "websites_present"], rows: markets },
|
|
16395
|
+
{ title: "Competitor Comparison", columns: ["source_location", "result_position", "business_name", "category", "review_stars", "review_count", "review_gap_to_top3_average", "comparison_note"], rows: compareRows.slice(0, 120) },
|
|
16396
|
+
{ title: "Profile Insights", columns: ["source_location", "business_name", "review_topics", "about_attributes", "error"], rows: profileRows }
|
|
16397
|
+
]
|
|
16398
|
+
}));
|
|
16399
|
+
const status = warnings.length ? "partial" : "succeeded";
|
|
16400
|
+
const counts = { markets: markets.length, competitors: compareRows.length, hydratedProfiles: profileRows.filter((row) => !row.error).length };
|
|
16401
|
+
await ctx.artifacts.writeManifest(status, counts, warnings, []);
|
|
16402
|
+
return { title: "Maps Comparison", summary, status, counts, warnings, errors: [], reportPath };
|
|
16403
|
+
}
|
|
16404
|
+
};
|
|
16405
|
+
serpComparisonWorkflowDefinition = {
|
|
16406
|
+
id: "serp-comparison",
|
|
16407
|
+
title: "SERP Comparison",
|
|
16408
|
+
description: "Compare ranking pages, SERP features, PAA evidence, AI Overview citations, and page-level content gaps.",
|
|
16409
|
+
inputSchema: SerpComparisonInputSchema,
|
|
16410
|
+
async run(input, ctx) {
|
|
16411
|
+
await ctx.artifacts.writeManifest("running", {}, [], []);
|
|
16412
|
+
const warnings = [];
|
|
16413
|
+
const targetDomain = normalizeDomain2(input.domain ?? input.url ?? null);
|
|
16414
|
+
const serp = await ctx.client.post("/harvest/sync", {
|
|
16415
|
+
query: input.keyword,
|
|
16416
|
+
location: input.location,
|
|
16417
|
+
serpOnly: true,
|
|
16418
|
+
maxQuestions: 1,
|
|
16419
|
+
format: "json"
|
|
16420
|
+
}, 18e4);
|
|
16421
|
+
await ctx.artifacts.writeJson("SERP raw JSON", "raw/serp.json", serp);
|
|
16422
|
+
let paa = null;
|
|
16423
|
+
if (input.includePaa) {
|
|
16424
|
+
try {
|
|
16425
|
+
paa = await ctx.client.post("/harvest/sync", {
|
|
16426
|
+
query: input.keyword,
|
|
16427
|
+
location: input.location,
|
|
16428
|
+
maxQuestions: input.maxQuestions,
|
|
16429
|
+
format: "json"
|
|
16430
|
+
}, 28e4);
|
|
16431
|
+
await ctx.artifacts.writeJson("PAA raw JSON", "raw/paa.json", paa);
|
|
16432
|
+
} catch (err) {
|
|
16433
|
+
warnings.push(`PAA evidence unavailable: ${err instanceof Error ? err.message : String(err)}`);
|
|
16434
|
+
}
|
|
16435
|
+
}
|
|
16436
|
+
const organic = organicRows(serp, targetDomain).slice(0, input.maxResults);
|
|
16437
|
+
const organicSources = (serp.organicResults ?? []).slice(0, input.maxResults);
|
|
16438
|
+
const targetSource = input.url ? { position: 0, title: "Target page", url: input.url, domain: domainFromUrl2(input.url) } : organicSources.find((result) => targetDomain && (normalizeDomain2(result.domain ?? "") ?? domainFromUrl2(result.url)) === targetDomain);
|
|
16439
|
+
const competitorSources = organicSources.filter((result) => result.url && result.url !== targetSource?.url).filter((result) => !targetDomain || (normalizeDomain2(result.domain ?? "") ?? domainFromUrl2(result.url)) !== targetDomain).slice(0, input.extractTop);
|
|
16440
|
+
const targetPages = targetSource ? await extractPages(ctx, [targetSource], warnings, "Target") : [];
|
|
16441
|
+
const competitorPages = input.extractTop > 0 ? await extractPages(ctx, competitorSources, warnings, "Competitor") : [];
|
|
16442
|
+
const targetPage = targetPages[0]?.page ?? null;
|
|
16443
|
+
const pageRows = [
|
|
16444
|
+
...targetPages.map(({ source, page }) => pageSummaryRow(page, { url: source.url ?? "", domain: normalizeDomain2(source.domain ?? "") ?? domainFromUrl2(source.url), position: source.position, title: source.title })),
|
|
16445
|
+
...competitorPages.map(({ source, page }) => pageSummaryRow(page, { url: source.url ?? "", domain: normalizeDomain2(source.domain ?? "") ?? domainFromUrl2(source.url), position: source.position, title: source.title }))
|
|
16446
|
+
];
|
|
16447
|
+
const gaps = pageGapRows(targetPage, competitorPages);
|
|
16448
|
+
const questions = questionRows(paa);
|
|
16449
|
+
const aiRows = (serp.aiOverview?.citations ?? []).map((citation, index) => ({
|
|
16450
|
+
citation_position: index + 1,
|
|
16451
|
+
citation_text: citation.text ?? "",
|
|
16452
|
+
url: citation.href ?? "",
|
|
16453
|
+
domain: domainFromUrl2(citation.href ?? ""),
|
|
16454
|
+
is_target: targetDomain ? domainFromUrl2(citation.href ?? "") === targetDomain : false
|
|
16455
|
+
}));
|
|
16456
|
+
await ctx.artifacts.writeCsv("Organic results CSV", "organic-results.csv", ["position", "title", "url", "domain", "snippet", "is_target"], organic);
|
|
16457
|
+
await ctx.artifacts.writeCsv("Page comparison CSV", "page-comparison.csv", ["position", "domain", "url", "serp_title", "page_title", "h1", "meta_description", "word_count", "heading_count", "schema_types"], pageRows);
|
|
16458
|
+
await ctx.artifacts.writeCsv("Content gaps CSV", "content-gaps.csv", ["source_position", "source_domain", "source_url", "heading_level", "competitor_heading", "target_coverage", "terms"], gaps);
|
|
16459
|
+
await ctx.artifacts.writeCsv("PAA questions CSV", "paa-questions.csv", ["position", "intent", "question", "answer_excerpt", "source_title", "source_domain", "source_url"], questions);
|
|
16460
|
+
await ctx.artifacts.writeCsv("AI Overview citations CSV", "ai-overview-citations.csv", ["citation_position", "citation_text", "url", "domain", "is_target"], aiRows);
|
|
16461
|
+
await ctx.artifacts.writeJson("SERP comparison evidence", "evidence.json", { input, serp, paa, organic, pageRows, gaps, questions, aiRows, warnings });
|
|
16462
|
+
await ctx.artifacts.writeText("Writer brief", "brief.md", [
|
|
16463
|
+
`# SERP Comparison Brief: ${input.keyword}`,
|
|
16464
|
+
"",
|
|
16465
|
+
`Target: ${targetDomain ?? input.url ?? "not specified"}`,
|
|
16466
|
+
`Location: ${input.location ?? "not specified"}`,
|
|
16467
|
+
"",
|
|
16468
|
+
"## Recommended Actions",
|
|
16469
|
+
"- Use `content-gaps.csv` to decide which missing sections deserve coverage.",
|
|
16470
|
+
"- Use `paa-questions.csv` for FAQ and answer-block candidates.",
|
|
16471
|
+
"- Use `ai-overview-citations.csv` to see whether the target is cited in AI Overview evidence.",
|
|
16472
|
+
"- Treat extracted page headings as evidence, not a complete semantic analysis."
|
|
16473
|
+
].join("\n"));
|
|
16474
|
+
const summary = `${organic.length} organic results, ${pageRows.length} extracted pages, ${gaps.length} heading gaps, ${questions.length} PAA questions.`;
|
|
16475
|
+
const reportPath = await ctx.artifacts.writeHtml("HTML report", "report.html", renderWorkflowReport({
|
|
16476
|
+
title: "SERP Comparison",
|
|
16477
|
+
subtitle: `${input.keyword}${input.location ? ` \xB7 ${input.location}` : ""}`,
|
|
16478
|
+
summary,
|
|
16479
|
+
warnings,
|
|
16480
|
+
tables: [
|
|
16481
|
+
{ title: "Organic Results", columns: ["position", "title", "domain", "is_target"], rows: organic },
|
|
16482
|
+
{ title: "Page Comparison", columns: ["position", "domain", "h1", "word_count", "heading_count", "schema_types"], rows: pageRows },
|
|
16483
|
+
{ title: "Content Gaps", columns: ["source_position", "source_domain", "competitor_heading", "target_coverage", "terms"], rows: gaps.slice(0, 80) },
|
|
16484
|
+
{ title: "PAA Questions", columns: ["position", "intent", "question", "source_domain"], rows: questions.slice(0, 80) }
|
|
16485
|
+
]
|
|
16486
|
+
}));
|
|
16487
|
+
const status = warnings.length ? "partial" : "succeeded";
|
|
16488
|
+
const counts = { organic: organic.length, pages: pageRows.length, gaps: gaps.length, questions: questions.length, aiCitations: aiRows.length };
|
|
16489
|
+
await ctx.artifacts.writeManifest(status, counts, warnings, []);
|
|
16490
|
+
return { title: "SERP Comparison", summary, status, counts, warnings, errors: [], reportPath };
|
|
16491
|
+
}
|
|
16492
|
+
};
|
|
16493
|
+
paaExpansionBriefWorkflowDefinition = {
|
|
16494
|
+
id: "paa-expansion-brief",
|
|
16495
|
+
title: "PAA Expansion Brief",
|
|
16496
|
+
description: "Expand People Also Ask questions into an evidence-backed writer brief, section map, and source table.",
|
|
16497
|
+
inputSchema: PaaExpansionBriefInputSchema,
|
|
16498
|
+
async run(input, ctx) {
|
|
16499
|
+
await ctx.artifacts.writeManifest("running", {}, [], []);
|
|
16500
|
+
const warnings = [];
|
|
16501
|
+
const paa = await ctx.client.post("/harvest/sync", {
|
|
16502
|
+
query: input.keyword,
|
|
16503
|
+
location: input.location,
|
|
16504
|
+
maxQuestions: input.maxQuestions,
|
|
16505
|
+
depth: input.depth,
|
|
16506
|
+
format: "json"
|
|
16507
|
+
}, 3e5);
|
|
16508
|
+
await ctx.artifacts.writeJson("PAA raw JSON", "raw/paa.json", paa);
|
|
16509
|
+
const questions = questionRows(paa);
|
|
16510
|
+
const sourceRows2 = sourceDomainRows(questions);
|
|
16511
|
+
const byIntent = /* @__PURE__ */ new Map();
|
|
16512
|
+
for (const row of questions) {
|
|
16513
|
+
const list = byIntent.get(String(row.intent)) ?? [];
|
|
16514
|
+
list.push(row);
|
|
16515
|
+
byIntent.set(String(row.intent), list);
|
|
16516
|
+
}
|
|
16517
|
+
const sectionRows = [...byIntent.entries()].map(([intent, rows]) => ({
|
|
16518
|
+
recommended_section: intent,
|
|
16519
|
+
question_count: rows.length,
|
|
16520
|
+
sample_questions: rows.slice(0, 5).map((row) => row.question).join(" | "),
|
|
16521
|
+
source_domains: [...new Set(rows.map((row) => row.source_domain).filter(Boolean))].slice(0, 5).join("; "),
|
|
16522
|
+
terms: textTerms(rows.map((row) => `${row.question} ${row.answer_excerpt}`).join(" "), 10)
|
|
16523
|
+
})).sort((a, b) => Number(b.question_count) - Number(a.question_count));
|
|
16524
|
+
await ctx.artifacts.writeCsv("PAA questions CSV", "paa-questions.csv", ["position", "intent", "question", "answer_excerpt", "source_title", "source_domain", "source_url"], questions);
|
|
16525
|
+
await ctx.artifacts.writeCsv("Source domains CSV", "source-domains.csv", ["domain", "question_mentions", "source_url_count", "top_intents"], sourceRows2);
|
|
16526
|
+
await ctx.artifacts.writeCsv("Section map CSV", "section-map.csv", ["recommended_section", "question_count", "sample_questions", "source_domains", "terms"], sectionRows);
|
|
16527
|
+
await ctx.artifacts.writeJson("PAA brief evidence", "evidence.json", { input, paa, questions, sourceRows: sourceRows2, sectionRows });
|
|
16528
|
+
await ctx.artifacts.writeText("Writer brief", "writer-brief.md", [
|
|
16529
|
+
`# PAA Expansion Brief: ${input.keyword}`,
|
|
16530
|
+
"",
|
|
16531
|
+
`Location: ${input.location ?? "not specified"}`,
|
|
16532
|
+
"",
|
|
16533
|
+
"## Suggested Page Structure",
|
|
16534
|
+
...sectionRows.map((row) => `- ${row.recommended_section}: answer ${row.question_count} related question(s). Sample: ${row.sample_questions}`),
|
|
16535
|
+
"",
|
|
16536
|
+
"## Writing Rules",
|
|
16537
|
+
"- Answer the highest-frequency question in the first 60 words of each section.",
|
|
16538
|
+
"- Use exact customer question language from `paa-questions.csv` for H2/H3 candidates.",
|
|
16539
|
+
"- Use `source-domains.csv` to identify which source types Google is already rewarding.",
|
|
16540
|
+
"- Do not invent citations; cite only rows that have a source URL."
|
|
16541
|
+
].join("\n"));
|
|
16542
|
+
const summary = `${questions.length} PAA questions grouped into ${sectionRows.length} writing sections.`;
|
|
16543
|
+
const reportPath = await ctx.artifacts.writeHtml("HTML report", "report.html", renderWorkflowReport({
|
|
16544
|
+
title: "PAA Expansion Brief",
|
|
16545
|
+
subtitle: `${input.keyword}${input.location ? ` \xB7 ${input.location}` : ""}`,
|
|
16546
|
+
summary,
|
|
16547
|
+
warnings,
|
|
16548
|
+
tables: [
|
|
16549
|
+
{ title: "Section Map", columns: ["recommended_section", "question_count", "sample_questions", "source_domains", "terms"], rows: sectionRows },
|
|
16550
|
+
{ title: "Questions", columns: ["position", "intent", "question", "source_domain"], rows: questions.slice(0, 120) },
|
|
16551
|
+
{ title: "Source Domains", columns: ["domain", "question_mentions", "source_url_count", "top_intents"], rows: sourceRows2 }
|
|
16552
|
+
]
|
|
16553
|
+
}));
|
|
16554
|
+
const counts = { questions: questions.length, sections: sectionRows.length, sourceDomains: sourceRows2.length };
|
|
16555
|
+
await ctx.artifacts.writeManifest("succeeded", counts, warnings, []);
|
|
16556
|
+
return { title: "PAA Expansion Brief", summary, status: "succeeded", counts, warnings, errors: [], reportPath };
|
|
16557
|
+
}
|
|
16558
|
+
};
|
|
16559
|
+
aiOverviewLanguageWorkflowDefinition = {
|
|
16560
|
+
id: "ai-overview-language",
|
|
16561
|
+
title: "AI Overview Language Brief",
|
|
16562
|
+
description: "Turn AI Overview, citation, PAA, and ranking-page evidence into answer-block and citation-hook guidance.",
|
|
16563
|
+
inputSchema: AiOverviewLanguageInputSchema,
|
|
16564
|
+
async run(input, ctx) {
|
|
16565
|
+
await ctx.artifacts.writeManifest("running", {}, [], []);
|
|
16566
|
+
const warnings = [];
|
|
16567
|
+
const targetDomain = normalizeDomain2(input.domain ?? input.url ?? null);
|
|
16568
|
+
const serp = await ctx.client.post("/harvest/sync", {
|
|
16569
|
+
query: input.keyword,
|
|
16570
|
+
location: input.location,
|
|
16571
|
+
serpOnly: true,
|
|
16572
|
+
maxQuestions: 1,
|
|
16573
|
+
format: "json"
|
|
16574
|
+
}, 18e4);
|
|
16575
|
+
await ctx.artifacts.writeJson("SERP raw JSON", "raw/serp.json", serp);
|
|
16576
|
+
let paa = null;
|
|
16577
|
+
try {
|
|
16578
|
+
paa = await ctx.client.post("/harvest/sync", {
|
|
16579
|
+
query: input.keyword,
|
|
16580
|
+
location: input.location,
|
|
16581
|
+
maxQuestions: input.maxQuestions,
|
|
16582
|
+
format: "json"
|
|
16583
|
+
}, 28e4);
|
|
16584
|
+
await ctx.artifacts.writeJson("PAA raw JSON", "raw/paa.json", paa);
|
|
16585
|
+
} catch (err) {
|
|
16586
|
+
warnings.push(`PAA evidence unavailable: ${err instanceof Error ? err.message : String(err)}`);
|
|
16587
|
+
}
|
|
16588
|
+
const citations = (serp.aiOverview?.citations ?? []).map((citation, index) => {
|
|
16589
|
+
const domain = domainFromUrl2(citation.href ?? "");
|
|
16590
|
+
return {
|
|
16591
|
+
citation_position: index + 1,
|
|
16592
|
+
citation_text: citation.text ?? "",
|
|
16593
|
+
url: citation.href ?? "",
|
|
16594
|
+
domain,
|
|
16595
|
+
is_target: targetDomain ? domain === targetDomain : false
|
|
16596
|
+
};
|
|
16597
|
+
});
|
|
16598
|
+
const aioSentences = splitSentences(serp.aiOverview?.text);
|
|
16599
|
+
const claimRows = aioSentences.map((sentence, index) => ({
|
|
16600
|
+
position: index + 1,
|
|
16601
|
+
claim_type: classifySentence(sentence),
|
|
16602
|
+
sentence,
|
|
16603
|
+
reusable_pattern: sentence.length > 140 ? `${sentence.slice(0, 140)}...` : sentence
|
|
16604
|
+
}));
|
|
16605
|
+
const questions = questionRows(paa);
|
|
16606
|
+
const citationSources = (serp.aiOverview?.citations ?? []).filter((citation) => citation.href).map((citation, index) => ({ position: index + 1, title: citation.text, url: citation.href, domain: domainFromUrl2(citation.href) })).slice(0, input.extractTop);
|
|
16607
|
+
const extractedCitations = input.extractTop > 0 ? await extractPages(ctx, citationSources, warnings, "Citation") : [];
|
|
16608
|
+
const extractedRows = extractedCitations.map(({ source, page }) => pageSummaryRow(page, { url: source.url ?? "", domain: normalizeDomain2(source.domain ?? "") ?? domainFromUrl2(source.url), position: source.position, title: source.title }));
|
|
16609
|
+
const languageRows = [
|
|
16610
|
+
{
|
|
16611
|
+
block: "direct_answer",
|
|
16612
|
+
guidance: "Open with a 40-70 word answer that directly resolves the query before adding context.",
|
|
16613
|
+
evidence_basis: questions[0]?.question ?? input.keyword
|
|
16614
|
+
},
|
|
16615
|
+
{
|
|
16616
|
+
block: "criteria_or_steps",
|
|
16617
|
+
guidance: "List the criteria, steps, or decision factors Google is already compressing into AI Overview language.",
|
|
16618
|
+
evidence_basis: claimRows.filter((row) => ["criteria", "process"].includes(String(row.claim_type))).map((row) => row.sentence).slice(0, 3).join(" | ")
|
|
16619
|
+
},
|
|
16620
|
+
{
|
|
16621
|
+
block: "citation_hook",
|
|
16622
|
+
guidance: "Add source-worthy details competitors can cite: definitions, numbers, examples, process details, and named entity relationships.",
|
|
16623
|
+
evidence_basis: citations.map((row) => `${row.domain}: ${row.citation_text}`).slice(0, 5).join(" | ")
|
|
16624
|
+
},
|
|
16625
|
+
{
|
|
16626
|
+
block: "faq_followups",
|
|
16627
|
+
guidance: "Use PAA phrasing for follow-up sections so the page answers adjacent questions in Google language.",
|
|
16628
|
+
evidence_basis: questions.slice(0, 5).map((row) => row.question).join(" | ")
|
|
16629
|
+
}
|
|
16630
|
+
];
|
|
16631
|
+
await ctx.artifacts.writeCsv("AI Overview citations CSV", "ai-overview-citations.csv", ["citation_position", "citation_text", "url", "domain", "is_target"], citations);
|
|
16632
|
+
await ctx.artifacts.writeCsv("AI Overview claim patterns CSV", "claim-patterns.csv", ["position", "claim_type", "sentence", "reusable_pattern"], claimRows);
|
|
16633
|
+
await ctx.artifacts.writeCsv("Language guidance CSV", "language-guidance.csv", ["block", "guidance", "evidence_basis"], languageRows);
|
|
16634
|
+
await ctx.artifacts.writeCsv("PAA questions CSV", "paa-questions.csv", ["position", "intent", "question", "answer_excerpt", "source_title", "source_domain", "source_url"], questions);
|
|
16635
|
+
await ctx.artifacts.writeCsv("Extracted citation pages CSV", "citation-pages.csv", ["position", "domain", "url", "serp_title", "page_title", "h1", "meta_description", "word_count", "heading_count", "schema_types"], extractedRows);
|
|
16636
|
+
await ctx.artifacts.writeJson("AI Overview language evidence", "evidence.json", { input, serp, paa, citations, claimRows, languageRows, extractedRows, warnings });
|
|
16637
|
+
await ctx.artifacts.writeText("Answer block template", "answer-block-template.md", [
|
|
16638
|
+
`# AI Overview Language Brief: ${input.keyword}`,
|
|
16639
|
+
"",
|
|
16640
|
+
`AI Overview detected: ${serp.aiOverview?.detected ? "yes" : "no"}`,
|
|
16641
|
+
`Target cited: ${targetDomain ? citations.some((row) => row.is_target) ? "yes" : "no" : "target not specified"}`,
|
|
16642
|
+
"",
|
|
16643
|
+
"## Direct Answer Block",
|
|
16644
|
+
"Write one compact answer block that starts with the answer, not background. Keep it clear enough that Google could lift it as a standalone summary.",
|
|
16645
|
+
"",
|
|
16646
|
+
"## Suggested Follow-Up Blocks",
|
|
16647
|
+
...languageRows.map((row) => `- ${row.block}: ${row.guidance}`),
|
|
16648
|
+
"",
|
|
16649
|
+
"## Evidence to Mirror",
|
|
16650
|
+
...claimRows.slice(0, 8).map((row) => `- ${row.claim_type}: ${row.sentence}`),
|
|
16651
|
+
"",
|
|
16652
|
+
"## Citation Hooks",
|
|
16653
|
+
...citations.slice(0, 8).map((row) => `- ${row.domain}: ${row.citation_text}`)
|
|
16654
|
+
].join("\n"));
|
|
16655
|
+
const summary = `${citations.length} AI Overview citations, ${claimRows.length} claim patterns, ${questions.length} PAA questions, ${extractedRows.length} citation pages extracted.`;
|
|
16656
|
+
const reportPath = await ctx.artifacts.writeHtml("HTML report", "report.html", renderWorkflowReport({
|
|
16657
|
+
title: "AI Overview Language Brief",
|
|
16658
|
+
subtitle: `${input.keyword}${input.location ? ` \xB7 ${input.location}` : ""}`,
|
|
16659
|
+
summary,
|
|
16660
|
+
warnings,
|
|
16661
|
+
tables: [
|
|
16662
|
+
{ title: "Language Guidance", columns: ["block", "guidance", "evidence_basis"], rows: languageRows },
|
|
16663
|
+
{ title: "AI Overview Citations", columns: ["citation_position", "citation_text", "domain", "is_target"], rows: citations },
|
|
16664
|
+
{ title: "Claim Patterns", columns: ["position", "claim_type", "sentence"], rows: claimRows },
|
|
16665
|
+
{ title: "PAA Follow-Ups", columns: ["position", "intent", "question", "source_domain"], rows: questions.slice(0, 60) }
|
|
16666
|
+
]
|
|
16667
|
+
}));
|
|
16668
|
+
const status = warnings.length || !serp.aiOverview?.detected ? "partial" : "succeeded";
|
|
16669
|
+
const counts = { citations: citations.length, claimPatterns: claimRows.length, questions: questions.length, extractedCitationPages: extractedRows.length };
|
|
16670
|
+
await ctx.artifacts.writeManifest(status, counts, warnings, []);
|
|
16671
|
+
return { title: "AI Overview Language Brief", summary, status, counts, warnings, errors: [], reportPath };
|
|
16672
|
+
}
|
|
16673
|
+
};
|
|
16674
|
+
}
|
|
16675
|
+
});
|
|
16676
|
+
|
|
15649
16677
|
// src/workflows/registry.ts
|
|
15650
16678
|
function listWorkflowDefinitions() {
|
|
15651
16679
|
return DEFINITIONS.map(({ id, title, description }) => ({ id, title, description }));
|
|
@@ -15662,7 +16690,7 @@ async function runWorkflow(id, rawInput, options = {}) {
|
|
|
15662
16690
|
if (!apiKey) throw new Error("MCP_SCRAPER_API_KEY is required for workflow runs. Pass --api-key or set the environment variable.");
|
|
15663
16691
|
const apiUrl = options.apiUrl?.trim() || process.env.MCP_SCRAPER_API_URL?.trim() || "https://mcpscraper.dev";
|
|
15664
16692
|
const artifacts = await ArtifactWriter.create(definition.id, definition.title, input, options.outputDir, options.runId);
|
|
15665
|
-
const client2 = new WorkflowHttpClient(apiUrl, apiKey, options.fetchImpl);
|
|
16693
|
+
const client2 = new WorkflowHttpClient(apiUrl, apiKey, options.fetchImpl, options.headers);
|
|
15666
16694
|
try {
|
|
15667
16695
|
return await definition.run(input, {
|
|
15668
16696
|
runId: artifacts.runId,
|
|
@@ -15681,20 +16709,25 @@ ${message}
|
|
|
15681
16709
|
throw err;
|
|
15682
16710
|
}
|
|
15683
16711
|
}
|
|
15684
|
-
var
|
|
16712
|
+
var import_zod23, DEFINITIONS;
|
|
15685
16713
|
var init_registry2 = __esm({
|
|
15686
16714
|
"src/workflows/registry.ts"() {
|
|
15687
16715
|
"use strict";
|
|
15688
|
-
|
|
16716
|
+
import_zod23 = require("zod");
|
|
15689
16717
|
init_artifact_writer();
|
|
15690
16718
|
init_http_client2();
|
|
15691
16719
|
init_agent_packet();
|
|
15692
16720
|
init_directory();
|
|
15693
16721
|
init_local_competitive_audit();
|
|
16722
|
+
init_comparison_briefs();
|
|
15694
16723
|
DEFINITIONS = [
|
|
15695
16724
|
directoryWorkflowDefinition,
|
|
15696
16725
|
agentPacketWorkflowDefinition,
|
|
15697
|
-
localCompetitiveAuditWorkflowDefinition
|
|
16726
|
+
localCompetitiveAuditWorkflowDefinition,
|
|
16727
|
+
mapComparisonWorkflowDefinition,
|
|
16728
|
+
serpComparisonWorkflowDefinition,
|
|
16729
|
+
paaExpansionBriefWorkflowDefinition,
|
|
16730
|
+
aiOverviewLanguageWorkflowDefinition
|
|
15698
16731
|
];
|
|
15699
16732
|
}
|
|
15700
16733
|
});
|
|
@@ -15756,7 +16789,7 @@ async function readManifestFromSummary(summary) {
|
|
|
15756
16789
|
function webhookSignature(body, timestamp2) {
|
|
15757
16790
|
const secret2 = process.env.MCP_SCRAPER_WEBHOOK_SECRET?.trim();
|
|
15758
16791
|
if (!secret2) return null;
|
|
15759
|
-
return (0,
|
|
16792
|
+
return (0, import_node_crypto3.createHmac)("sha256", secret2).update(`${timestamp2}.${body}`).digest("hex");
|
|
15760
16793
|
}
|
|
15761
16794
|
async function deliverWorkflowWebhook(input) {
|
|
15762
16795
|
if (!input.webhookUrl) return;
|
|
@@ -15810,7 +16843,8 @@ async function executeWorkflowRun(input) {
|
|
|
15810
16843
|
apiKey: input.user.api_key,
|
|
15811
16844
|
apiUrl: input.apiUrl,
|
|
15812
16845
|
outputDir: hostedWorkflowOutputDir(),
|
|
15813
|
-
runId: input.runId
|
|
16846
|
+
runId: input.runId,
|
|
16847
|
+
headers: input.concurrencyLockId ? { "x-mcp-scraper-concurrency-lock": input.concurrencyLockId } : void 0
|
|
15814
16848
|
});
|
|
15815
16849
|
const manifest = await readManifestFromSummary(summary);
|
|
15816
16850
|
await completeWorkflowRunRecord(input.runId, summary.status, manifest ?? { summary });
|
|
@@ -15839,19 +16873,28 @@ async function dispatchDueWorkflowSchedules(apiUrl, limit = 3) {
|
|
|
15839
16873
|
continue;
|
|
15840
16874
|
}
|
|
15841
16875
|
const scheduledFor = schedule.next_run_at ?? now;
|
|
15842
|
-
const
|
|
15843
|
-
|
|
15844
|
-
scheduleId: schedule.id,
|
|
15845
|
-
workflowId: schedule.workflow_id,
|
|
15846
|
-
workflowInput: schedule.input,
|
|
15847
|
-
idempotencyKey: `${schedule.id}:${scheduledFor}`
|
|
16876
|
+
const gate = await acquireConcurrencyGate(user, "workflow_schedule_dispatch", {
|
|
16877
|
+
ttlSeconds: 60 * 60,
|
|
16878
|
+
metadata: { scheduleId: schedule.id, workflowId: schedule.workflow_id }
|
|
15848
16879
|
});
|
|
15849
|
-
|
|
15850
|
-
|
|
15851
|
-
results.push({ scheduleId: schedule.id, runId: run.id, status: run.status });
|
|
16880
|
+
if (!gate.ok) {
|
|
16881
|
+
results.push({ scheduleId: schedule.id, status: "deferred", error: concurrencyLimitExceededResponse(gate).message });
|
|
15852
16882
|
continue;
|
|
15853
16883
|
}
|
|
16884
|
+
let run = null;
|
|
15854
16885
|
try {
|
|
16886
|
+
run = await createWorkflowRun({
|
|
16887
|
+
userId: user.id,
|
|
16888
|
+
scheduleId: schedule.id,
|
|
16889
|
+
workflowId: schedule.workflow_id,
|
|
16890
|
+
workflowInput: schedule.input,
|
|
16891
|
+
idempotencyKey: `${schedule.id}:${scheduledFor}`
|
|
16892
|
+
});
|
|
16893
|
+
await markWorkflowScheduleRan(schedule.id, now, addCadence(scheduledFor, schedule.cadence));
|
|
16894
|
+
if (run.status !== "queued") {
|
|
16895
|
+
results.push({ scheduleId: schedule.id, runId: run.id, status: run.status });
|
|
16896
|
+
continue;
|
|
16897
|
+
}
|
|
15855
16898
|
const executed = await executeWorkflowRun({
|
|
15856
16899
|
runId: run.id,
|
|
15857
16900
|
user,
|
|
@@ -15859,55 +16902,59 @@ async function dispatchDueWorkflowSchedules(apiUrl, limit = 3) {
|
|
|
15859
16902
|
workflowInput: schedule.input,
|
|
15860
16903
|
apiUrl,
|
|
15861
16904
|
scheduleId: schedule.id,
|
|
15862
|
-
webhookUrl: schedule.webhook_url
|
|
16905
|
+
webhookUrl: schedule.webhook_url,
|
|
16906
|
+
concurrencyLockId: gate.lockId
|
|
15863
16907
|
});
|
|
15864
16908
|
results.push({ scheduleId: schedule.id, runId: executed.run.id, status: executed.run.status });
|
|
15865
16909
|
} catch (err) {
|
|
15866
|
-
results.push({ scheduleId: schedule.id, runId: run
|
|
16910
|
+
results.push({ scheduleId: schedule.id, runId: run?.id, status: "failed", error: err instanceof Error ? err.message : String(err) });
|
|
16911
|
+
} finally {
|
|
16912
|
+
await releaseConcurrencyGate(gate.lockId);
|
|
15867
16913
|
}
|
|
15868
16914
|
}
|
|
15869
16915
|
return { dispatched: results.length, results };
|
|
15870
16916
|
}
|
|
15871
|
-
var
|
|
16917
|
+
var import_node_crypto3, import_promises7, import_hono7, import_zod24, workflowApp, WorkflowInputSchema, WorkflowIdSchema, CadenceSchema, ScheduleStatusSchema, RunBodySchema, ScheduleCreateSchema, SchedulePatchSchema;
|
|
15872
16918
|
var init_workflow_routes = __esm({
|
|
15873
16919
|
"src/api/workflow-routes.ts"() {
|
|
15874
16920
|
"use strict";
|
|
15875
|
-
|
|
16921
|
+
import_node_crypto3 = require("crypto");
|
|
15876
16922
|
import_promises7 = require("fs/promises");
|
|
15877
16923
|
import_hono7 = require("hono");
|
|
15878
|
-
|
|
16924
|
+
import_zod24 = require("zod");
|
|
15879
16925
|
init_artifact_writer();
|
|
15880
16926
|
init_registry2();
|
|
15881
16927
|
init_api_auth();
|
|
15882
16928
|
init_db();
|
|
15883
16929
|
init_url_utils();
|
|
16930
|
+
init_concurrency_gates();
|
|
15884
16931
|
workflowApp = new import_hono7.Hono();
|
|
15885
|
-
WorkflowInputSchema =
|
|
15886
|
-
WorkflowIdSchema =
|
|
15887
|
-
CadenceSchema =
|
|
15888
|
-
ScheduleStatusSchema =
|
|
15889
|
-
RunBodySchema =
|
|
16932
|
+
WorkflowInputSchema = import_zod24.z.record(import_zod24.z.unknown()).default({});
|
|
16933
|
+
WorkflowIdSchema = import_zod24.z.string().min(1);
|
|
16934
|
+
CadenceSchema = import_zod24.z.enum(["daily", "weekly", "monthly"]);
|
|
16935
|
+
ScheduleStatusSchema = import_zod24.z.enum(["active", "paused"]);
|
|
16936
|
+
RunBodySchema = import_zod24.z.object({
|
|
15890
16937
|
workflowId: WorkflowIdSchema,
|
|
15891
16938
|
input: WorkflowInputSchema,
|
|
15892
|
-
webhookUrl:
|
|
16939
|
+
webhookUrl: import_zod24.z.string().url().optional()
|
|
15893
16940
|
});
|
|
15894
|
-
ScheduleCreateSchema =
|
|
16941
|
+
ScheduleCreateSchema = import_zod24.z.object({
|
|
15895
16942
|
workflowId: WorkflowIdSchema,
|
|
15896
|
-
name:
|
|
16943
|
+
name: import_zod24.z.string().min(1).max(120).optional(),
|
|
15897
16944
|
input: WorkflowInputSchema,
|
|
15898
16945
|
cadence: CadenceSchema.default("weekly"),
|
|
15899
|
-
timezone:
|
|
15900
|
-
webhookUrl:
|
|
15901
|
-
nextRunAt:
|
|
16946
|
+
timezone: import_zod24.z.string().min(1).max(64).default("UTC"),
|
|
16947
|
+
webhookUrl: import_zod24.z.string().url().optional(),
|
|
16948
|
+
nextRunAt: import_zod24.z.string().datetime().optional()
|
|
15902
16949
|
});
|
|
15903
|
-
SchedulePatchSchema =
|
|
15904
|
-
name:
|
|
16950
|
+
SchedulePatchSchema = import_zod24.z.object({
|
|
16951
|
+
name: import_zod24.z.string().min(1).max(120).optional(),
|
|
15905
16952
|
status: ScheduleStatusSchema.optional(),
|
|
15906
16953
|
input: WorkflowInputSchema.optional(),
|
|
15907
16954
|
cadence: CadenceSchema.optional(),
|
|
15908
|
-
timezone:
|
|
15909
|
-
webhookUrl:
|
|
15910
|
-
nextRunAt:
|
|
16955
|
+
timezone: import_zod24.z.string().min(1).max(64).optional(),
|
|
16956
|
+
webhookUrl: import_zod24.z.string().url().nullable().optional(),
|
|
16957
|
+
nextRunAt: import_zod24.z.string().datetime().nullable().optional()
|
|
15911
16958
|
});
|
|
15912
16959
|
workflowApp.get("/definitions", createApiKeyAuth(), (c) => {
|
|
15913
16960
|
return c.json({ workflows: listWorkflowDefinitions() });
|
|
@@ -15924,20 +16971,29 @@ var init_workflow_routes = __esm({
|
|
|
15924
16971
|
} catch (err) {
|
|
15925
16972
|
return c.json({ error: err instanceof Error ? err.message : String(err) }, 400);
|
|
15926
16973
|
}
|
|
15927
|
-
const
|
|
16974
|
+
const gate = await acquireConcurrencyGate(user, "workflow_run", {
|
|
16975
|
+
ttlSeconds: 60 * 60,
|
|
16976
|
+
metadata: { workflowId: parsed.data.workflowId }
|
|
16977
|
+
});
|
|
16978
|
+
if (!gate.ok) return c.json(concurrencyLimitExceededResponse(gate), 429, { "Retry-After": String(gate.retryAfterSeconds) });
|
|
16979
|
+
let run = null;
|
|
15928
16980
|
try {
|
|
16981
|
+
run = await createWorkflowRun({ userId: user.id, workflowId: parsed.data.workflowId, workflowInput: input });
|
|
15929
16982
|
const executed = await executeWorkflowRun({
|
|
15930
16983
|
runId: run.id,
|
|
15931
16984
|
user,
|
|
15932
16985
|
workflowId: parsed.data.workflowId,
|
|
15933
16986
|
workflowInput: input,
|
|
15934
16987
|
apiUrl: originFromUrl(c.req.url),
|
|
15935
|
-
webhookUrl
|
|
16988
|
+
webhookUrl,
|
|
16989
|
+
concurrencyLockId: gate.lockId
|
|
15936
16990
|
});
|
|
15937
16991
|
return c.json({ run: exposeRun(c, executed.run, executed.artifacts), summary: executed.summary });
|
|
15938
16992
|
} catch (err) {
|
|
15939
|
-
const failed = await getWorkflowRun(run.id, user.id);
|
|
16993
|
+
const failed = run ? await getWorkflowRun(run.id, user.id) : null;
|
|
15940
16994
|
return c.json({ run: failed, error: err instanceof Error ? err.message : String(err) }, 500);
|
|
16995
|
+
} finally {
|
|
16996
|
+
await releaseConcurrencyGate(gate.lockId);
|
|
15941
16997
|
}
|
|
15942
16998
|
});
|
|
15943
16999
|
workflowApp.get("/runs", createApiKeyAuth(), async (c) => {
|
|
@@ -16035,8 +17091,14 @@ var init_workflow_routes = __esm({
|
|
|
16035
17091
|
const user = c.get("user");
|
|
16036
17092
|
const schedule = await getWorkflowSchedule(c.req.param("id"), user.id);
|
|
16037
17093
|
if (!schedule) return c.json({ error: "Schedule not found" }, 404);
|
|
16038
|
-
const
|
|
17094
|
+
const gate = await acquireConcurrencyGate(user, "workflow_schedule_run", {
|
|
17095
|
+
ttlSeconds: 60 * 60,
|
|
17096
|
+
metadata: { scheduleId: schedule.id, workflowId: schedule.workflow_id }
|
|
17097
|
+
});
|
|
17098
|
+
if (!gate.ok) return c.json(concurrencyLimitExceededResponse(gate), 429, { "Retry-After": String(gate.retryAfterSeconds) });
|
|
17099
|
+
let run = null;
|
|
16039
17100
|
try {
|
|
17101
|
+
run = await createWorkflowRun({ userId: user.id, scheduleId: schedule.id, workflowId: schedule.workflow_id, workflowInput: schedule.input });
|
|
16040
17102
|
const executed = await executeWorkflowRun({
|
|
16041
17103
|
runId: run.id,
|
|
16042
17104
|
user,
|
|
@@ -16044,12 +17106,15 @@ var init_workflow_routes = __esm({
|
|
|
16044
17106
|
workflowInput: schedule.input,
|
|
16045
17107
|
apiUrl: originFromUrl(c.req.url),
|
|
16046
17108
|
scheduleId: schedule.id,
|
|
16047
|
-
webhookUrl: schedule.webhook_url
|
|
17109
|
+
webhookUrl: schedule.webhook_url,
|
|
17110
|
+
concurrencyLockId: gate.lockId
|
|
16048
17111
|
});
|
|
16049
17112
|
return c.json({ run: exposeRun(c, executed.run, executed.artifacts), summary: executed.summary });
|
|
16050
17113
|
} catch (err) {
|
|
16051
|
-
const failed = await getWorkflowRun(run.id, user.id);
|
|
17114
|
+
const failed = run ? await getWorkflowRun(run.id, user.id) : null;
|
|
16052
17115
|
return c.json({ run: failed, error: err instanceof Error ? err.message : String(err) }, 500);
|
|
17116
|
+
} finally {
|
|
17117
|
+
await releaseConcurrencyGate(gate.lockId);
|
|
16053
17118
|
}
|
|
16054
17119
|
});
|
|
16055
17120
|
workflowApp.post("/cron/dispatch", async (c) => {
|
|
@@ -16065,7 +17130,7 @@ var init_workflow_routes = __esm({
|
|
|
16065
17130
|
// src/serp-intelligence/page-snapshot-extractor.ts
|
|
16066
17131
|
function sha256(value) {
|
|
16067
17132
|
if (!value) return null;
|
|
16068
|
-
return (0,
|
|
17133
|
+
return (0, import_node_crypto4.createHash)("sha256").update(value).digest("hex");
|
|
16069
17134
|
}
|
|
16070
17135
|
function countWords(markdown) {
|
|
16071
17136
|
const matches = markdown.trim().match(/\b[\p{L}\p{N}][\p{L}\p{N}'-]*\b/gu);
|
|
@@ -16365,11 +17430,11 @@ async function capturePageSnapshots(targets, options = {}) {
|
|
|
16365
17430
|
}
|
|
16366
17431
|
};
|
|
16367
17432
|
}
|
|
16368
|
-
var
|
|
17433
|
+
var import_node_crypto4, import_p_limit3, DEFAULT_TIMEOUT_MS, DEFAULT_MAX_CONCURRENCY, DEFAULT_MAX_CONTENT_CHARS;
|
|
16369
17434
|
var init_page_snapshot_extractor = __esm({
|
|
16370
17435
|
"src/serp-intelligence/page-snapshot-extractor.ts"() {
|
|
16371
17436
|
"use strict";
|
|
16372
|
-
|
|
17437
|
+
import_node_crypto4 = require("crypto");
|
|
16373
17438
|
import_p_limit3 = __toESM(require("p-limit"), 1);
|
|
16374
17439
|
init_kpo_extractor();
|
|
16375
17440
|
init_url_utils();
|
|
@@ -18246,11 +19311,11 @@ function isPublicHttpUrl(value) {
|
|
|
18246
19311
|
return false;
|
|
18247
19312
|
}
|
|
18248
19313
|
}
|
|
18249
|
-
var
|
|
19314
|
+
var import_zod25, SerpIntelligenceDeviceValues, SerpIntelligenceProxyModeValues, SerpIntelligenceAttemptOutcomeValues, SerpIntelligenceLocalizationStatusValues, SerpPageSnapshotSourceKindValues, SerpPageFetchStatusValues, SerpPageFetchedViaValues, HostnameSuffixPattern, Ipv4Pattern, SerpIntelligencePublicHttpUrlSchema, SerpIntelligenceCaptureBodySchema, SerpIntelligencePageSnapshotRequestSchema, SerpIntelligencePageSnapshotsBodySchema, SerpIntelligenceAICitationSchema, SerpIntelligenceOrganicResultSchema, SerpIntelligenceLocationEvidenceSchema, SerpIntelligenceHarvestResultSchema, SerpIntelligenceCaptureAttemptSchema, SerpPageSnapshotCaptureSchema, SerpIntelligenceCaptureResponseSchema, SerpIntelligencePageSnapshotsResponseSchema;
|
|
18250
19315
|
var init_schemas4 = __esm({
|
|
18251
19316
|
"src/serp-intelligence/schemas.ts"() {
|
|
18252
19317
|
"use strict";
|
|
18253
|
-
|
|
19318
|
+
import_zod25 = require("zod");
|
|
18254
19319
|
SerpIntelligenceDeviceValues = ["desktop", "mobile"];
|
|
18255
19320
|
SerpIntelligenceProxyModeValues = ["location", "configured", "none"];
|
|
18256
19321
|
SerpIntelligenceAttemptOutcomeValues = [
|
|
@@ -18284,171 +19349,171 @@ var init_schemas4 = __esm({
|
|
|
18284
19349
|
SerpPageFetchedViaValues = ["fetch", "headless", "browser", "mcp"];
|
|
18285
19350
|
HostnameSuffixPattern = /(^|\.)localhost$/i;
|
|
18286
19351
|
Ipv4Pattern = /^\d{1,3}(?:\.\d{1,3}){3}$/;
|
|
18287
|
-
SerpIntelligencePublicHttpUrlSchema =
|
|
18288
|
-
SerpIntelligenceCaptureBodySchema =
|
|
18289
|
-
query:
|
|
18290
|
-
location:
|
|
18291
|
-
gl:
|
|
18292
|
-
hl:
|
|
18293
|
-
device:
|
|
18294
|
-
proxyMode:
|
|
18295
|
-
proxyZip:
|
|
18296
|
-
pages:
|
|
18297
|
-
debug:
|
|
18298
|
-
includePageSnapshots:
|
|
18299
|
-
pageSnapshotLimit:
|
|
19352
|
+
SerpIntelligencePublicHttpUrlSchema = import_zod25.z.string().url().refine(isPublicHttpUrl, "url must be a public HTTP or HTTPS URL");
|
|
19353
|
+
SerpIntelligenceCaptureBodySchema = import_zod25.z.object({
|
|
19354
|
+
query: import_zod25.z.string().trim().min(1, "query is required"),
|
|
19355
|
+
location: import_zod25.z.string().trim().min(1).optional(),
|
|
19356
|
+
gl: import_zod25.z.string().trim().length(2).default("us"),
|
|
19357
|
+
hl: import_zod25.z.string().trim().length(2).default("en"),
|
|
19358
|
+
device: import_zod25.z.enum(SerpIntelligenceDeviceValues).default("desktop"),
|
|
19359
|
+
proxyMode: import_zod25.z.enum(SerpIntelligenceProxyModeValues).default("location"),
|
|
19360
|
+
proxyZip: import_zod25.z.string().regex(/^\d{5}$/).optional(),
|
|
19361
|
+
pages: import_zod25.z.number().int().min(1).max(2).default(1),
|
|
19362
|
+
debug: import_zod25.z.boolean().default(false),
|
|
19363
|
+
includePageSnapshots: import_zod25.z.boolean().default(false),
|
|
19364
|
+
pageSnapshotLimit: import_zod25.z.number().int().min(0).max(10).default(0)
|
|
18300
19365
|
}).strict();
|
|
18301
|
-
SerpIntelligencePageSnapshotRequestSchema =
|
|
19366
|
+
SerpIntelligencePageSnapshotRequestSchema = import_zod25.z.object({
|
|
18302
19367
|
url: SerpIntelligencePublicHttpUrlSchema,
|
|
18303
|
-
sourceKind:
|
|
18304
|
-
sourcePosition:
|
|
19368
|
+
sourceKind: import_zod25.z.enum(SerpPageSnapshotSourceKindValues).default("configured_target"),
|
|
19369
|
+
sourcePosition: import_zod25.z.number().int().min(1).optional()
|
|
18305
19370
|
}).strict();
|
|
18306
|
-
SerpIntelligencePageSnapshotsBodySchema =
|
|
18307
|
-
urls:
|
|
18308
|
-
targets:
|
|
18309
|
-
maxConcurrency:
|
|
18310
|
-
timeoutMs:
|
|
18311
|
-
debug:
|
|
19371
|
+
SerpIntelligencePageSnapshotsBodySchema = import_zod25.z.object({
|
|
19372
|
+
urls: import_zod25.z.array(SerpIntelligencePublicHttpUrlSchema).min(1).max(25),
|
|
19373
|
+
targets: import_zod25.z.array(SerpIntelligencePageSnapshotRequestSchema).min(1).max(25).optional(),
|
|
19374
|
+
maxConcurrency: import_zod25.z.number().int().min(1).max(5).default(2),
|
|
19375
|
+
timeoutMs: import_zod25.z.number().int().min(1e3).max(6e4).default(15e3),
|
|
19376
|
+
debug: import_zod25.z.boolean().default(false)
|
|
18312
19377
|
}).strict();
|
|
18313
|
-
SerpIntelligenceAICitationSchema =
|
|
18314
|
-
text:
|
|
18315
|
-
href:
|
|
19378
|
+
SerpIntelligenceAICitationSchema = import_zod25.z.object({
|
|
19379
|
+
text: import_zod25.z.string(),
|
|
19380
|
+
href: import_zod25.z.string()
|
|
18316
19381
|
}).strict();
|
|
18317
|
-
SerpIntelligenceOrganicResultSchema =
|
|
18318
|
-
position:
|
|
18319
|
-
title:
|
|
18320
|
-
url:
|
|
18321
|
-
domain:
|
|
18322
|
-
cite:
|
|
18323
|
-
snippet:
|
|
18324
|
-
isRedditStyle:
|
|
18325
|
-
inlineRating:
|
|
18326
|
-
value:
|
|
18327
|
-
count:
|
|
19382
|
+
SerpIntelligenceOrganicResultSchema = import_zod25.z.object({
|
|
19383
|
+
position: import_zod25.z.number().int().min(1),
|
|
19384
|
+
title: import_zod25.z.string(),
|
|
19385
|
+
url: import_zod25.z.string(),
|
|
19386
|
+
domain: import_zod25.z.string(),
|
|
19387
|
+
cite: import_zod25.z.string().nullable(),
|
|
19388
|
+
snippet: import_zod25.z.string().nullable(),
|
|
19389
|
+
isRedditStyle: import_zod25.z.boolean(),
|
|
19390
|
+
inlineRating: import_zod25.z.object({
|
|
19391
|
+
value: import_zod25.z.string(),
|
|
19392
|
+
count: import_zod25.z.string()
|
|
18328
19393
|
}).strict().nullable()
|
|
18329
19394
|
}).strict();
|
|
18330
|
-
SerpIntelligenceLocationEvidenceSchema =
|
|
18331
|
-
status:
|
|
18332
|
-
expected:
|
|
18333
|
-
city:
|
|
18334
|
-
regionCode:
|
|
18335
|
-
canonicalLocation:
|
|
19395
|
+
SerpIntelligenceLocationEvidenceSchema = import_zod25.z.object({
|
|
19396
|
+
status: import_zod25.z.enum(SerpIntelligenceLocalizationStatusValues),
|
|
19397
|
+
expected: import_zod25.z.object({
|
|
19398
|
+
city: import_zod25.z.string(),
|
|
19399
|
+
regionCode: import_zod25.z.string().nullable(),
|
|
19400
|
+
canonicalLocation: import_zod25.z.string()
|
|
18336
19401
|
}).strict().nullable(),
|
|
18337
|
-
candidates:
|
|
18338
|
-
city:
|
|
18339
|
-
regionCode:
|
|
18340
|
-
count:
|
|
18341
|
-
examples:
|
|
19402
|
+
candidates: import_zod25.z.array(import_zod25.z.object({
|
|
19403
|
+
city: import_zod25.z.string(),
|
|
19404
|
+
regionCode: import_zod25.z.string(),
|
|
19405
|
+
count: import_zod25.z.number().int().min(0),
|
|
19406
|
+
examples: import_zod25.z.array(import_zod25.z.string())
|
|
18342
19407
|
}).strict())
|
|
18343
19408
|
}).strict();
|
|
18344
|
-
SerpIntelligenceHarvestResultSchema =
|
|
18345
|
-
seed:
|
|
18346
|
-
location:
|
|
18347
|
-
extractedAt:
|
|
18348
|
-
totalQuestions:
|
|
18349
|
-
surface:
|
|
18350
|
-
aiOverview:
|
|
18351
|
-
detected:
|
|
18352
|
-
text:
|
|
18353
|
-
citations:
|
|
18354
|
-
expanded:
|
|
18355
|
-
fullyExpanded:
|
|
18356
|
-
sections:
|
|
19409
|
+
SerpIntelligenceHarvestResultSchema = import_zod25.z.object({
|
|
19410
|
+
seed: import_zod25.z.string(),
|
|
19411
|
+
location: import_zod25.z.string().nullable(),
|
|
19412
|
+
extractedAt: import_zod25.z.string(),
|
|
19413
|
+
totalQuestions: import_zod25.z.number().int().min(0),
|
|
19414
|
+
surface: import_zod25.z.enum(["web", "aim", "unknown"]),
|
|
19415
|
+
aiOverview: import_zod25.z.object({
|
|
19416
|
+
detected: import_zod25.z.boolean(),
|
|
19417
|
+
text: import_zod25.z.string().nullable(),
|
|
19418
|
+
citations: import_zod25.z.array(SerpIntelligenceAICitationSchema),
|
|
19419
|
+
expanded: import_zod25.z.boolean().optional(),
|
|
19420
|
+
fullyExpanded: import_zod25.z.boolean().optional(),
|
|
19421
|
+
sections: import_zod25.z.array(import_zod25.z.string()).optional()
|
|
18357
19422
|
}).strict(),
|
|
18358
|
-
aiMode:
|
|
18359
|
-
detected:
|
|
18360
|
-
text:
|
|
18361
|
-
citations:
|
|
19423
|
+
aiMode: import_zod25.z.object({
|
|
19424
|
+
detected: import_zod25.z.boolean(),
|
|
19425
|
+
text: import_zod25.z.string().nullable(),
|
|
19426
|
+
citations: import_zod25.z.array(SerpIntelligenceAICitationSchema)
|
|
18362
19427
|
}).strict(),
|
|
18363
|
-
tree:
|
|
18364
|
-
flat:
|
|
18365
|
-
videos:
|
|
18366
|
-
forums:
|
|
18367
|
-
organicResults:
|
|
18368
|
-
localPack:
|
|
18369
|
-
entityIds:
|
|
18370
|
-
entities:
|
|
18371
|
-
name:
|
|
18372
|
-
kgId:
|
|
18373
|
-
cid:
|
|
18374
|
-
gcid:
|
|
19428
|
+
tree: import_zod25.z.array(import_zod25.z.unknown()),
|
|
19429
|
+
flat: import_zod25.z.array(import_zod25.z.unknown()),
|
|
19430
|
+
videos: import_zod25.z.array(import_zod25.z.unknown()),
|
|
19431
|
+
forums: import_zod25.z.array(import_zod25.z.unknown()),
|
|
19432
|
+
organicResults: import_zod25.z.array(SerpIntelligenceOrganicResultSchema),
|
|
19433
|
+
localPack: import_zod25.z.array(import_zod25.z.unknown()),
|
|
19434
|
+
entityIds: import_zod25.z.object({
|
|
19435
|
+
entities: import_zod25.z.array(import_zod25.z.object({
|
|
19436
|
+
name: import_zod25.z.string(),
|
|
19437
|
+
kgId: import_zod25.z.string().nullable(),
|
|
19438
|
+
cid: import_zod25.z.string().nullable(),
|
|
19439
|
+
gcid: import_zod25.z.string().nullable()
|
|
18375
19440
|
}).strict()),
|
|
18376
|
-
kgIds:
|
|
18377
|
-
cids:
|
|
18378
|
-
gcids:
|
|
19441
|
+
kgIds: import_zod25.z.array(import_zod25.z.string()),
|
|
19442
|
+
cids: import_zod25.z.array(import_zod25.z.string()),
|
|
19443
|
+
gcids: import_zod25.z.array(import_zod25.z.string())
|
|
18379
19444
|
}).strict(),
|
|
18380
|
-
stats:
|
|
18381
|
-
seed:
|
|
18382
|
-
totalQuestions:
|
|
18383
|
-
maxDepthReached:
|
|
18384
|
-
durationMs:
|
|
18385
|
-
errorCount:
|
|
19445
|
+
stats: import_zod25.z.object({
|
|
19446
|
+
seed: import_zod25.z.string(),
|
|
19447
|
+
totalQuestions: import_zod25.z.number().int().min(0),
|
|
19448
|
+
maxDepthReached: import_zod25.z.number().int().min(0),
|
|
19449
|
+
durationMs: import_zod25.z.number().min(0),
|
|
19450
|
+
errorCount: import_zod25.z.number().int().min(0)
|
|
18386
19451
|
}).strict(),
|
|
18387
|
-
diagnostics:
|
|
18388
|
-
completionStatus:
|
|
18389
|
-
problem:
|
|
18390
|
-
warnings:
|
|
18391
|
-
debug:
|
|
19452
|
+
diagnostics: import_zod25.z.object({
|
|
19453
|
+
completionStatus: import_zod25.z.enum(["paa_found", "no_paa", "serp_only"]),
|
|
19454
|
+
problem: import_zod25.z.null(),
|
|
19455
|
+
warnings: import_zod25.z.array(import_zod25.z.unknown()).optional(),
|
|
19456
|
+
debug: import_zod25.z.object({
|
|
18392
19457
|
locationEvidence: SerpIntelligenceLocationEvidenceSchema.optional()
|
|
18393
19458
|
}).passthrough().optional()
|
|
18394
19459
|
}).passthrough(),
|
|
18395
|
-
whatPeopleSaying:
|
|
19460
|
+
whatPeopleSaying: import_zod25.z.array(import_zod25.z.unknown())
|
|
18396
19461
|
}).strict();
|
|
18397
|
-
SerpIntelligenceCaptureAttemptSchema =
|
|
18398
|
-
attemptNumber:
|
|
18399
|
-
outcome:
|
|
18400
|
-
startedAt:
|
|
18401
|
-
completedAt:
|
|
18402
|
-
durationMs:
|
|
18403
|
-
problemCode:
|
|
18404
|
-
message:
|
|
18405
|
-
kernelSessionId:
|
|
18406
|
-
cleanupSucceeded:
|
|
19462
|
+
SerpIntelligenceCaptureAttemptSchema = import_zod25.z.object({
|
|
19463
|
+
attemptNumber: import_zod25.z.number().int().min(1),
|
|
19464
|
+
outcome: import_zod25.z.enum(SerpIntelligenceAttemptOutcomeValues),
|
|
19465
|
+
startedAt: import_zod25.z.string().optional(),
|
|
19466
|
+
completedAt: import_zod25.z.string().optional(),
|
|
19467
|
+
durationMs: import_zod25.z.number().min(0).optional(),
|
|
19468
|
+
problemCode: import_zod25.z.string().optional(),
|
|
19469
|
+
message: import_zod25.z.string().optional(),
|
|
19470
|
+
kernelSessionId: import_zod25.z.string().nullable().optional(),
|
|
19471
|
+
cleanupSucceeded: import_zod25.z.boolean().nullable().optional()
|
|
18407
19472
|
}).strict();
|
|
18408
|
-
SerpPageSnapshotCaptureSchema =
|
|
19473
|
+
SerpPageSnapshotCaptureSchema = import_zod25.z.object({
|
|
18409
19474
|
url: SerpIntelligencePublicHttpUrlSchema,
|
|
18410
19475
|
requestedUrl: SerpIntelligencePublicHttpUrlSchema,
|
|
18411
19476
|
finalUrl: SerpIntelligencePublicHttpUrlSchema.nullable(),
|
|
18412
|
-
sourceKind:
|
|
18413
|
-
sourcePosition:
|
|
18414
|
-
status:
|
|
18415
|
-
fetchedVia:
|
|
18416
|
-
httpStatus:
|
|
18417
|
-
contentType:
|
|
18418
|
-
title:
|
|
19477
|
+
sourceKind: import_zod25.z.enum(SerpPageSnapshotSourceKindValues),
|
|
19478
|
+
sourcePosition: import_zod25.z.number().int().min(1).nullable(),
|
|
19479
|
+
status: import_zod25.z.enum(SerpPageFetchStatusValues),
|
|
19480
|
+
fetchedVia: import_zod25.z.enum(SerpPageFetchedViaValues).nullable(),
|
|
19481
|
+
httpStatus: import_zod25.z.number().int().min(100).max(599).nullable(),
|
|
19482
|
+
contentType: import_zod25.z.string().nullable(),
|
|
19483
|
+
title: import_zod25.z.string().nullable(),
|
|
18419
19484
|
canonicalUrl: SerpIntelligencePublicHttpUrlSchema.nullable(),
|
|
18420
|
-
metaDescription:
|
|
18421
|
-
headings:
|
|
18422
|
-
level:
|
|
18423
|
-
text:
|
|
19485
|
+
metaDescription: import_zod25.z.string().nullable(),
|
|
19486
|
+
headings: import_zod25.z.array(import_zod25.z.object({
|
|
19487
|
+
level: import_zod25.z.number().int().min(1).max(6),
|
|
19488
|
+
text: import_zod25.z.string()
|
|
18424
19489
|
}).strict()).default([]),
|
|
18425
|
-
artifact:
|
|
18426
|
-
htmlBlobUrl:
|
|
18427
|
-
textBlobUrl:
|
|
18428
|
-
markdownBlobUrl:
|
|
18429
|
-
screenshotBlobUrl:
|
|
18430
|
-
contentSha256:
|
|
18431
|
-
capturedAt:
|
|
19490
|
+
artifact: import_zod25.z.object({
|
|
19491
|
+
htmlBlobUrl: import_zod25.z.string().url().nullable(),
|
|
19492
|
+
textBlobUrl: import_zod25.z.string().url().nullable(),
|
|
19493
|
+
markdownBlobUrl: import_zod25.z.string().url().nullable(),
|
|
19494
|
+
screenshotBlobUrl: import_zod25.z.string().url().nullable(),
|
|
19495
|
+
contentSha256: import_zod25.z.string().nullable(),
|
|
19496
|
+
capturedAt: import_zod25.z.string().nullable()
|
|
18432
19497
|
}).strict(),
|
|
18433
|
-
error:
|
|
18434
|
-
code:
|
|
18435
|
-
message:
|
|
19498
|
+
error: import_zod25.z.object({
|
|
19499
|
+
code: import_zod25.z.string(),
|
|
19500
|
+
message: import_zod25.z.string()
|
|
18436
19501
|
}).strict().nullable()
|
|
18437
19502
|
}).strict();
|
|
18438
|
-
SerpIntelligenceCaptureResponseSchema =
|
|
19503
|
+
SerpIntelligenceCaptureResponseSchema = import_zod25.z.object({
|
|
18439
19504
|
harvestResult: SerpIntelligenceHarvestResultSchema,
|
|
18440
|
-
attempts:
|
|
19505
|
+
attempts: import_zod25.z.array(SerpIntelligenceCaptureAttemptSchema),
|
|
18441
19506
|
locationEvidence: SerpIntelligenceLocationEvidenceSchema.nullable(),
|
|
18442
|
-
pageSnapshotArtifacts:
|
|
18443
|
-
billing:
|
|
18444
|
-
creditsUsed:
|
|
18445
|
-
requestId:
|
|
18446
|
-
jobId:
|
|
19507
|
+
pageSnapshotArtifacts: import_zod25.z.array(SerpPageSnapshotCaptureSchema),
|
|
19508
|
+
billing: import_zod25.z.object({
|
|
19509
|
+
creditsUsed: import_zod25.z.number().min(0).optional(),
|
|
19510
|
+
requestId: import_zod25.z.string().optional(),
|
|
19511
|
+
jobId: import_zod25.z.string().optional()
|
|
18447
19512
|
}).strict().optional()
|
|
18448
19513
|
}).strict();
|
|
18449
|
-
SerpIntelligencePageSnapshotsResponseSchema =
|
|
18450
|
-
pageSnapshotArtifacts:
|
|
18451
|
-
attempts:
|
|
19514
|
+
SerpIntelligencePageSnapshotsResponseSchema = import_zod25.z.object({
|
|
19515
|
+
pageSnapshotArtifacts: import_zod25.z.array(SerpPageSnapshotCaptureSchema),
|
|
19516
|
+
attempts: import_zod25.z.array(SerpIntelligenceCaptureAttemptSchema).default([])
|
|
18452
19517
|
}).strict();
|
|
18453
19518
|
}
|
|
18454
19519
|
});
|
|
@@ -18665,26 +19730,6 @@ async function enforceSerpIntelligenceRateLimit(c, userId) {
|
|
|
18665
19730
|
{ "Retry-After": String(result.resetSeconds) }
|
|
18666
19731
|
);
|
|
18667
19732
|
}
|
|
18668
|
-
async function enforceSerpIntelligenceConcurrency(user) {
|
|
18669
|
-
const active = await countActiveJobsForUser(user.id);
|
|
18670
|
-
const limit = 1 + (user.extra_concurrency_slots ?? 0);
|
|
18671
|
-
if (active < limit) return null;
|
|
18672
|
-
return new Response(
|
|
18673
|
-
JSON.stringify(structuredError({
|
|
18674
|
-
error_code: "concurrency_limit_exceeded",
|
|
18675
|
-
error_type: "concurrency_limit",
|
|
18676
|
-
message: `You have ${active} active job${active !== 1 ? "s" : ""}. Your account allows ${limit} concurrent job${limit !== 1 ? "s" : ""}.`,
|
|
18677
|
-
retryable: true
|
|
18678
|
-
})),
|
|
18679
|
-
{
|
|
18680
|
-
status: 429,
|
|
18681
|
-
headers: {
|
|
18682
|
-
"Content-Type": "application/json",
|
|
18683
|
-
"Retry-After": "30"
|
|
18684
|
-
}
|
|
18685
|
-
}
|
|
18686
|
-
);
|
|
18687
|
-
}
|
|
18688
19733
|
function pageSnapshotTargetsFromBody(body) {
|
|
18689
19734
|
return body.targets?.length ? body.targets : body.urls.map((url) => ({
|
|
18690
19735
|
url,
|
|
@@ -18704,6 +19749,7 @@ var init_serp_intelligence_routes = __esm({
|
|
|
18704
19749
|
init_api_auth();
|
|
18705
19750
|
init_db();
|
|
18706
19751
|
init_rates();
|
|
19752
|
+
init_concurrency_gates();
|
|
18707
19753
|
SERP_INTELLIGENCE_RATE_LIMIT = 60;
|
|
18708
19754
|
SERP_INTELLIGENCE_RATE_WINDOW_SECONDS = 60;
|
|
18709
19755
|
POST_CAPTURE_ROUTE_LABEL = "POST /capture";
|
|
@@ -18718,17 +19764,22 @@ var init_serp_intelligence_routes = __esm({
|
|
|
18718
19764
|
if (!parsed.success) return c.json(formatZodError(parsed.error), 400);
|
|
18719
19765
|
const limited = await enforceSerpIntelligenceRateLimit(c, user.id);
|
|
18720
19766
|
if (limited) return limited;
|
|
18721
|
-
const
|
|
18722
|
-
|
|
19767
|
+
const gate = await acquireConcurrencyGate(user, "serp_intelligence_capture", {
|
|
19768
|
+
reuseLockId: c.req.header("x-mcp-scraper-concurrency-lock"),
|
|
19769
|
+
metadata: { query: parsed.data.query, location: parsed.data.location }
|
|
19770
|
+
});
|
|
19771
|
+
if (!gate.ok) return c.json(concurrencyLimitExceededResponse(gate), 429, { "Retry-After": String(gate.retryAfterSeconds) });
|
|
18723
19772
|
const cost = MC_COSTS.serp;
|
|
18724
|
-
|
|
18725
|
-
user.id,
|
|
18726
|
-
cost,
|
|
18727
|
-
LedgerOperation.SERP,
|
|
18728
|
-
parsed.data.query
|
|
18729
|
-
);
|
|
18730
|
-
if (!ok) return c.json(insufficientBalanceResponse(balance_mc, cost), 402);
|
|
19773
|
+
let debited = false;
|
|
18731
19774
|
try {
|
|
19775
|
+
const { ok, balance_mc } = await debitMc(
|
|
19776
|
+
user.id,
|
|
19777
|
+
cost,
|
|
19778
|
+
LedgerOperation.SERP,
|
|
19779
|
+
parsed.data.query
|
|
19780
|
+
);
|
|
19781
|
+
if (!ok) return c.json(insufficientBalanceResponse(balance_mc, cost), 402);
|
|
19782
|
+
debited = true;
|
|
18732
19783
|
const result = await captureSerpIntelligenceSnapshot(parsed.data, {
|
|
18733
19784
|
kernelApiKey: browserServiceApiKey(),
|
|
18734
19785
|
kernelProxyId: browserServiceProxyId(),
|
|
@@ -18746,7 +19797,7 @@ var init_serp_intelligence_routes = __esm({
|
|
|
18746
19797
|
});
|
|
18747
19798
|
return c.json(result);
|
|
18748
19799
|
} catch (error) {
|
|
18749
|
-
await creditMc(user.id, cost, LedgerOperation.REFUND, "failed serp intelligence capture");
|
|
19800
|
+
if (debited) await creditMc(user.id, cost, LedgerOperation.REFUND, "failed serp intelligence capture");
|
|
18750
19801
|
const body = error instanceof SerpIntelligenceCaptureError ? error.toJSON() : structuredError({
|
|
18751
19802
|
error_code: "capture_failed",
|
|
18752
19803
|
error_type: "capture_error",
|
|
@@ -18763,6 +19814,8 @@ var init_serp_intelligence_routes = __esm({
|
|
|
18763
19814
|
});
|
|
18764
19815
|
const status = error instanceof SerpIntelligenceCaptureError ? error.httpStatus : 500;
|
|
18765
19816
|
return c.json(body, status);
|
|
19817
|
+
} finally {
|
|
19818
|
+
await releaseConcurrencyGate(gate.lockId);
|
|
18766
19819
|
}
|
|
18767
19820
|
});
|
|
18768
19821
|
serpIntelligenceApp.post("/page-snapshots", async (c) => {
|
|
@@ -18773,18 +19826,23 @@ var init_serp_intelligence_routes = __esm({
|
|
|
18773
19826
|
if (!parsed.success) return c.json(formatZodError(parsed.error), 400);
|
|
18774
19827
|
const limited = await enforceSerpIntelligenceRateLimit(c, user.id);
|
|
18775
19828
|
if (limited) return limited;
|
|
18776
|
-
const concurrencyLimited = await enforceSerpIntelligenceConcurrency(user);
|
|
18777
|
-
if (concurrencyLimited) return concurrencyLimited;
|
|
18778
19829
|
const targets = pageSnapshotTargetsFromBody(parsed.data);
|
|
19830
|
+
const gate = await acquireConcurrencyGate(user, "serp_intelligence_page_snapshots", {
|
|
19831
|
+
reuseLockId: c.req.header("x-mcp-scraper-concurrency-lock"),
|
|
19832
|
+
metadata: { targets: targets.length }
|
|
19833
|
+
});
|
|
19834
|
+
if (!gate.ok) return c.json(concurrencyLimitExceededResponse(gate), 429, { "Retry-After": String(gate.retryAfterSeconds) });
|
|
18779
19835
|
const cost = targets.length * MC_COSTS.page_scrape;
|
|
18780
|
-
|
|
18781
|
-
user.id,
|
|
18782
|
-
cost,
|
|
18783
|
-
LedgerOperation.EXTRACT_URL,
|
|
18784
|
-
`serp intelligence page snapshots: ${targets.length}`
|
|
18785
|
-
);
|
|
18786
|
-
if (!ok) return c.json(insufficientBalanceResponse(balance_mc, cost), 402);
|
|
19836
|
+
let debited = false;
|
|
18787
19837
|
try {
|
|
19838
|
+
const { ok, balance_mc } = await debitMc(
|
|
19839
|
+
user.id,
|
|
19840
|
+
cost,
|
|
19841
|
+
LedgerOperation.EXTRACT_URL,
|
|
19842
|
+
`serp intelligence page snapshots: ${targets.length}`
|
|
19843
|
+
);
|
|
19844
|
+
if (!ok) return c.json(insufficientBalanceResponse(balance_mc, cost), 402);
|
|
19845
|
+
debited = true;
|
|
18788
19846
|
const result = await capturePageSnapshots(targets, {
|
|
18789
19847
|
kernelApiKey: browserServiceApiKey(),
|
|
18790
19848
|
timeoutMs: parsed.data.timeoutMs,
|
|
@@ -18801,7 +19859,7 @@ var init_serp_intelligence_routes = __esm({
|
|
|
18801
19859
|
});
|
|
18802
19860
|
return c.json(result);
|
|
18803
19861
|
} catch (error) {
|
|
18804
|
-
await creditMc(user.id, cost, LedgerOperation.REFUND, "failed serp intelligence page snapshots");
|
|
19862
|
+
if (debited) await creditMc(user.id, cost, LedgerOperation.REFUND, "failed serp intelligence page snapshots");
|
|
18805
19863
|
const body = structuredError({
|
|
18806
19864
|
error_code: "page_snapshot_failed",
|
|
18807
19865
|
error_type: "capture_error",
|
|
@@ -18816,6 +19874,8 @@ var init_serp_intelligence_routes = __esm({
|
|
|
18816
19874
|
error: body.message
|
|
18817
19875
|
});
|
|
18818
19876
|
return c.json(body, 500);
|
|
19877
|
+
} finally {
|
|
19878
|
+
await releaseConcurrencyGate(gate.lockId);
|
|
18819
19879
|
}
|
|
18820
19880
|
});
|
|
18821
19881
|
}
|
|
@@ -18826,133 +19886,133 @@ var PACKAGE_VERSION;
|
|
|
18826
19886
|
var init_version = __esm({
|
|
18827
19887
|
"src/version.ts"() {
|
|
18828
19888
|
"use strict";
|
|
18829
|
-
PACKAGE_VERSION = "0.2.
|
|
19889
|
+
PACKAGE_VERSION = "0.2.14";
|
|
18830
19890
|
}
|
|
18831
19891
|
});
|
|
18832
19892
|
|
|
18833
19893
|
// src/mcp/mcp-tool-schemas.ts
|
|
18834
|
-
var
|
|
19894
|
+
var import_zod26, HarvestPaaInputSchema, ExtractUrlInputSchema, MapSiteUrlsInputSchema, ExtractSiteInputSchema, YoutubeHarvestInputSchema, YoutubeTranscribeInputSchema, FacebookPageIntelInputSchema, FacebookAdSearchInputSchema, FacebookAdTranscribeInputSchema, MapsPlaceIntelInputSchema, MapsSearchInputSchema, DirectoryWorkflowInputSchema, RankTrackerModeSchema, RankTrackerBlueprintInputSchema, NullableString, MapsSearchAttemptOutput, MapsSearchOutputSchema, DirectoryMapsBusinessOutput, DirectoryWorkflowOutputSchema, RankTrackerToolPlanOutput, RankTrackerTableOutput, RankTrackerCronJobOutput, RankTrackerBlueprintOutputSchema, OrganicResultOutput, AiOverviewOutput, EntityIdsOutput, HarvestPaaOutputSchema, SearchSerpOutputSchema, ExtractUrlOutputSchema, ExtractSiteOutputSchema, MapsPlaceIntelOutputSchema, CreditsInfoOutputSchema, MapSiteUrlsOutputSchema, YoutubeHarvestOutputSchema, FacebookAdSearchOutputSchema, FacebookPageIntelOutputSchema, CreditsInfoInputSchema, SearchSerpInputSchema, CaptureSerpSnapshotInputSchema, ScreenshotInputSchema, CaptureSerpPageSnapshotsInputSchema;
|
|
18835
19895
|
var init_mcp_tool_schemas = __esm({
|
|
18836
19896
|
"src/mcp/mcp-tool-schemas.ts"() {
|
|
18837
19897
|
"use strict";
|
|
18838
|
-
|
|
19898
|
+
import_zod26 = require("zod");
|
|
18839
19899
|
HarvestPaaInputSchema = {
|
|
18840
|
-
query:
|
|
18841
|
-
location:
|
|
18842
|
-
maxQuestions:
|
|
18843
|
-
gl:
|
|
18844
|
-
hl:
|
|
18845
|
-
device:
|
|
18846
|
-
proxyMode:
|
|
18847
|
-
proxyZip:
|
|
18848
|
-
debug:
|
|
19900
|
+
query: import_zod26.z.string().min(1).describe('Core search topic only. If the user says "best hvac company in Denver CO", use query="best hvac company" and location="Denver, CO". Do not include the location in query when it can be separated.'),
|
|
19901
|
+
location: import_zod26.z.string().optional().describe('City, region, or country for geo-targeted results, inferred from the user request when present, e.g. "Denver, CO", "Tokyo, Japan", "London, UK".'),
|
|
19902
|
+
maxQuestions: import_zod26.z.number().int().min(1).max(200).default(30).describe("Number of PAA questions to extract. Default 30. Maximum 200. Use 10 for quick probes, 30 for normal research, 100-200 when the user asks for everything/full/deep research. Larger harvests get a longer server time budget (151-200 questions \u2192 up to 280s). Credits are charged by extracted question; unused request hold is refunded."),
|
|
19903
|
+
gl: import_zod26.z.string().length(2).default("us").describe("Google country code inferred from location or user language. Examples: United States us, United Kingdom gb, Japan jp, Canada ca, Australia au."),
|
|
19904
|
+
hl: import_zod26.z.string().default("en").describe("Google interface/content language inferred from the user request. Use en unless the user asks for another language or locale."),
|
|
19905
|
+
device: import_zod26.z.enum(["desktop", "mobile"]).default("desktop").describe("SERP device context. Use desktop by default; use mobile only when the user asks for mobile rankings."),
|
|
19906
|
+
proxyMode: import_zod26.z.enum(["location", "configured", "none"]).default("location").describe("Proxy targeting mode. Use location by default for US city/state SERPs; it creates a fresh residential proxy ID per attempt. If Google shows a CAPTCHA/challenge, browser-service sessions briefly wait for the automatic solver first, then retry with a fresh proxy/session if the challenge does not clear. Also retries proxy tunnel failure and wrong-location evidence before returning. Use configured only for the static configured proxy. Use none only for direct-network debugging."),
|
|
19907
|
+
proxyZip: import_zod26.z.string().regex(/^\d{5}$/).optional().describe("Optional US ZIP override for residential location proxy targeting. Use when the user gives a specific ZIP or when city-center targeting needs to be forced. With proxyMode location this ZIP is used for each fresh proxy attempt."),
|
|
19908
|
+
debug: import_zod26.z.boolean().default(false).describe("Include sanitized browser/session/location diagnostics in the response. Use true when debugging localization, CAPTCHA, or proxy behavior.")
|
|
18849
19909
|
};
|
|
18850
19910
|
ExtractUrlInputSchema = {
|
|
18851
|
-
url:
|
|
18852
|
-
screenshot:
|
|
18853
|
-
screenshotDevice:
|
|
18854
|
-
extractBranding:
|
|
18855
|
-
downloadMedia:
|
|
18856
|
-
mediaTypes:
|
|
18857
|
-
allowLocal:
|
|
19911
|
+
url: import_zod26.z.string().url().describe("Public http/https URL to extract. Use this when the user provides one specific page URL."),
|
|
19912
|
+
screenshot: import_zod26.z.boolean().default(false).describe("Also capture a full-page screenshot of the URL. Saved to ~/Downloads/mcp-scraper/screenshots/ and returned inline. Use when the user asks to see or capture the page visually."),
|
|
19913
|
+
screenshotDevice: import_zod26.z.enum(["desktop", "mobile"]).default("desktop").describe("Viewport for screenshot. desktop = 1440\xD7900. mobile = 390\xD7844. Default desktop."),
|
|
19914
|
+
extractBranding: import_zod26.z.boolean().default(false).describe("Extract brand colors, fonts, logo, and favicon using a rendered browser session. Returns colorScheme (light/dark), colors (primary/accent/background/text/heading as hex), fonts (heading/body family names), and assets (logo URL, favicon URL). Use when the user asks about brand colors, site theme, or brand assets."),
|
|
19915
|
+
downloadMedia: import_zod26.z.boolean().default(false).describe("Extract and download all page media (images, video, audio) to ~/Downloads/mcp-scraper/media/. Ad networks, tracking pixels, and noise URLs are filtered automatically. Use when the user asks to download or harvest assets from a page."),
|
|
19916
|
+
mediaTypes: import_zod26.z.array(import_zod26.z.enum(["image", "video", "audio"])).default(["image", "video", "audio"]).describe("Which media types to download. Default all three."),
|
|
19917
|
+
allowLocal: import_zod26.z.boolean().default(false).describe("Allow localhost and private-network URLs. For local development only.")
|
|
18858
19918
|
};
|
|
18859
19919
|
MapSiteUrlsInputSchema = {
|
|
18860
|
-
url:
|
|
18861
|
-
maxUrls:
|
|
19920
|
+
url: import_zod26.z.string().url().describe("Public website URL or domain to crawl for internal URLs. Use before extract_site when the user asks to audit/map/crawl a site."),
|
|
19921
|
+
maxUrls: import_zod26.z.number().int().min(1).max(500).optional().describe("Maximum URLs to discover. Use 100 for normal maps, higher when the user asks for a full inventory.")
|
|
18862
19922
|
};
|
|
18863
19923
|
ExtractSiteInputSchema = {
|
|
18864
|
-
url:
|
|
18865
|
-
maxPages:
|
|
19924
|
+
url: import_zod26.z.string().url().describe("Public website URL or domain to extract across multiple pages. Use when the user asks for a site audit, website crawl, or full-site content/schema extraction."),
|
|
19925
|
+
maxPages: import_zod26.z.number().int().min(1).max(50).optional().describe("Maximum pages to extract. Use 50 when the user asks for full results or a complete crawl within MCP limits.")
|
|
18866
19926
|
};
|
|
18867
19927
|
YoutubeHarvestInputSchema = {
|
|
18868
|
-
mode:
|
|
18869
|
-
query:
|
|
18870
|
-
channelHandle:
|
|
18871
|
-
maxVideos:
|
|
19928
|
+
mode: import_zod26.z.enum(["search", "channel"]).describe("Use search for topic/keyword requests. Use channel when the user provides @handle, channel ID, or channel URL."),
|
|
19929
|
+
query: import_zod26.z.string().optional().describe("Required when mode is search. The YouTube search topic in the user\u2019s words."),
|
|
19930
|
+
channelHandle: import_zod26.z.string().optional().describe("YouTube channel handle, channel ID, or URL. Examples: @mkbhd, UC..., https://youtube.com/@mkbhd."),
|
|
19931
|
+
maxVideos: import_zod26.z.number().int().min(1).max(500).default(50).describe("Number of videos to return. Default 50. Increase when user asks for full channel/history.")
|
|
18872
19932
|
};
|
|
18873
19933
|
YoutubeTranscribeInputSchema = {
|
|
18874
|
-
videoId:
|
|
19934
|
+
videoId: import_zod26.z.string().min(1).describe("YouTube video ID, e.g. dQw4w9WgXcQ")
|
|
18875
19935
|
};
|
|
18876
19936
|
FacebookPageIntelInputSchema = {
|
|
18877
|
-
pageId:
|
|
18878
|
-
libraryId:
|
|
18879
|
-
query:
|
|
18880
|
-
maxAds:
|
|
18881
|
-
country:
|
|
19937
|
+
pageId: import_zod26.z.string().optional(),
|
|
19938
|
+
libraryId: import_zod26.z.string().optional(),
|
|
19939
|
+
query: import_zod26.z.string().optional().describe("Advertiser or brand name when pageId/libraryId is not known. One of pageId, libraryId, or query is required."),
|
|
19940
|
+
maxAds: import_zod26.z.number().int().min(1).max(200).default(50),
|
|
19941
|
+
country: import_zod26.z.string().length(2).default("US")
|
|
18882
19942
|
};
|
|
18883
19943
|
FacebookAdSearchInputSchema = {
|
|
18884
|
-
query:
|
|
18885
|
-
country:
|
|
18886
|
-
maxResults:
|
|
19944
|
+
query: import_zod26.z.string().min(1).describe("Advertiser, brand, competitor, niche, or keyword to search in Facebook Ad Library."),
|
|
19945
|
+
country: import_zod26.z.string().length(2).default("US"),
|
|
19946
|
+
maxResults: import_zod26.z.number().int().min(1).max(20).default(10)
|
|
18887
19947
|
};
|
|
18888
19948
|
FacebookAdTranscribeInputSchema = {
|
|
18889
|
-
videoUrl:
|
|
19949
|
+
videoUrl: import_zod26.z.string().url().describe("Facebook CDN video URL from a facebook_page_intel result")
|
|
18890
19950
|
};
|
|
18891
19951
|
MapsPlaceIntelInputSchema = {
|
|
18892
|
-
businessName:
|
|
18893
|
-
location:
|
|
18894
|
-
gl:
|
|
18895
|
-
hl:
|
|
18896
|
-
includeReviews:
|
|
18897
|
-
maxReviews:
|
|
19952
|
+
businessName: import_zod26.z.string().min(1).describe('Business name only. If user says "Elite Roofing Denver CO", use businessName="Elite Roofing" and location="Denver, CO".'),
|
|
19953
|
+
location: import_zod26.z.string().min(1).describe('City/region/country where the business should be searched, e.g. "Denver, CO". Infer from the user request when possible.'),
|
|
19954
|
+
gl: import_zod26.z.string().length(2).default("us").describe("Google country code inferred from location."),
|
|
19955
|
+
hl: import_zod26.z.string().length(2).default("en").describe("Language inferred from user request."),
|
|
19956
|
+
includeReviews: import_zod26.z.boolean().default(false).describe("Whether to fetch individual review cards"),
|
|
19957
|
+
maxReviews: import_zod26.z.number().int().min(1).max(500).default(50).describe("Max review cards to return (requires includeReviews: true)")
|
|
18898
19958
|
};
|
|
18899
19959
|
MapsSearchInputSchema = {
|
|
18900
|
-
query:
|
|
18901
|
-
location:
|
|
18902
|
-
gl:
|
|
18903
|
-
hl:
|
|
18904
|
-
maxResults:
|
|
18905
|
-
proxyMode:
|
|
18906
|
-
proxyZip:
|
|
18907
|
-
debug:
|
|
19960
|
+
query: import_zod26.z.string().min(1).describe('Business category, niche, keyword, or search term. If the user says "roofers in Denver CO", use query="roofers" and location="Denver, CO". Do not put the location here when it can be separated.'),
|
|
19961
|
+
location: import_zod26.z.string().optional().describe('City, region, country, or service area for the Maps search, e.g. "Denver, CO". Infer from the user request when present.'),
|
|
19962
|
+
gl: import_zod26.z.string().length(2).default("us").describe("Google country code inferred from location."),
|
|
19963
|
+
hl: import_zod26.z.string().length(2).default("en").describe("Language inferred from user request."),
|
|
19964
|
+
maxResults: import_zod26.z.number().int().min(1).max(50).default(10).describe("Number of Google Maps business/profile candidates to return. Default 10. Maximum 50. Use 10 unless the user asks for more."),
|
|
19965
|
+
proxyMode: import_zod26.z.enum(["location", "configured", "none"]).default("location").describe("Proxy targeting mode. Use location by default for US city/state Maps searches; retryable failures create a new residential proxy ID and new browser session for up to 5 attempts. Use configured for the server proxy ID, and none only for local direct-network debugging."),
|
|
19966
|
+
proxyZip: import_zod26.z.string().regex(/^\d{5}$/).optional().describe("Optional US ZIP override for residential location proxy targeting. Use when the user gives a specific ZIP or city-center ZIP."),
|
|
19967
|
+
debug: import_zod26.z.boolean().default(false).describe("Include sanitized browser/proxy diagnostics when debugging Maps localization, CAPTCHA, or proxy behavior.")
|
|
18908
19968
|
};
|
|
18909
19969
|
DirectoryWorkflowInputSchema = {
|
|
18910
|
-
query:
|
|
18911
|
-
state:
|
|
18912
|
-
minPopulation:
|
|
18913
|
-
populationYear:
|
|
18914
|
-
maxCities:
|
|
18915
|
-
maxResultsPerCity:
|
|
18916
|
-
concurrency:
|
|
18917
|
-
includeZipGroups:
|
|
18918
|
-
usZipsCsvPath:
|
|
18919
|
-
saveCsv:
|
|
18920
|
-
proxyMode:
|
|
18921
|
-
proxyZip:
|
|
18922
|
-
debug:
|
|
19970
|
+
query: import_zod26.z.string().min(1).describe("Business category, niche, or keyword to search on Google Maps for every selected market, e.g. roofers, dentists, med spas. Do not include the city here."),
|
|
19971
|
+
state: import_zod26.z.string().min(2).default("TN").describe("US state abbreviation or state name used to select Census places, e.g. TN or Tennessee."),
|
|
19972
|
+
minPopulation: import_zod26.z.number().int().min(0).default(1e5).describe('Minimum Census place population for market selection. Use 100000 for "cities above 100k population".'),
|
|
19973
|
+
populationYear: import_zod26.z.number().int().min(2020).max(2025).default(2025).describe("Census population estimate year from the 2020-2025 Population Estimates Program city/place dataset."),
|
|
19974
|
+
maxCities: import_zod26.z.number().int().min(1).max(100).default(25).describe("Maximum number of markets to process after sorting by population descending."),
|
|
19975
|
+
maxResultsPerCity: import_zod26.z.number().int().min(1).max(50).default(50).describe("Google Maps business/profile candidates to collect for each city. Maximum 50."),
|
|
19976
|
+
concurrency: import_zod26.z.number().int().min(1).max(5).default(5).describe("How many city Maps searches to run in parallel. Use 5 for broad directory batches unless debugging."),
|
|
19977
|
+
includeZipGroups: import_zod26.z.boolean().default(true).describe("Attach ZIP groups from a configured US ZIPS CSV when available. Set MCP_SCRAPER_USZIPS_CSV_PATH on the API server or pass usZipsCsvPath in local/test mode."),
|
|
19978
|
+
usZipsCsvPath: import_zod26.z.string().optional().describe("Local/test-only path to a US ZIPS CSV with state_abbr, zipcode, county, city columns, such as Lead Magician tools/analytics/data/uszips.csv. Deployed APIs should use MCP_SCRAPER_USZIPS_CSV_PATH instead."),
|
|
19979
|
+
saveCsv: import_zod26.z.boolean().default(true).describe("Save a directory-ready CSV to the MCP Scraper output directory and return its path. CSV rows include source_location, result_position, business_name, review_stars, review_count, category, address, phone, hours_status, website_url, directions_url, place_url, CID fields, population, and ZIP groups."),
|
|
19980
|
+
proxyMode: import_zod26.z.enum(["location", "configured", "none"]).default("location").describe("Proxy targeting mode for every city Maps search. Use location by default for US city/state batches; retryable failures create a new residential proxy ID and new browser session for up to 5 attempts per city. Use configured for the server proxy ID, and none only for local direct-network debugging."),
|
|
19981
|
+
proxyZip: import_zod26.z.string().regex(/^\d{5}$/).optional().describe("Optional ZIP override for proxy targeting. Normally omit it so each city can use its Lead Magician ZIP group or city/state location."),
|
|
19982
|
+
debug: import_zod26.z.boolean().default(false).describe("Include sanitized browser/proxy diagnostics in each Maps browser session when supported.")
|
|
18923
19983
|
};
|
|
18924
|
-
RankTrackerModeSchema =
|
|
19984
|
+
RankTrackerModeSchema = import_zod26.z.enum(["maps", "organic", "ai_overview", "paa"]);
|
|
18925
19985
|
RankTrackerBlueprintInputSchema = {
|
|
18926
|
-
projectName:
|
|
18927
|
-
targetDomain:
|
|
18928
|
-
targetBusinessName:
|
|
18929
|
-
trackingModes:
|
|
18930
|
-
keywords:
|
|
18931
|
-
locations:
|
|
18932
|
-
competitors:
|
|
18933
|
-
database:
|
|
18934
|
-
scheduleCadence:
|
|
18935
|
-
customCron:
|
|
18936
|
-
timezone:
|
|
18937
|
-
includeCron:
|
|
18938
|
-
includeDashboard:
|
|
18939
|
-
includeAlerts:
|
|
18940
|
-
notes:
|
|
19986
|
+
projectName: import_zod26.z.string().min(1).optional().describe("Optional name for the rank tracker project, client, or campaign."),
|
|
19987
|
+
targetDomain: import_zod26.z.string().min(1).optional().describe("Primary domain to track in organic results, AI Overview citations, and PAA sources, e.g. example.com."),
|
|
19988
|
+
targetBusinessName: import_zod26.z.string().min(1).optional().describe("Primary Google Business Profile or brand/business name to match in Maps results. Required for reliable Maps rank tracking."),
|
|
19989
|
+
trackingModes: import_zod26.z.array(RankTrackerModeSchema).min(1).max(4).default(["maps", "organic", "ai_overview", "paa"]).describe("Rank tracker surfaces to build. maps uses directory_workflow/maps_search. organic uses search_serp. ai_overview uses search_serp/harvest_paa. paa uses harvest_paa source presence."),
|
|
19990
|
+
keywords: import_zod26.z.array(import_zod26.z.string().min(1)).max(200).default([]).describe("Seed keywords or service queries to track. Leave empty when the downstream AI should create the keyword table from user input."),
|
|
19991
|
+
locations: import_zod26.z.array(import_zod26.z.string().min(1)).max(100).default([]).describe("Markets, cities, ZIPs, or service areas to track. Use city/state strings like Denver, CO for localized SERP and Maps checks."),
|
|
19992
|
+
competitors: import_zod26.z.array(import_zod26.z.string().min(1)).max(100).default([]).describe("Optional competitor domains or business names to persist as comparison targets."),
|
|
19993
|
+
database: import_zod26.z.enum(["postgres", "neon", "supabase", "sqlite", "mysql"]).default("postgres").describe("Database family the downstream AI should target when generating migrations."),
|
|
19994
|
+
scheduleCadence: import_zod26.z.enum(["daily", "weekly", "monthly", "custom"]).default("weekly").describe("Default recurring rank check cadence for the generated cron/heartbeat plan."),
|
|
19995
|
+
customCron: import_zod26.z.string().min(1).optional().describe("Cron expression to use when scheduleCadence is custom."),
|
|
19996
|
+
timezone: import_zod26.z.string().min(1).default("UTC").describe("IANA timezone for scheduled rank checks, e.g. America/Denver."),
|
|
19997
|
+
includeCron: import_zod26.z.boolean().default(true).describe("Include a cron or heartbeat worker plan. Keep true for production rank trackers."),
|
|
19998
|
+
includeDashboard: import_zod26.z.boolean().default(true).describe("Include dashboard/reporting requirements in the generated prompt."),
|
|
19999
|
+
includeAlerts: import_zod26.z.boolean().default(true).describe("Include alert rules for rank movement and SERP feature gains/losses."),
|
|
20000
|
+
notes: import_zod26.z.string().max(4e3).optional().describe("Extra product, client, stack, or hosting requirements to include in the implementation prompt.")
|
|
18941
20001
|
};
|
|
18942
|
-
NullableString =
|
|
18943
|
-
MapsSearchAttemptOutput =
|
|
18944
|
-
attemptNumber:
|
|
18945
|
-
maxAttempts:
|
|
18946
|
-
status:
|
|
18947
|
-
outcome:
|
|
18948
|
-
willRetry:
|
|
18949
|
-
durationMs:
|
|
18950
|
-
resultCount:
|
|
20002
|
+
NullableString = import_zod26.z.string().nullable();
|
|
20003
|
+
MapsSearchAttemptOutput = import_zod26.z.object({
|
|
20004
|
+
attemptNumber: import_zod26.z.number().int().min(1),
|
|
20005
|
+
maxAttempts: import_zod26.z.number().int().min(1),
|
|
20006
|
+
status: import_zod26.z.enum(["ok", "failed"]),
|
|
20007
|
+
outcome: import_zod26.z.string(),
|
|
20008
|
+
willRetry: import_zod26.z.boolean(),
|
|
20009
|
+
durationMs: import_zod26.z.number().int().min(0),
|
|
20010
|
+
resultCount: import_zod26.z.number().int().min(0),
|
|
18951
20011
|
error: NullableString,
|
|
18952
|
-
proxyMode:
|
|
18953
|
-
proxyResolutionSource:
|
|
20012
|
+
proxyMode: import_zod26.z.enum(["location", "configured", "none"]),
|
|
20013
|
+
proxyResolutionSource: import_zod26.z.enum(["disabled", "location_reused", "location_created", "configured_fallback", "unavailable"]).nullable(),
|
|
18954
20014
|
proxyIdSuffix: NullableString,
|
|
18955
|
-
proxyTargetLevel:
|
|
20015
|
+
proxyTargetLevel: import_zod26.z.enum(["zip", "city", "state"]).nullable(),
|
|
18956
20016
|
proxyTargetLocation: NullableString,
|
|
18957
20017
|
proxyTargetZip: NullableString,
|
|
18958
20018
|
browserSessionIdSuffix: NullableString,
|
|
@@ -18961,17 +20021,17 @@ var init_mcp_tool_schemas = __esm({
|
|
|
18961
20021
|
observedRegion: NullableString
|
|
18962
20022
|
});
|
|
18963
20023
|
MapsSearchOutputSchema = {
|
|
18964
|
-
query:
|
|
18965
|
-
location:
|
|
18966
|
-
searchQuery:
|
|
18967
|
-
searchUrl:
|
|
18968
|
-
extractedAt:
|
|
18969
|
-
requestedMaxResults:
|
|
18970
|
-
resultCount:
|
|
18971
|
-
results:
|
|
18972
|
-
position:
|
|
18973
|
-
name:
|
|
18974
|
-
placeUrl:
|
|
20024
|
+
query: import_zod26.z.string(),
|
|
20025
|
+
location: import_zod26.z.string().nullable(),
|
|
20026
|
+
searchQuery: import_zod26.z.string(),
|
|
20027
|
+
searchUrl: import_zod26.z.string().url(),
|
|
20028
|
+
extractedAt: import_zod26.z.string(),
|
|
20029
|
+
requestedMaxResults: import_zod26.z.number().int().min(1).max(50),
|
|
20030
|
+
resultCount: import_zod26.z.number().int().min(0).max(50),
|
|
20031
|
+
results: import_zod26.z.array(import_zod26.z.object({
|
|
20032
|
+
position: import_zod26.z.number().int().min(1),
|
|
20033
|
+
name: import_zod26.z.string(),
|
|
20034
|
+
placeUrl: import_zod26.z.string().url(),
|
|
18975
20035
|
cid: NullableString,
|
|
18976
20036
|
cidDecimal: NullableString,
|
|
18977
20037
|
rating: NullableString,
|
|
@@ -18982,15 +20042,15 @@ var init_mcp_tool_schemas = __esm({
|
|
|
18982
20042
|
hoursStatus: NullableString,
|
|
18983
20043
|
websiteUrl: NullableString,
|
|
18984
20044
|
directionsUrl: NullableString,
|
|
18985
|
-
metadata:
|
|
20045
|
+
metadata: import_zod26.z.array(import_zod26.z.string())
|
|
18986
20046
|
})),
|
|
18987
|
-
attempts:
|
|
18988
|
-
durationMs:
|
|
20047
|
+
attempts: import_zod26.z.array(MapsSearchAttemptOutput),
|
|
20048
|
+
durationMs: import_zod26.z.number().int().min(0)
|
|
18989
20049
|
};
|
|
18990
|
-
DirectoryMapsBusinessOutput =
|
|
18991
|
-
position:
|
|
18992
|
-
name:
|
|
18993
|
-
placeUrl:
|
|
20050
|
+
DirectoryMapsBusinessOutput = import_zod26.z.object({
|
|
20051
|
+
position: import_zod26.z.number().int().min(1),
|
|
20052
|
+
name: import_zod26.z.string(),
|
|
20053
|
+
placeUrl: import_zod26.z.string().url(),
|
|
18994
20054
|
cid: NullableString,
|
|
18995
20055
|
cidDecimal: NullableString,
|
|
18996
20056
|
rating: NullableString,
|
|
@@ -19001,113 +20061,113 @@ var init_mcp_tool_schemas = __esm({
|
|
|
19001
20061
|
hoursStatus: NullableString,
|
|
19002
20062
|
websiteUrl: NullableString,
|
|
19003
20063
|
directionsUrl: NullableString,
|
|
19004
|
-
metadata:
|
|
20064
|
+
metadata: import_zod26.z.array(import_zod26.z.string())
|
|
19005
20065
|
});
|
|
19006
20066
|
DirectoryWorkflowOutputSchema = {
|
|
19007
|
-
query:
|
|
19008
|
-
state:
|
|
19009
|
-
minPopulation:
|
|
19010
|
-
populationYear:
|
|
19011
|
-
maxResultsPerCity:
|
|
19012
|
-
concurrency:
|
|
19013
|
-
censusSourceUrl:
|
|
20067
|
+
query: import_zod26.z.string(),
|
|
20068
|
+
state: import_zod26.z.string(),
|
|
20069
|
+
minPopulation: import_zod26.z.number().int().min(0),
|
|
20070
|
+
populationYear: import_zod26.z.number().int().min(2020).max(2025),
|
|
20071
|
+
maxResultsPerCity: import_zod26.z.number().int().min(1).max(50),
|
|
20072
|
+
concurrency: import_zod26.z.number().int().min(1).max(5),
|
|
20073
|
+
censusSourceUrl: import_zod26.z.string().url(),
|
|
19014
20074
|
usZipsSourcePath: NullableString,
|
|
19015
|
-
warnings:
|
|
19016
|
-
extractedAt:
|
|
19017
|
-
selectedCityCount:
|
|
19018
|
-
totalResultCount:
|
|
20075
|
+
warnings: import_zod26.z.array(import_zod26.z.string()),
|
|
20076
|
+
extractedAt: import_zod26.z.string(),
|
|
20077
|
+
selectedCityCount: import_zod26.z.number().int().min(0),
|
|
20078
|
+
totalResultCount: import_zod26.z.number().int().min(0),
|
|
19019
20079
|
csvPath: NullableString,
|
|
19020
|
-
cities:
|
|
19021
|
-
city:
|
|
19022
|
-
state:
|
|
19023
|
-
location:
|
|
19024
|
-
cityKey:
|
|
19025
|
-
censusName:
|
|
19026
|
-
population:
|
|
19027
|
-
populationYear:
|
|
19028
|
-
zips:
|
|
19029
|
-
counties:
|
|
19030
|
-
status:
|
|
20080
|
+
cities: import_zod26.z.array(import_zod26.z.object({
|
|
20081
|
+
city: import_zod26.z.string(),
|
|
20082
|
+
state: import_zod26.z.string(),
|
|
20083
|
+
location: import_zod26.z.string(),
|
|
20084
|
+
cityKey: import_zod26.z.string(),
|
|
20085
|
+
censusName: import_zod26.z.string(),
|
|
20086
|
+
population: import_zod26.z.number().int().min(0),
|
|
20087
|
+
populationYear: import_zod26.z.number().int().min(2020).max(2025),
|
|
20088
|
+
zips: import_zod26.z.array(import_zod26.z.string()),
|
|
20089
|
+
counties: import_zod26.z.array(import_zod26.z.string()),
|
|
20090
|
+
status: import_zod26.z.enum(["ok", "empty", "failed"]),
|
|
19031
20091
|
error: NullableString,
|
|
19032
|
-
resultCount:
|
|
19033
|
-
durationMs:
|
|
19034
|
-
attempts:
|
|
19035
|
-
results:
|
|
20092
|
+
resultCount: import_zod26.z.number().int().min(0),
|
|
20093
|
+
durationMs: import_zod26.z.number().int().min(0),
|
|
20094
|
+
attempts: import_zod26.z.array(MapsSearchAttemptOutput),
|
|
20095
|
+
results: import_zod26.z.array(DirectoryMapsBusinessOutput)
|
|
19036
20096
|
})),
|
|
19037
|
-
durationMs:
|
|
20097
|
+
durationMs: import_zod26.z.number().int().min(0)
|
|
19038
20098
|
};
|
|
19039
|
-
RankTrackerToolPlanOutput =
|
|
19040
|
-
tool:
|
|
19041
|
-
purpose:
|
|
20099
|
+
RankTrackerToolPlanOutput = import_zod26.z.object({
|
|
20100
|
+
tool: import_zod26.z.string(),
|
|
20101
|
+
purpose: import_zod26.z.string()
|
|
19042
20102
|
});
|
|
19043
|
-
RankTrackerTableOutput =
|
|
19044
|
-
name:
|
|
19045
|
-
purpose:
|
|
19046
|
-
keyColumns:
|
|
20103
|
+
RankTrackerTableOutput = import_zod26.z.object({
|
|
20104
|
+
name: import_zod26.z.string(),
|
|
20105
|
+
purpose: import_zod26.z.string(),
|
|
20106
|
+
keyColumns: import_zod26.z.array(import_zod26.z.string())
|
|
19047
20107
|
});
|
|
19048
|
-
RankTrackerCronJobOutput =
|
|
19049
|
-
name:
|
|
19050
|
-
purpose:
|
|
19051
|
-
modes:
|
|
19052
|
-
recommendedTools:
|
|
20108
|
+
RankTrackerCronJobOutput = import_zod26.z.object({
|
|
20109
|
+
name: import_zod26.z.string(),
|
|
20110
|
+
purpose: import_zod26.z.string(),
|
|
20111
|
+
modes: import_zod26.z.array(RankTrackerModeSchema),
|
|
20112
|
+
recommendedTools: import_zod26.z.array(import_zod26.z.string())
|
|
19053
20113
|
});
|
|
19054
20114
|
RankTrackerBlueprintOutputSchema = {
|
|
19055
|
-
projectName:
|
|
20115
|
+
projectName: import_zod26.z.string(),
|
|
19056
20116
|
targetDomain: NullableString,
|
|
19057
20117
|
targetBusinessName: NullableString,
|
|
19058
|
-
trackingModes:
|
|
19059
|
-
database:
|
|
19060
|
-
recommendedTools:
|
|
19061
|
-
tables:
|
|
19062
|
-
cron:
|
|
19063
|
-
enabled:
|
|
19064
|
-
cadence:
|
|
19065
|
-
expression:
|
|
19066
|
-
timezone:
|
|
19067
|
-
jobs:
|
|
20118
|
+
trackingModes: import_zod26.z.array(RankTrackerModeSchema),
|
|
20119
|
+
database: import_zod26.z.string(),
|
|
20120
|
+
recommendedTools: import_zod26.z.array(RankTrackerToolPlanOutput),
|
|
20121
|
+
tables: import_zod26.z.array(RankTrackerTableOutput),
|
|
20122
|
+
cron: import_zod26.z.object({
|
|
20123
|
+
enabled: import_zod26.z.boolean(),
|
|
20124
|
+
cadence: import_zod26.z.string(),
|
|
20125
|
+
expression: import_zod26.z.string(),
|
|
20126
|
+
timezone: import_zod26.z.string(),
|
|
20127
|
+
jobs: import_zod26.z.array(RankTrackerCronJobOutput)
|
|
19068
20128
|
}),
|
|
19069
|
-
metrics:
|
|
19070
|
-
implementationPrompt:
|
|
20129
|
+
metrics: import_zod26.z.array(import_zod26.z.string()),
|
|
20130
|
+
implementationPrompt: import_zod26.z.string()
|
|
19071
20131
|
};
|
|
19072
|
-
OrganicResultOutput =
|
|
19073
|
-
position:
|
|
19074
|
-
title:
|
|
19075
|
-
url:
|
|
19076
|
-
domain:
|
|
20132
|
+
OrganicResultOutput = import_zod26.z.object({
|
|
20133
|
+
position: import_zod26.z.number().int(),
|
|
20134
|
+
title: import_zod26.z.string(),
|
|
20135
|
+
url: import_zod26.z.string(),
|
|
20136
|
+
domain: import_zod26.z.string(),
|
|
19077
20137
|
snippet: NullableString
|
|
19078
20138
|
});
|
|
19079
|
-
AiOverviewOutput =
|
|
19080
|
-
detected:
|
|
20139
|
+
AiOverviewOutput = import_zod26.z.object({
|
|
20140
|
+
detected: import_zod26.z.boolean(),
|
|
19081
20141
|
text: NullableString
|
|
19082
20142
|
}).nullable();
|
|
19083
|
-
EntityIdsOutput =
|
|
19084
|
-
kgIds:
|
|
19085
|
-
cids:
|
|
19086
|
-
gcids:
|
|
20143
|
+
EntityIdsOutput = import_zod26.z.object({
|
|
20144
|
+
kgIds: import_zod26.z.array(import_zod26.z.string()),
|
|
20145
|
+
cids: import_zod26.z.array(import_zod26.z.string()),
|
|
20146
|
+
gcids: import_zod26.z.array(import_zod26.z.string())
|
|
19087
20147
|
}).nullable();
|
|
19088
20148
|
HarvestPaaOutputSchema = {
|
|
19089
|
-
query:
|
|
20149
|
+
query: import_zod26.z.string(),
|
|
19090
20150
|
location: NullableString,
|
|
19091
|
-
questionCount:
|
|
20151
|
+
questionCount: import_zod26.z.number().int().min(0),
|
|
19092
20152
|
completionStatus: NullableString,
|
|
19093
|
-
questions:
|
|
19094
|
-
question:
|
|
20153
|
+
questions: import_zod26.z.array(import_zod26.z.object({
|
|
20154
|
+
question: import_zod26.z.string(),
|
|
19095
20155
|
answer: NullableString,
|
|
19096
20156
|
sourceTitle: NullableString,
|
|
19097
20157
|
sourceSite: NullableString
|
|
19098
20158
|
})),
|
|
19099
|
-
organicResults:
|
|
20159
|
+
organicResults: import_zod26.z.array(OrganicResultOutput),
|
|
19100
20160
|
aiOverview: AiOverviewOutput,
|
|
19101
20161
|
entityIds: EntityIdsOutput,
|
|
19102
|
-
durationMs:
|
|
20162
|
+
durationMs: import_zod26.z.number().min(0).nullable()
|
|
19103
20163
|
};
|
|
19104
20164
|
SearchSerpOutputSchema = {
|
|
19105
|
-
query:
|
|
20165
|
+
query: import_zod26.z.string(),
|
|
19106
20166
|
location: NullableString,
|
|
19107
|
-
organicResults:
|
|
19108
|
-
localPack:
|
|
19109
|
-
position:
|
|
19110
|
-
name:
|
|
20167
|
+
organicResults: import_zod26.z.array(OrganicResultOutput),
|
|
20168
|
+
localPack: import_zod26.z.array(import_zod26.z.object({
|
|
20169
|
+
position: import_zod26.z.number().int(),
|
|
20170
|
+
name: import_zod26.z.string(),
|
|
19111
20171
|
rating: NullableString,
|
|
19112
20172
|
reviewCount: NullableString,
|
|
19113
20173
|
websiteUrl: NullableString
|
|
@@ -19116,31 +20176,31 @@ var init_mcp_tool_schemas = __esm({
|
|
|
19116
20176
|
entityIds: EntityIdsOutput
|
|
19117
20177
|
};
|
|
19118
20178
|
ExtractUrlOutputSchema = {
|
|
19119
|
-
url:
|
|
20179
|
+
url: import_zod26.z.string(),
|
|
19120
20180
|
title: NullableString,
|
|
19121
|
-
headings:
|
|
19122
|
-
level:
|
|
19123
|
-
text:
|
|
20181
|
+
headings: import_zod26.z.array(import_zod26.z.object({
|
|
20182
|
+
level: import_zod26.z.number().int(),
|
|
20183
|
+
text: import_zod26.z.string()
|
|
19124
20184
|
})),
|
|
19125
|
-
schemaBlockCount:
|
|
20185
|
+
schemaBlockCount: import_zod26.z.number().int().min(0),
|
|
19126
20186
|
entityName: NullableString,
|
|
19127
|
-
entityTypes:
|
|
19128
|
-
napScore:
|
|
19129
|
-
missingSchemaFields:
|
|
20187
|
+
entityTypes: import_zod26.z.array(import_zod26.z.string()),
|
|
20188
|
+
napScore: import_zod26.z.number().nullable(),
|
|
20189
|
+
missingSchemaFields: import_zod26.z.array(import_zod26.z.string()),
|
|
19130
20190
|
screenshotSaved: NullableString
|
|
19131
20191
|
};
|
|
19132
20192
|
ExtractSiteOutputSchema = {
|
|
19133
|
-
url:
|
|
19134
|
-
pageCount:
|
|
19135
|
-
pages:
|
|
19136
|
-
url:
|
|
20193
|
+
url: import_zod26.z.string(),
|
|
20194
|
+
pageCount: import_zod26.z.number().int().min(0),
|
|
20195
|
+
pages: import_zod26.z.array(import_zod26.z.object({
|
|
20196
|
+
url: import_zod26.z.string(),
|
|
19137
20197
|
title: NullableString,
|
|
19138
|
-
schemaTypes:
|
|
20198
|
+
schemaTypes: import_zod26.z.array(import_zod26.z.string())
|
|
19139
20199
|
})),
|
|
19140
|
-
durationMs:
|
|
20200
|
+
durationMs: import_zod26.z.number().min(0)
|
|
19141
20201
|
};
|
|
19142
20202
|
MapsPlaceIntelOutputSchema = {
|
|
19143
|
-
name:
|
|
20203
|
+
name: import_zod26.z.string(),
|
|
19144
20204
|
rating: NullableString,
|
|
19145
20205
|
reviewCount: NullableString,
|
|
19146
20206
|
category: NullableString,
|
|
@@ -19152,60 +20212,75 @@ var init_mcp_tool_schemas = __esm({
|
|
|
19152
20212
|
kgmid: NullableString,
|
|
19153
20213
|
cidDecimal: NullableString,
|
|
19154
20214
|
cidUrl: NullableString,
|
|
19155
|
-
lat:
|
|
19156
|
-
lng:
|
|
19157
|
-
reviewsStatus:
|
|
19158
|
-
reviewsCollected:
|
|
19159
|
-
reviewTopics:
|
|
19160
|
-
label:
|
|
19161
|
-
count:
|
|
20215
|
+
lat: import_zod26.z.number().nullable(),
|
|
20216
|
+
lng: import_zod26.z.number().nullable(),
|
|
20217
|
+
reviewsStatus: import_zod26.z.string(),
|
|
20218
|
+
reviewsCollected: import_zod26.z.number().int().min(0),
|
|
20219
|
+
reviewTopics: import_zod26.z.array(import_zod26.z.object({
|
|
20220
|
+
label: import_zod26.z.string(),
|
|
20221
|
+
count: import_zod26.z.string()
|
|
19162
20222
|
}))
|
|
19163
20223
|
};
|
|
19164
20224
|
CreditsInfoOutputSchema = {
|
|
19165
|
-
balanceCredits:
|
|
19166
|
-
matchedCost:
|
|
19167
|
-
label:
|
|
19168
|
-
credits:
|
|
19169
|
-
unit:
|
|
20225
|
+
balanceCredits: import_zod26.z.number().nullable(),
|
|
20226
|
+
matchedCost: import_zod26.z.object({
|
|
20227
|
+
label: import_zod26.z.string(),
|
|
20228
|
+
credits: import_zod26.z.number(),
|
|
20229
|
+
unit: import_zod26.z.string(),
|
|
19170
20230
|
notes: NullableString
|
|
19171
20231
|
}).nullable(),
|
|
19172
|
-
costs:
|
|
19173
|
-
key:
|
|
19174
|
-
label:
|
|
19175
|
-
credits:
|
|
19176
|
-
unit:
|
|
20232
|
+
costs: import_zod26.z.array(import_zod26.z.object({
|
|
20233
|
+
key: import_zod26.z.string(),
|
|
20234
|
+
label: import_zod26.z.string(),
|
|
20235
|
+
credits: import_zod26.z.number(),
|
|
20236
|
+
unit: import_zod26.z.string(),
|
|
19177
20237
|
notes: NullableString
|
|
19178
20238
|
})),
|
|
19179
|
-
ledger:
|
|
19180
|
-
createdAt:
|
|
19181
|
-
operation:
|
|
19182
|
-
credits:
|
|
20239
|
+
ledger: import_zod26.z.array(import_zod26.z.object({
|
|
20240
|
+
createdAt: import_zod26.z.string(),
|
|
20241
|
+
operation: import_zod26.z.string(),
|
|
20242
|
+
credits: import_zod26.z.number(),
|
|
19183
20243
|
description: NullableString
|
|
19184
|
-
}))
|
|
20244
|
+
})),
|
|
20245
|
+
concurrency: import_zod26.z.object({
|
|
20246
|
+
currentExtraSlots: import_zod26.z.number().int().min(0),
|
|
20247
|
+
currentLimit: import_zod26.z.number().int().min(1),
|
|
20248
|
+
hasSubscription: import_zod26.z.boolean(),
|
|
20249
|
+
upgrade: import_zod26.z.object({
|
|
20250
|
+
product: import_zod26.z.string(),
|
|
20251
|
+
priceLabel: import_zod26.z.string(),
|
|
20252
|
+
unitAmountUsd: import_zod26.z.number(),
|
|
20253
|
+
currency: import_zod26.z.string(),
|
|
20254
|
+
interval: import_zod26.z.string(),
|
|
20255
|
+
billingUrl: import_zod26.z.string().url(),
|
|
20256
|
+
terminalCommand: import_zod26.z.string(),
|
|
20257
|
+
terminalCommandWithApiKeyEnv: import_zod26.z.string()
|
|
20258
|
+
})
|
|
20259
|
+
}).nullable()
|
|
19185
20260
|
};
|
|
19186
20261
|
MapSiteUrlsOutputSchema = {
|
|
19187
|
-
startUrl:
|
|
19188
|
-
totalFound:
|
|
19189
|
-
truncated:
|
|
19190
|
-
okCount:
|
|
19191
|
-
redirectCount:
|
|
19192
|
-
brokenCount:
|
|
19193
|
-
urls:
|
|
19194
|
-
url:
|
|
19195
|
-
status:
|
|
20262
|
+
startUrl: import_zod26.z.string(),
|
|
20263
|
+
totalFound: import_zod26.z.number().int().min(0),
|
|
20264
|
+
truncated: import_zod26.z.boolean(),
|
|
20265
|
+
okCount: import_zod26.z.number().int().min(0),
|
|
20266
|
+
redirectCount: import_zod26.z.number().int().min(0),
|
|
20267
|
+
brokenCount: import_zod26.z.number().int().min(0),
|
|
20268
|
+
urls: import_zod26.z.array(import_zod26.z.object({
|
|
20269
|
+
url: import_zod26.z.string(),
|
|
20270
|
+
status: import_zod26.z.number().int().nullable()
|
|
19196
20271
|
})),
|
|
19197
|
-
durationMs:
|
|
20272
|
+
durationMs: import_zod26.z.number().min(0)
|
|
19198
20273
|
};
|
|
19199
20274
|
YoutubeHarvestOutputSchema = {
|
|
19200
|
-
mode:
|
|
19201
|
-
videoCount:
|
|
19202
|
-
channel:
|
|
20275
|
+
mode: import_zod26.z.string(),
|
|
20276
|
+
videoCount: import_zod26.z.number().int().min(0),
|
|
20277
|
+
channel: import_zod26.z.object({
|
|
19203
20278
|
title: NullableString,
|
|
19204
20279
|
subscriberCount: NullableString
|
|
19205
20280
|
}).nullable(),
|
|
19206
|
-
videos:
|
|
19207
|
-
videoId:
|
|
19208
|
-
title:
|
|
20281
|
+
videos: import_zod26.z.array(import_zod26.z.object({
|
|
20282
|
+
videoId: import_zod26.z.string(),
|
|
20283
|
+
title: import_zod26.z.string(),
|
|
19209
20284
|
channelName: NullableString,
|
|
19210
20285
|
views: NullableString,
|
|
19211
20286
|
duration: NullableString,
|
|
@@ -19213,21 +20288,21 @@ var init_mcp_tool_schemas = __esm({
|
|
|
19213
20288
|
}))
|
|
19214
20289
|
};
|
|
19215
20290
|
FacebookAdSearchOutputSchema = {
|
|
19216
|
-
query:
|
|
19217
|
-
advertiserCount:
|
|
19218
|
-
advertisers:
|
|
20291
|
+
query: import_zod26.z.string(),
|
|
20292
|
+
advertiserCount: import_zod26.z.number().int().min(0),
|
|
20293
|
+
advertisers: import_zod26.z.array(import_zod26.z.object({
|
|
19219
20294
|
name: NullableString,
|
|
19220
|
-
adCount:
|
|
20295
|
+
adCount: import_zod26.z.number().int().nullable(),
|
|
19221
20296
|
libraryId: NullableString
|
|
19222
20297
|
}))
|
|
19223
20298
|
};
|
|
19224
20299
|
FacebookPageIntelOutputSchema = {
|
|
19225
20300
|
advertiserName: NullableString,
|
|
19226
|
-
totalAds:
|
|
19227
|
-
activeCount:
|
|
19228
|
-
videoCount:
|
|
19229
|
-
imageCount:
|
|
19230
|
-
ads:
|
|
20301
|
+
totalAds: import_zod26.z.number().int().min(0),
|
|
20302
|
+
activeCount: import_zod26.z.number().int().min(0),
|
|
20303
|
+
videoCount: import_zod26.z.number().int().min(0),
|
|
20304
|
+
imageCount: import_zod26.z.number().int().min(0),
|
|
20305
|
+
ads: import_zod26.z.array(import_zod26.z.object({
|
|
19231
20306
|
libraryId: NullableString,
|
|
19232
20307
|
status: NullableString,
|
|
19233
20308
|
creativeType: NullableString,
|
|
@@ -19235,58 +20310,58 @@ var init_mcp_tool_schemas = __esm({
|
|
|
19235
20310
|
cta: NullableString,
|
|
19236
20311
|
startDate: NullableString,
|
|
19237
20312
|
videoUrl: NullableString,
|
|
19238
|
-
variations:
|
|
20313
|
+
variations: import_zod26.z.number().int().nullable()
|
|
19239
20314
|
}))
|
|
19240
20315
|
};
|
|
19241
20316
|
CreditsInfoInputSchema = {
|
|
19242
|
-
item:
|
|
19243
|
-
includeLedger:
|
|
20317
|
+
item: import_zod26.z.string().optional().describe('Optional tool, action, or feature to look up, e.g. "maps reviews", "extract_url", "YouTube transcription", or "concurrency"'),
|
|
20318
|
+
includeLedger: import_zod26.z.boolean().default(false).describe("Whether to include recent credit ledger entries")
|
|
19244
20319
|
};
|
|
19245
20320
|
SearchSerpInputSchema = {
|
|
19246
|
-
query:
|
|
19247
|
-
location:
|
|
19248
|
-
gl:
|
|
19249
|
-
hl:
|
|
19250
|
-
device:
|
|
19251
|
-
proxyMode:
|
|
19252
|
-
proxyZip:
|
|
19253
|
-
debug:
|
|
19254
|
-
pages:
|
|
20321
|
+
query: import_zod26.z.string().min(1).describe('Core search topic only. Separate location when possible. If user says "best dentist in Brooklyn NY serp", use query="best dentist" and location="Brooklyn, NY".'),
|
|
20322
|
+
location: import_zod26.z.string().optional().describe("City, region, or country for geo-targeted results, inferred from user request when present."),
|
|
20323
|
+
gl: import_zod26.z.string().length(2).default("us").describe("Google country code inferred from location or user language."),
|
|
20324
|
+
hl: import_zod26.z.string().default("en").describe("Google interface/content language inferred from user request."),
|
|
20325
|
+
device: import_zod26.z.enum(["desktop", "mobile"]).default("desktop").describe("SERP device context. Use desktop by default; use mobile only when the user asks for mobile rankings."),
|
|
20326
|
+
proxyMode: import_zod26.z.enum(["location", "configured", "none"]).default("location").describe("Proxy targeting mode. Use location by default for US city/state SERPs; it creates a fresh residential proxy ID per attempt. If Google shows a CAPTCHA/challenge, browser-service sessions briefly wait for the automatic solver first, then retry with a fresh proxy/session if the challenge does not clear. Also retries proxy tunnel failure and wrong-location evidence before returning. Use configured only for the static configured proxy. Use none only for direct-network debugging."),
|
|
20327
|
+
proxyZip: import_zod26.z.string().regex(/^\d{5}$/).optional().describe("Optional US ZIP override for residential location proxy targeting. Use when the user gives a specific ZIP or when city-center targeting needs to be forced. With proxyMode location this ZIP is used for each fresh proxy attempt."),
|
|
20328
|
+
debug: import_zod26.z.boolean().default(false).describe("Include sanitized browser/session/location diagnostics in the response. Use true when debugging localization, CAPTCHA, or proxy behavior."),
|
|
20329
|
+
pages: import_zod26.z.number().int().min(1).max(2).default(1).describe("Number of result pages to fetch (1\u20132)")
|
|
19255
20330
|
};
|
|
19256
20331
|
CaptureSerpSnapshotInputSchema = {
|
|
19257
|
-
query:
|
|
19258
|
-
location:
|
|
19259
|
-
gl:
|
|
19260
|
-
hl:
|
|
19261
|
-
device:
|
|
19262
|
-
proxyMode:
|
|
19263
|
-
proxyZip:
|
|
19264
|
-
pages:
|
|
19265
|
-
debug:
|
|
19266
|
-
includePageSnapshots:
|
|
19267
|
-
pageSnapshotLimit:
|
|
20332
|
+
query: import_zod26.z.string().min(1).describe("Core search query to capture as a structured SERP Intelligence snapshot. Separate the place into location when the user gives a city, region, country, or ZIP."),
|
|
20333
|
+
location: import_zod26.z.string().optional().describe("City, region, country, or service area used for localized Google results. MCP Scraper records location evidence; UULE alone is not proof of localization."),
|
|
20334
|
+
gl: import_zod26.z.string().length(2).default("us").describe("Google country code inferred from the requested market, e.g. us, gb, ca, au."),
|
|
20335
|
+
hl: import_zod26.z.string().default("en").describe("Google interface/content language inferred from the user request."),
|
|
20336
|
+
device: import_zod26.z.enum(["desktop", "mobile"]).default("desktop").describe("SERP device context. Use mobile only when the user asks for mobile rankings or mobile SERP evidence."),
|
|
20337
|
+
proxyMode: import_zod26.z.enum(["location", "configured", "none"]).default("location").describe("Proxy behavior for capture. Use location for localized US residential evidence; it creates a fresh proxy ID per attempt. If Google shows a CAPTCHA/challenge, browser-service sessions briefly wait for the automatic solver first, then retry with a fresh proxy/session if the challenge does not clear. Also retries proxy tunnel failure and wrong-location evidence before returning. Use configured only for the static residential proxy, and none only for direct-network debugging."),
|
|
20338
|
+
proxyZip: import_zod26.z.string().regex(/^\d{5}$/).optional().describe("Optional US ZIP override for residential location proxy targeting when a precise city-center or ZIP proxy is needed. With proxyMode location this ZIP is used for each fresh proxy attempt."),
|
|
20339
|
+
pages: import_zod26.z.number().int().min(1).max(2).default(1).describe("Number of Google result pages to capture. Use 1 normally and 2 only when the user needs deeper ranking evidence."),
|
|
20340
|
+
debug: import_zod26.z.boolean().default(false).describe("Include sanitized browser, proxy, and location diagnostics. Use true when debugging localization, CAPTCHA, proxy selection, or capture reliability."),
|
|
20341
|
+
includePageSnapshots: import_zod26.z.boolean().default(false).describe("Also capture ranking-page snapshots for selected SERP URLs through the same product capture path."),
|
|
20342
|
+
pageSnapshotLimit: import_zod26.z.number().int().min(0).max(10).default(0).describe("Maximum ranking-page snapshots to capture when includePageSnapshots is true. Use 0 when only SERP evidence is needed.")
|
|
19268
20343
|
};
|
|
19269
20344
|
ScreenshotInputSchema = {
|
|
19270
|
-
url:
|
|
19271
|
-
device:
|
|
19272
|
-
allowLocal:
|
|
20345
|
+
url: import_zod26.z.string().url().describe("URL to capture as a full-page screenshot. Use http or https. Pass allowLocal: true to capture localhost or private-network URLs during development."),
|
|
20346
|
+
device: import_zod26.z.enum(["desktop", "mobile"]).default("desktop").describe("Viewport profile. desktop = 1440\xD7900. mobile = 390\xD7844. Use desktop by default; use mobile when the user asks for a mobile view."),
|
|
20347
|
+
allowLocal: import_zod26.z.boolean().default(false).describe("Allow localhost and private-network URLs (127.x, 192.168.x, 10.x, etc.). For local development only \u2014 not for production use.")
|
|
19273
20348
|
};
|
|
19274
20349
|
CaptureSerpPageSnapshotsInputSchema = {
|
|
19275
|
-
urls:
|
|
19276
|
-
targets:
|
|
19277
|
-
url:
|
|
19278
|
-
sourceKind:
|
|
19279
|
-
sourcePosition:
|
|
20350
|
+
urls: import_zod26.z.array(import_zod26.z.string().url()).min(1).max(25).describe("Public HTTP/HTTPS URLs to capture as SERP Intelligence page snapshots. Do not pass localhost, private IPs, file URLs, or internal admin URLs."),
|
|
20351
|
+
targets: import_zod26.z.array(import_zod26.z.object({
|
|
20352
|
+
url: import_zod26.z.string().url().describe("Public HTTP/HTTPS URL to capture."),
|
|
20353
|
+
sourceKind: import_zod26.z.enum(["organic", "ai_citation", "local_pack_website", "configured_target", "site_subject"]).default("configured_target").describe("Why this page is being captured for SERP Intelligence evidence."),
|
|
20354
|
+
sourcePosition: import_zod26.z.number().int().min(1).optional().describe("Ranking or citation position when the page came from SERP evidence.")
|
|
19280
20355
|
}).strict()).min(1).max(25).optional().describe("Structured page snapshot targets. Use this instead of urls when source kind or position should be preserved."),
|
|
19281
|
-
maxConcurrency:
|
|
19282
|
-
timeoutMs:
|
|
19283
|
-
debug:
|
|
20356
|
+
maxConcurrency: import_zod26.z.number().int().min(1).max(5).default(2).describe("Parallel page captures. Use 2 normally; higher values can increase proxy/browser pressure."),
|
|
20357
|
+
timeoutMs: import_zod26.z.number().int().min(1e3).max(6e4).default(15e3).describe("Per-page capture timeout in milliseconds. Increase for slow pages; timeout artifacts are returned as structured capture failures."),
|
|
20358
|
+
debug: import_zod26.z.boolean().default(false).describe("Include sanitized browser/proxy diagnostics for page snapshot debugging. Use true for capture, network, or proxy troubleshooting.")
|
|
19284
20359
|
};
|
|
19285
20360
|
}
|
|
19286
20361
|
});
|
|
19287
20362
|
|
|
19288
20363
|
// src/mcp/rank-tracker-blueprint.ts
|
|
19289
|
-
function
|
|
20364
|
+
function normalizeDomain3(value) {
|
|
19290
20365
|
const trimmed = value?.trim();
|
|
19291
20366
|
if (!trimmed) return null;
|
|
19292
20367
|
try {
|
|
@@ -19548,7 +20623,7 @@ function buildPrompt(input, modes, tools, tables, jobs, metrics, expression, tar
|
|
|
19548
20623
|
}
|
|
19549
20624
|
function buildRankTrackerBlueprint(input) {
|
|
19550
20625
|
const trackingModes = uniqueModes(input.trackingModes);
|
|
19551
|
-
const targetDomain =
|
|
20626
|
+
const targetDomain = normalizeDomain3(input.targetDomain);
|
|
19552
20627
|
const targetBusinessName = input.targetBusinessName?.trim() || null;
|
|
19553
20628
|
const projectName = input.projectName?.trim() || "MCP Scraper Rank Tracker";
|
|
19554
20629
|
const database = input.database || "postgres";
|
|
@@ -19775,7 +20850,7 @@ function registerPaaExtractorMcpTools(server, executor, options = {}) {
|
|
|
19775
20850
|
}, async (input) => buildRankTrackerBlueprint(input));
|
|
19776
20851
|
server.registerTool("credits_info", {
|
|
19777
20852
|
title: "MCP Scraper Credits & Costs",
|
|
19778
|
-
description: "Answer questions about MCP Scraper credits: current credit balance, what a specific tool/action costs, the full cost table, and
|
|
20853
|
+
description: "Answer questions about MCP Scraper credits, usage limits, and concurrency upgrades: current credit balance, what a specific tool/action costs, the full cost table, current concurrency limit, the extra-slot price, billing URL, and the terminal checkout command. Does not expose payment methods or credit card information.",
|
|
19779
20854
|
inputSchema: CreditsInfoInputSchema,
|
|
19780
20855
|
outputSchema: CreditsInfoOutputSchema,
|
|
19781
20856
|
annotations: {
|
|
@@ -20008,6 +21083,7 @@ async function migrateBrowserAgent() {
|
|
|
20008
21083
|
status TEXT NOT NULL DEFAULT 'open',
|
|
20009
21084
|
label TEXT,
|
|
20010
21085
|
user_id INTEGER,
|
|
21086
|
+
concurrency_lock_id TEXT,
|
|
20011
21087
|
created_at TEXT NOT NULL DEFAULT (datetime('now')),
|
|
20012
21088
|
closed_at TEXT,
|
|
20013
21089
|
last_action_at TEXT,
|
|
@@ -20017,6 +21093,11 @@ async function migrateBrowserAgent() {
|
|
|
20017
21093
|
`);
|
|
20018
21094
|
await db.execute(`CREATE INDEX IF NOT EXISTS browser_agent_sessions_status ON browser_agent_sessions(status)`);
|
|
20019
21095
|
await db.execute(`CREATE INDEX IF NOT EXISTS browser_agent_sessions_user ON browser_agent_sessions(user_id)`);
|
|
21096
|
+
try {
|
|
21097
|
+
await db.execute(`ALTER TABLE browser_agent_sessions ADD COLUMN concurrency_lock_id TEXT`);
|
|
21098
|
+
} catch {
|
|
21099
|
+
}
|
|
21100
|
+
await db.execute(`CREATE INDEX IF NOT EXISTS browser_agent_sessions_concurrency_lock ON browser_agent_sessions(concurrency_lock_id)`);
|
|
20020
21101
|
await db.execute(`
|
|
20021
21102
|
CREATE TABLE IF NOT EXISTS browser_agent_actions (
|
|
20022
21103
|
id TEXT PRIMARY KEY,
|
|
@@ -20044,11 +21125,11 @@ async function migrateBrowserAgent() {
|
|
|
20044
21125
|
}
|
|
20045
21126
|
async function createSessionRow(input) {
|
|
20046
21127
|
const db = getDb();
|
|
20047
|
-
const id = `bas_${(0,
|
|
21128
|
+
const id = `bas_${(0, import_node_crypto5.randomUUID)().replace(/-/g, "").slice(0, 20)}`;
|
|
20048
21129
|
await db.execute({
|
|
20049
|
-
sql: `INSERT INTO browser_agent_sessions (id, runtime_session_id, live_view_url, cdp_ws_url, status, label, user_id, last_action_at)
|
|
20050
|
-
VALUES (?, ?, ?, ?, 'open', ?, ?, datetime('now'))`,
|
|
20051
|
-
args: [id, input.runtimeSessionId, input.liveViewUrl, input.cdpWsUrl, input.label, input.userId]
|
|
21130
|
+
sql: `INSERT INTO browser_agent_sessions (id, runtime_session_id, live_view_url, cdp_ws_url, status, label, user_id, concurrency_lock_id, last_action_at)
|
|
21131
|
+
VALUES (?, ?, ?, ?, 'open', ?, ?, ?, datetime('now'))`,
|
|
21132
|
+
args: [id, input.runtimeSessionId, input.liveViewUrl, input.cdpWsUrl, input.label, input.userId, input.concurrencyLockId ?? null]
|
|
20052
21133
|
});
|
|
20053
21134
|
const row = await getSessionRow(id);
|
|
20054
21135
|
if (!row) throw new Error("session insert failed");
|
|
@@ -20105,7 +21186,7 @@ async function recordAction(input) {
|
|
|
20105
21186
|
sql: `INSERT INTO browser_agent_actions (id, session_id, type, params_json, ok, error)
|
|
20106
21187
|
VALUES (?, ?, ?, ?, ?, ?)`,
|
|
20107
21188
|
args: [
|
|
20108
|
-
`baa_${(0,
|
|
21189
|
+
`baa_${(0, import_node_crypto5.randomUUID)().replace(/-/g, "").slice(0, 20)}`,
|
|
20109
21190
|
input.sessionId,
|
|
20110
21191
|
input.type,
|
|
20111
21192
|
input.params == null ? null : JSON.stringify(input.params),
|
|
@@ -20145,11 +21226,11 @@ async function listReplayRows(sessionId) {
|
|
|
20145
21226
|
});
|
|
20146
21227
|
return res.rows;
|
|
20147
21228
|
}
|
|
20148
|
-
var
|
|
21229
|
+
var import_node_crypto5, _ready;
|
|
20149
21230
|
var init_browser_agent_db = __esm({
|
|
20150
21231
|
"src/api/browser-agent-db.ts"() {
|
|
20151
21232
|
"use strict";
|
|
20152
|
-
|
|
21233
|
+
import_node_crypto5 = require("crypto");
|
|
20153
21234
|
init_db();
|
|
20154
21235
|
_ready = false;
|
|
20155
21236
|
}
|
|
@@ -20469,6 +21550,12 @@ var init_browser_agent_service = __esm({
|
|
|
20469
21550
|
});
|
|
20470
21551
|
|
|
20471
21552
|
// src/api/browser-agent-routes.ts
|
|
21553
|
+
function browserSessionLockTtlSeconds(timeoutSeconds) {
|
|
21554
|
+
if (typeof timeoutSeconds === "number" && Number.isFinite(timeoutSeconds) && timeoutSeconds > 0) {
|
|
21555
|
+
return Math.max(30 * 60, Math.min(Math.trunc(timeoutSeconds) + 5 * 60, 24 * 60 * 60));
|
|
21556
|
+
}
|
|
21557
|
+
return DEFAULT_BROWSER_SESSION_LOCK_TTL_SECONDS;
|
|
21558
|
+
}
|
|
20472
21559
|
async function charge(sessionId, userId, startedAtMs) {
|
|
20473
21560
|
const elapsedMs = Date.now() - startedAtMs;
|
|
20474
21561
|
const { active_ms, billed_mc } = await addActiveMs(sessionId, elapsedMs);
|
|
@@ -20524,6 +21611,8 @@ async function loadOpenSession(id, userId) {
|
|
|
20524
21611
|
const row = await getSessionRow(id);
|
|
20525
21612
|
if (!row) return null;
|
|
20526
21613
|
if (row.user_id != null && row.user_id !== userId) return null;
|
|
21614
|
+
if (row.status !== "open") return null;
|
|
21615
|
+
await extendConcurrencyGate(row.concurrency_lock_id, DEFAULT_BROWSER_SESSION_LOCK_TTL_SECONDS);
|
|
20527
21616
|
return row;
|
|
20528
21617
|
}
|
|
20529
21618
|
function buildBrowserAgentRoutes() {
|
|
@@ -20539,23 +21628,39 @@ function buildBrowserAgentRoutes() {
|
|
|
20539
21628
|
return c.json(insufficientBalanceResponse(Number(user.balance_mc ?? 0), BROWSER_OPEN_MIN_BALANCE_MC), 402);
|
|
20540
21629
|
}
|
|
20541
21630
|
const body = await c.req.json().catch(() => ({}));
|
|
21631
|
+
const timeoutSeconds = typeof body.timeout_seconds === "number" ? body.timeout_seconds : void 0;
|
|
21632
|
+
const gate = await acquireConcurrencyGate(user, "browser_agent_session", {
|
|
21633
|
+
ttlSeconds: browserSessionLockTtlSeconds(timeoutSeconds),
|
|
21634
|
+
metadata: { label: typeof body.label === "string" ? body.label : null }
|
|
21635
|
+
});
|
|
21636
|
+
if (!gate.ok) return c.json(concurrencyLimitExceededResponse(gate), 429, { "Retry-After": String(gate.retryAfterSeconds) });
|
|
21637
|
+
let runtimeSessionId = null;
|
|
20542
21638
|
try {
|
|
20543
21639
|
const created = await createSession({
|
|
20544
|
-
timeoutSeconds
|
|
21640
|
+
timeoutSeconds,
|
|
20545
21641
|
proxyId: typeof body.proxy_id === "string" ? body.proxy_id : void 0,
|
|
20546
21642
|
profileName: typeof body.profile === "string" ? body.profile : void 0,
|
|
20547
21643
|
disableDefaultProxy: body.disable_default_proxy === true,
|
|
20548
21644
|
viewport: body.viewport && typeof body.viewport === "object" ? body.viewport : void 0
|
|
20549
21645
|
});
|
|
21646
|
+
runtimeSessionId = created.runtimeSessionId;
|
|
20550
21647
|
const row = await createSessionRow({
|
|
20551
21648
|
runtimeSessionId: created.runtimeSessionId,
|
|
20552
21649
|
liveViewUrl: created.liveViewUrl,
|
|
20553
21650
|
cdpWsUrl: created.cdpWsUrl,
|
|
20554
21651
|
label: typeof body.label === "string" ? body.label : null,
|
|
20555
|
-
userId: user.id
|
|
21652
|
+
userId: user.id,
|
|
21653
|
+
concurrencyLockId: gate.lockId
|
|
20556
21654
|
});
|
|
20557
21655
|
return c.json({ ...publicSession(row), watch_url: `/console/${row.id}` });
|
|
20558
21656
|
} catch (err) {
|
|
21657
|
+
if (runtimeSessionId) {
|
|
21658
|
+
try {
|
|
21659
|
+
await closeSession(runtimeSessionId);
|
|
21660
|
+
} catch {
|
|
21661
|
+
}
|
|
21662
|
+
}
|
|
21663
|
+
await releaseConcurrencyGate(gate.lockId);
|
|
20559
21664
|
return c.json(failure(err), 502);
|
|
20560
21665
|
}
|
|
20561
21666
|
});
|
|
@@ -20586,6 +21691,7 @@ function buildBrowserAgentRoutes() {
|
|
|
20586
21691
|
} catch {
|
|
20587
21692
|
}
|
|
20588
21693
|
await markSessionClosed(row.id);
|
|
21694
|
+
await releaseConcurrencyGate(row.concurrency_lock_id);
|
|
20589
21695
|
return c.json({ ok: true });
|
|
20590
21696
|
});
|
|
20591
21697
|
app2.post("/sessions/:id/goto", async (c) => {
|
|
@@ -20843,7 +21949,7 @@ function buildBrowserAgentRoutes() {
|
|
|
20843
21949
|
});
|
|
20844
21950
|
return app2;
|
|
20845
21951
|
}
|
|
20846
|
-
var import_hono10, auth;
|
|
21952
|
+
var import_hono10, auth, DEFAULT_BROWSER_SESSION_LOCK_TTL_SECONDS;
|
|
20847
21953
|
var init_browser_agent_routes = __esm({
|
|
20848
21954
|
"src/api/browser-agent-routes.ts"() {
|
|
20849
21955
|
"use strict";
|
|
@@ -20854,7 +21960,9 @@ var init_browser_agent_routes = __esm({
|
|
|
20854
21960
|
init_rates();
|
|
20855
21961
|
init_browser_agent_db();
|
|
20856
21962
|
init_browser_agent_service();
|
|
21963
|
+
init_concurrency_gates();
|
|
20857
21964
|
auth = createApiKeyAuth();
|
|
21965
|
+
DEFAULT_BROWSER_SESSION_LOCK_TTL_SECONDS = 2 * 60 * 60;
|
|
20858
21966
|
}
|
|
20859
21967
|
});
|
|
20860
21968
|
|
|
@@ -21170,31 +22278,31 @@ var init_site_audit_worker = __esm({
|
|
|
21170
22278
|
});
|
|
21171
22279
|
|
|
21172
22280
|
// src/api/billing-schemas.ts
|
|
21173
|
-
var
|
|
22281
|
+
var import_zod27, BillingCheckoutBodySchema, FreeCreditBreakdownSchema, BillingBalanceResponseSchema, MonthlyRefreshSweepResultSchema;
|
|
21174
22282
|
var init_billing_schemas = __esm({
|
|
21175
22283
|
"src/api/billing-schemas.ts"() {
|
|
21176
22284
|
"use strict";
|
|
21177
|
-
|
|
21178
|
-
BillingCheckoutBodySchema =
|
|
21179
|
-
priceId:
|
|
22285
|
+
import_zod27 = require("zod");
|
|
22286
|
+
BillingCheckoutBodySchema = import_zod27.z.object({
|
|
22287
|
+
priceId: import_zod27.z.string().min(1)
|
|
21180
22288
|
});
|
|
21181
|
-
FreeCreditBreakdownSchema =
|
|
21182
|
-
signup_grant_mc:
|
|
21183
|
-
monthly_refresh_mc:
|
|
21184
|
-
total_free_mc:
|
|
21185
|
-
signup_grant_credits:
|
|
21186
|
-
monthly_refresh_credits:
|
|
21187
|
-
total_free_credits:
|
|
22289
|
+
FreeCreditBreakdownSchema = import_zod27.z.object({
|
|
22290
|
+
signup_grant_mc: import_zod27.z.number().int().nonnegative(),
|
|
22291
|
+
monthly_refresh_mc: import_zod27.z.number().int().nonnegative(),
|
|
22292
|
+
total_free_mc: import_zod27.z.number().int().nonnegative(),
|
|
22293
|
+
signup_grant_credits: import_zod27.z.number().nonnegative(),
|
|
22294
|
+
monthly_refresh_credits: import_zod27.z.number().nonnegative(),
|
|
22295
|
+
total_free_credits: import_zod27.z.number().nonnegative()
|
|
21188
22296
|
});
|
|
21189
|
-
BillingBalanceResponseSchema =
|
|
21190
|
-
balance_mc:
|
|
21191
|
-
balance_credits:
|
|
22297
|
+
BillingBalanceResponseSchema = import_zod27.z.object({
|
|
22298
|
+
balance_mc: import_zod27.z.number().int().nonnegative(),
|
|
22299
|
+
balance_credits: import_zod27.z.number().nonnegative(),
|
|
21192
22300
|
free_credits: FreeCreditBreakdownSchema,
|
|
21193
|
-
ledger:
|
|
22301
|
+
ledger: import_zod27.z.array(import_zod27.z.any())
|
|
21194
22302
|
});
|
|
21195
|
-
MonthlyRefreshSweepResultSchema =
|
|
21196
|
-
usersRefreshed:
|
|
21197
|
-
totalMcGranted:
|
|
22303
|
+
MonthlyRefreshSweepResultSchema = import_zod27.z.object({
|
|
22304
|
+
usersRefreshed: import_zod27.z.number().int().nonnegative(),
|
|
22305
|
+
totalMcGranted: import_zod27.z.number().int().nonnegative()
|
|
21198
22306
|
});
|
|
21199
22307
|
}
|
|
21200
22308
|
});
|
|
@@ -21356,14 +22464,14 @@ function getSessionSecret() {
|
|
|
21356
22464
|
function safeEqualHex(a, b) {
|
|
21357
22465
|
if (a.length !== b.length) return false;
|
|
21358
22466
|
try {
|
|
21359
|
-
return (0,
|
|
22467
|
+
return (0, import_node_crypto6.timingSafeEqual)(Buffer.from(a, "hex"), Buffer.from(b, "hex"));
|
|
21360
22468
|
} catch {
|
|
21361
22469
|
return false;
|
|
21362
22470
|
}
|
|
21363
22471
|
}
|
|
21364
22472
|
function signSession(userId) {
|
|
21365
22473
|
const payload = String(userId);
|
|
21366
|
-
const sig = (0,
|
|
22474
|
+
const sig = (0, import_node_crypto6.createHmac)("sha256", secret()).update(payload).digest("hex");
|
|
21367
22475
|
return `${payload}.${sig}`;
|
|
21368
22476
|
}
|
|
21369
22477
|
function verifySession(token) {
|
|
@@ -21371,16 +22479,16 @@ function verifySession(token) {
|
|
|
21371
22479
|
if (dot === -1) return null;
|
|
21372
22480
|
const payload = token.slice(0, dot);
|
|
21373
22481
|
const sig = token.slice(dot + 1);
|
|
21374
|
-
const expected = (0,
|
|
22482
|
+
const expected = (0, import_node_crypto6.createHmac)("sha256", secret()).update(payload).digest("hex");
|
|
21375
22483
|
if (!safeEqualHex(sig, expected)) return null;
|
|
21376
22484
|
const id = parseInt(payload);
|
|
21377
22485
|
return isNaN(id) ? null : id;
|
|
21378
22486
|
}
|
|
21379
|
-
var
|
|
22487
|
+
var import_node_crypto6, isProduction, secret;
|
|
21380
22488
|
var init_session = __esm({
|
|
21381
22489
|
"src/api/session.ts"() {
|
|
21382
22490
|
"use strict";
|
|
21383
|
-
|
|
22491
|
+
import_node_crypto6 = require("crypto");
|
|
21384
22492
|
isProduction = () => process.env.NODE_ENV === "production" || process.env.VERCEL === "1";
|
|
21385
22493
|
secret = () => getSessionSecret();
|
|
21386
22494
|
}
|
|
@@ -21589,14 +22697,23 @@ function countPaaQuestions2(result) {
|
|
|
21589
22697
|
function paaCostForQuestionCount2(questionCount) {
|
|
21590
22698
|
return Math.max(1, questionCount) * MC_COSTS.paa;
|
|
21591
22699
|
}
|
|
21592
|
-
async function checkHarvestLimits(
|
|
21593
|
-
if (
|
|
21594
|
-
|
|
21595
|
-
const
|
|
21596
|
-
|
|
22700
|
+
async function checkHarvestLimits(user, reuseLockId) {
|
|
22701
|
+
if (reuseLockId && await reuseExistingConcurrencyGate(user.id, reuseLockId, 30 * 60)) return null;
|
|
22702
|
+
if (isConcurrencyLimitBypassed(user)) return null;
|
|
22703
|
+
const limit = concurrencyLimitForUser(user);
|
|
22704
|
+
const active = await countActiveUsageForUser(user.id);
|
|
22705
|
+
if (active >= limit) {
|
|
22706
|
+
return concurrencyLimitExceededResponse({
|
|
22707
|
+
ok: false,
|
|
22708
|
+
active,
|
|
22709
|
+
limit,
|
|
22710
|
+
operation: "harvest",
|
|
22711
|
+
retryAfterSeconds: 30
|
|
22712
|
+
});
|
|
22713
|
+
}
|
|
21597
22714
|
return null;
|
|
21598
22715
|
}
|
|
21599
|
-
var import_resend, import_hono12, import_hono13, import_factory6, import_cookie, import_stripe2, secureCookies, isProduction2, sessionCookieOptions, requireAllowedOrigin, auth2, adminAuth, sessionAuth, app, STRIPE_API_VERSION,
|
|
22716
|
+
var import_resend, import_hono12, import_hono13, import_factory6, import_cookie, import_stripe2, secureCookies, isProduction2, sessionCookieOptions, requireAllowedOrigin, auth2, adminAuth, sessionAuth, app, STRIPE_API_VERSION, SYNC_HARVEST_TIMEOUT_OVERRIDE_MS;
|
|
21600
22717
|
var init_server = __esm({
|
|
21601
22718
|
"src/api/server.ts"() {
|
|
21602
22719
|
"use strict";
|
|
@@ -21642,6 +22759,7 @@ var init_server = __esm({
|
|
|
21642
22759
|
init_harvest_problems();
|
|
21643
22760
|
init_harvest_attempt_events();
|
|
21644
22761
|
init_session();
|
|
22762
|
+
init_concurrency_gates();
|
|
21645
22763
|
secureCookies = process.env.NODE_ENV === "production" || process.env.VERCEL === "1";
|
|
21646
22764
|
isProduction2 = secureCookies;
|
|
21647
22765
|
sessionCookieOptions = {
|
|
@@ -21838,9 +22956,6 @@ var init_server = __esm({
|
|
|
21838
22956
|
await revokeApiKey(user.id);
|
|
21839
22957
|
return c.json({ ok: true });
|
|
21840
22958
|
});
|
|
21841
|
-
BYPASS_EMAILS = new Set(
|
|
21842
|
-
(process.env.HARVEST_LIMIT_BYPASS_EMAILS ?? "").split(",").map((e) => e.trim()).filter(Boolean)
|
|
21843
|
-
);
|
|
21844
22959
|
SYNC_HARVEST_TIMEOUT_OVERRIDE_MS = (() => {
|
|
21845
22960
|
const raw = process.env.SYNC_HARVEST_TIMEOUT_MS;
|
|
21846
22961
|
const parsed = raw === void 0 ? NaN : Number(raw);
|
|
@@ -21858,8 +22973,8 @@ var init_server = __esm({
|
|
|
21858
22973
|
if (checked.error) return c.json({ error: checked.error }, 400);
|
|
21859
22974
|
body.callback_url = checked.parsed?.href;
|
|
21860
22975
|
}
|
|
21861
|
-
const limitErr = await checkHarvestLimits(user
|
|
21862
|
-
if (limitErr) return c.json(limitErr, 429);
|
|
22976
|
+
const limitErr = await checkHarvestLimits(user, c.req.header("x-mcp-scraper-concurrency-lock"));
|
|
22977
|
+
if (limitErr) return c.json(limitErr, 429, { "Retry-After": "30" });
|
|
21863
22978
|
const options = {
|
|
21864
22979
|
query: body.query.trim(),
|
|
21865
22980
|
location: body.location,
|
|
@@ -21894,8 +23009,8 @@ var init_server = __esm({
|
|
|
21894
23009
|
const bodyResult = HarvestBodySchema.safeParse(raw);
|
|
21895
23010
|
if (!bodyResult.success) return c.json({ error: bodyResult.error.issues[0]?.message ?? "Invalid request" }, 400);
|
|
21896
23011
|
const body = bodyResult.data;
|
|
21897
|
-
const limitErr = await checkHarvestLimits(user
|
|
21898
|
-
if (limitErr) return c.json(limitErr, 429);
|
|
23012
|
+
const limitErr = await checkHarvestLimits(user, c.req.header("x-mcp-scraper-concurrency-lock"));
|
|
23013
|
+
if (limitErr) return c.json(limitErr, 429, { "Retry-After": "30" });
|
|
21899
23014
|
const options = {
|
|
21900
23015
|
query: body.query.trim(),
|
|
21901
23016
|
location: body.location,
|
|
@@ -22054,9 +23169,16 @@ var init_server = __esm({
|
|
|
22054
23169
|
}
|
|
22055
23170
|
})();
|
|
22056
23171
|
const user = c.get("user");
|
|
22057
|
-
const
|
|
22058
|
-
|
|
23172
|
+
const gate = await acquireConcurrencyGate(user, "extract_url", {
|
|
23173
|
+
reuseLockId: c.req.header("x-mcp-scraper-concurrency-lock"),
|
|
23174
|
+
metadata: { url: canonicalUrl }
|
|
23175
|
+
});
|
|
23176
|
+
if (!gate.ok) return c.json(concurrencyLimitExceededResponse(gate), 429, { "Retry-After": String(gate.retryAfterSeconds) });
|
|
23177
|
+
let debited = false;
|
|
22059
23178
|
try {
|
|
23179
|
+
const { ok: euOk, balance_mc: euBal } = await debitMc(user.id, MC_COSTS.page_scrape, LedgerOperation.EXTRACT_URL, new URL(canonicalUrl).hostname);
|
|
23180
|
+
if (!euOk) return c.json(insufficientBalanceResponse(euBal, MC_COSTS.page_scrape), 402);
|
|
23181
|
+
debited = true;
|
|
22060
23182
|
const kernelApiKey = browserServiceApiKey();
|
|
22061
23183
|
const device = screenshotDevice === "mobile" ? "mobile" : "desktop";
|
|
22062
23184
|
const [result, pageData] = await Promise.all([
|
|
@@ -22074,9 +23196,11 @@ var init_server = __esm({
|
|
|
22074
23196
|
return c.json({ ...result, screenshot: screenshotMeta, branding: brandingData, media: mediaMeta });
|
|
22075
23197
|
} catch (err) {
|
|
22076
23198
|
const msg = err instanceof Error ? err.message : String(err);
|
|
22077
|
-
await creditMc(user.id, MC_COSTS.page_scrape, LedgerOperation.EXTRACT_URL, "failed call");
|
|
23199
|
+
if (debited) await creditMc(user.id, MC_COSTS.page_scrape, LedgerOperation.EXTRACT_URL, "failed call");
|
|
22078
23200
|
await logRequestEvent({ userId: user.id, source: "extract_url", status: "failed", query: canonicalUrl, error: msg });
|
|
22079
23201
|
return c.json({ error: msg }, 500);
|
|
23202
|
+
} finally {
|
|
23203
|
+
await releaseConcurrencyGate(gate.lockId);
|
|
22080
23204
|
}
|
|
22081
23205
|
});
|
|
22082
23206
|
app.post("/map-urls", auth2, async (c) => {
|
|
@@ -22088,9 +23212,16 @@ var init_server = __esm({
|
|
|
22088
23212
|
if (checked.error || !checked.parsed) return c.json({ error: checked.error ?? "Invalid URL" }, 400);
|
|
22089
23213
|
const parsed = checked.parsed;
|
|
22090
23214
|
const user = c.get("user");
|
|
22091
|
-
const
|
|
22092
|
-
|
|
23215
|
+
const gate = await acquireConcurrencyGate(user, "map_urls", {
|
|
23216
|
+
reuseLockId: c.req.header("x-mcp-scraper-concurrency-lock"),
|
|
23217
|
+
metadata: { url: parsed.href }
|
|
23218
|
+
});
|
|
23219
|
+
if (!gate.ok) return c.json(concurrencyLimitExceededResponse(gate), 429, { "Retry-After": String(gate.retryAfterSeconds) });
|
|
23220
|
+
let debited = false;
|
|
22093
23221
|
try {
|
|
23222
|
+
const { ok: mapOk, balance_mc: mapBal } = await debitMc(user.id, MC_COSTS.url_map, LedgerOperation.URL_MAP, parsed.hostname);
|
|
23223
|
+
if (!mapOk) return c.json(insufficientBalanceResponse(mapBal, MC_COSTS.url_map), 402);
|
|
23224
|
+
debited = true;
|
|
22094
23225
|
const result = await spiderSite({
|
|
22095
23226
|
startUrl: parsed.href,
|
|
22096
23227
|
maxUrls: Math.min(2e3, Math.max(1, body.maxUrls ?? 500)),
|
|
@@ -22108,7 +23239,7 @@ var init_server = __esm({
|
|
|
22108
23239
|
return c.json(result);
|
|
22109
23240
|
} catch (err) {
|
|
22110
23241
|
const msg = err instanceof Error ? err.message : String(err);
|
|
22111
|
-
await creditMc(user.id, MC_COSTS.url_map, LedgerOperation.URL_MAP_REFUND, "failed call");
|
|
23242
|
+
if (debited) await creditMc(user.id, MC_COSTS.url_map, LedgerOperation.URL_MAP_REFUND, "failed call");
|
|
22112
23243
|
await logRequestEvent({
|
|
22113
23244
|
userId: user.id,
|
|
22114
23245
|
source: "map_urls",
|
|
@@ -22117,6 +23248,8 @@ var init_server = __esm({
|
|
|
22117
23248
|
error: msg
|
|
22118
23249
|
});
|
|
22119
23250
|
return c.json({ error: msg }, 500);
|
|
23251
|
+
} finally {
|
|
23252
|
+
await releaseConcurrencyGate(gate.lockId);
|
|
22120
23253
|
}
|
|
22121
23254
|
});
|
|
22122
23255
|
app.post("/extract-site", auth2, async (c) => {
|
|
@@ -22129,9 +23262,16 @@ var init_server = __esm({
|
|
|
22129
23262
|
const parsed = checked.parsed;
|
|
22130
23263
|
const user = c.get("user");
|
|
22131
23264
|
const siteHoldMc = MC_COSTS.page_scrape * 10;
|
|
22132
|
-
const
|
|
22133
|
-
|
|
23265
|
+
const gate = await acquireConcurrencyGate(user, "extract_site", {
|
|
23266
|
+
reuseLockId: c.req.header("x-mcp-scraper-concurrency-lock"),
|
|
23267
|
+
metadata: { url: parsed.href }
|
|
23268
|
+
});
|
|
23269
|
+
if (!gate.ok) return c.json(concurrencyLimitExceededResponse(gate), 429, { "Retry-After": String(gate.retryAfterSeconds) });
|
|
23270
|
+
let debited = false;
|
|
22134
23271
|
try {
|
|
23272
|
+
const { ok: siteOk, balance_mc: siteBal } = await debitMc(user.id, siteHoldMc, LedgerOperation.EXTRACT_SITE_HOLD, parsed.hostname);
|
|
23273
|
+
if (!siteOk) return c.json(insufficientBalanceResponse(siteBal, siteHoldMc), 402);
|
|
23274
|
+
debited = true;
|
|
22135
23275
|
const result = await extractSite({
|
|
22136
23276
|
startUrl: parsed.href,
|
|
22137
23277
|
maxPages: Math.min(200, Math.max(1, body.maxPages ?? 100)),
|
|
@@ -22152,7 +23292,7 @@ var init_server = __esm({
|
|
|
22152
23292
|
});
|
|
22153
23293
|
return c.json(result);
|
|
22154
23294
|
} catch (err) {
|
|
22155
|
-
await creditMc(user.id, siteHoldMc, LedgerOperation.EXTRACT_SITE_REFUND, "failed call");
|
|
23295
|
+
if (debited) await creditMc(user.id, siteHoldMc, LedgerOperation.EXTRACT_SITE_REFUND, "failed call");
|
|
22156
23296
|
const msg = err instanceof Error ? err.message : String(err);
|
|
22157
23297
|
await logRequestEvent({
|
|
22158
23298
|
userId: user.id,
|
|
@@ -22162,6 +23302,8 @@ var init_server = __esm({
|
|
|
22162
23302
|
error: msg
|
|
22163
23303
|
});
|
|
22164
23304
|
return c.json({ error: msg }, 500);
|
|
23305
|
+
} finally {
|
|
23306
|
+
await releaseConcurrencyGate(gate.lockId);
|
|
22165
23307
|
}
|
|
22166
23308
|
});
|
|
22167
23309
|
app.post("/billing/checkout", requireAllowedOrigin, sessionAuth, async (c) => {
|
|
@@ -22230,6 +23372,60 @@ var init_server = __esm({
|
|
|
22230
23372
|
return c.json({ error: message }, 500);
|
|
22231
23373
|
}
|
|
22232
23374
|
});
|
|
23375
|
+
app.post("/billing/concurrency/terminal-checkout", auth2, async (c) => {
|
|
23376
|
+
try {
|
|
23377
|
+
const user = c.get("user");
|
|
23378
|
+
const upgrade = concurrencySlotBillingInfo();
|
|
23379
|
+
if (user.concurrency_stripe_sub_id) {
|
|
23380
|
+
return c.json({
|
|
23381
|
+
error: "Already subscribed. Your account already has an active extra concurrency slot subscription.",
|
|
23382
|
+
error_code: "concurrency_subscription_exists",
|
|
23383
|
+
concurrency: {
|
|
23384
|
+
extra_slots: user.extra_concurrency_slots,
|
|
23385
|
+
limit: concurrencyLimitForUser(user),
|
|
23386
|
+
has_subscription: true,
|
|
23387
|
+
upgrade
|
|
23388
|
+
}
|
|
23389
|
+
}, 409);
|
|
23390
|
+
}
|
|
23391
|
+
const secret2 = process.env.STRIPE_SECRET_KEY?.trim();
|
|
23392
|
+
if (!secret2) return c.json({ error: "Stripe is not configured." }, 503);
|
|
23393
|
+
const stripeClient = new import_stripe2.default(secret2, { apiVersion: STRIPE_API_VERSION });
|
|
23394
|
+
let customerId = user.stripe_customer_id;
|
|
23395
|
+
if (!customerId) {
|
|
23396
|
+
const customer = await stripeClient.customers.create({ email: user.email });
|
|
23397
|
+
customerId = customer.id;
|
|
23398
|
+
await setStripeCustomerId(user.id, customerId);
|
|
23399
|
+
}
|
|
23400
|
+
const session = await stripeClient.checkout.sessions.create({
|
|
23401
|
+
customer: customerId,
|
|
23402
|
+
mode: "subscription",
|
|
23403
|
+
line_items: [{ price: CONCURRENCY_PRICE_ID, quantity: 1 }],
|
|
23404
|
+
automatic_tax: { enabled: true },
|
|
23405
|
+
billing_address_collection: "required",
|
|
23406
|
+
customer_update: { address: "auto", name: "auto" },
|
|
23407
|
+
tax_id_collection: { enabled: true },
|
|
23408
|
+
success_url: `${appOrigin()}/billing?slot_added=1&source=terminal`,
|
|
23409
|
+
cancel_url: `${appOrigin()}/billing?slot_cancelled=1&source=terminal`
|
|
23410
|
+
});
|
|
23411
|
+
if (!session.url) return c.json({ error: "Stripe did not return a checkout URL." }, 502);
|
|
23412
|
+
return c.json({
|
|
23413
|
+
checkout_url: session.url,
|
|
23414
|
+
price: upgrade,
|
|
23415
|
+
concurrency: {
|
|
23416
|
+
current_extra_slots: user.extra_concurrency_slots,
|
|
23417
|
+
current_limit: concurrencyLimitForUser(user),
|
|
23418
|
+
after_checkout_extra_slots: user.extra_concurrency_slots + 1,
|
|
23419
|
+
after_checkout_limit: concurrencyLimitForUser({ extra_concurrency_slots: user.extra_concurrency_slots + 1 })
|
|
23420
|
+
},
|
|
23421
|
+
next_step: "Open checkout_url in a browser, complete checkout, then restart or retry the MCP request."
|
|
23422
|
+
});
|
|
23423
|
+
} catch (err) {
|
|
23424
|
+
const message = err instanceof Error ? err.message : "Unable to start terminal checkout.";
|
|
23425
|
+
console.error("[billing/concurrency/terminal-checkout]", message);
|
|
23426
|
+
return c.json({ error: message }, 500);
|
|
23427
|
+
}
|
|
23428
|
+
});
|
|
22233
23429
|
app.post("/billing/concurrency/cancel", requireAllowedOrigin, sessionAuth, async (c) => {
|
|
22234
23430
|
const user = c.get("sessionUser");
|
|
22235
23431
|
if (!user.concurrency_stripe_sub_id) return c.json({ error: "No active concurrency subscription." }, 404);
|
|
@@ -22237,9 +23433,10 @@ var init_server = __esm({
|
|
|
22237
23433
|
if (!stripeSecret) return c.json({ error: "Stripe is not configured." }, 503);
|
|
22238
23434
|
const stripeClient = new import_stripe2.default(stripeSecret, { apiVersion: STRIPE_API_VERSION });
|
|
22239
23435
|
await stripeClient.subscriptions.cancel(user.concurrency_stripe_sub_id);
|
|
22240
|
-
|
|
23436
|
+
const nextSlots = Math.max(0, user.extra_concurrency_slots - 1);
|
|
23437
|
+
await setExtraConcurrencySlots(user.id, nextSlots);
|
|
22241
23438
|
await setConcurrencySubId(user.id, null);
|
|
22242
|
-
return c.json({ ok: true, concurrency_limit:
|
|
23439
|
+
return c.json({ ok: true, extra_concurrency_slots: nextSlots, concurrency_limit: 1 + nextSlots });
|
|
22243
23440
|
});
|
|
22244
23441
|
app.get("/billing/balance", auth2, async (c) => {
|
|
22245
23442
|
const user = c.get("user");
|
|
@@ -22274,6 +23471,12 @@ var init_server = __esm({
|
|
|
22274
23471
|
item: body.item ?? null,
|
|
22275
23472
|
matched_cost: matchedCost ? (({ aliases, ...cost }) => cost)(matchedCost) : null,
|
|
22276
23473
|
costs,
|
|
23474
|
+
concurrency: {
|
|
23475
|
+
current_extra_slots: user.extra_concurrency_slots,
|
|
23476
|
+
current_limit: concurrencyLimitForUser(user),
|
|
23477
|
+
has_subscription: !!user.concurrency_stripe_sub_id,
|
|
23478
|
+
upgrade: concurrencySlotBillingInfo()
|
|
23479
|
+
},
|
|
22277
23480
|
ledger
|
|
22278
23481
|
});
|
|
22279
23482
|
});
|