mcp-scraper 0.21.3 → 0.21.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/dist/bin/api-server.cjs +728 -181
- package/dist/bin/api-server.cjs.map +1 -1
- package/dist/bin/api-server.js +3 -3
- package/dist/bin/mcp-scraper-cli.cjs +1 -1
- package/dist/bin/mcp-scraper-cli.cjs.map +1 -1
- package/dist/bin/mcp-scraper-cli.js +1 -1
- package/dist/bin/mcp-scraper-install.cjs +1 -1
- package/dist/bin/mcp-scraper-install.cjs.map +1 -1
- package/dist/bin/mcp-scraper-install.js +1 -1
- package/dist/bin/mcp-stdio-server.cjs +97 -32
- package/dist/bin/mcp-stdio-server.cjs.map +1 -1
- package/dist/bin/mcp-stdio-server.js +5 -5
- package/dist/bin/paa-harvest.cjs +21 -0
- package/dist/bin/paa-harvest.cjs.map +1 -1
- package/dist/bin/paa-harvest.js +3 -3
- package/dist/{chunk-JJ5RMEGX.js → chunk-E5J4HJBO.js} +2 -2
- package/dist/{chunk-H7APH5UG.js → chunk-ICT7DDHL.js} +366 -50
- package/dist/chunk-ICT7DDHL.js.map +1 -0
- package/dist/{chunk-OBNKOYOE.js → chunk-JHI373VB.js} +7 -5
- package/dist/chunk-JHI373VB.js.map +1 -0
- package/dist/{chunk-5IU5ZFVO.js → chunk-MTSBI7ZH.js} +23 -6
- package/dist/chunk-MTSBI7ZH.js.map +1 -0
- package/dist/{chunk-7LO6KLKD.js → chunk-N4JHXPW6.js} +89 -27
- package/dist/chunk-N4JHXPW6.js.map +1 -0
- package/dist/{chunk-XDRHBGJR.js → chunk-PNBYYAEK.js} +3 -3
- package/dist/{chunk-2P5VFIJM.js → chunk-SDHPQTFM.js} +2 -2
- package/dist/chunk-WQI7JIIN.js +7 -0
- package/dist/chunk-WQI7JIIN.js.map +1 -0
- package/dist/{db-Z35S224E.js → db-WRILVOFY.js} +4 -2
- package/dist/{extract-bundle-4RO3YU4Q.js → extract-bundle-NFFVMUMT.js} +2 -2
- package/dist/index.cjs +21 -0
- package/dist/index.cjs.map +1 -1
- package/dist/index.js +3 -3
- package/dist/{server-PBBRSLBP.js → server-OLOOFN5W.js} +246 -100
- package/dist/server-OLOOFN5W.js.map +1 -0
- package/dist/{site-extract-repository-FDH6ZXKT.js → site-extract-repository-2QQSL2QC.js} +4 -4
- package/dist/{worker-BXJQ7J4N.js → worker-RV2TYXE3.js} +9 -8
- package/dist/worker-RV2TYXE3.js.map +1 -0
- package/docs/mcp-tool-manifest.generated.json +90 -11
- package/docs/specs/oauth-mcp-spec.md +2 -2
- package/package.json +1 -1
- package/dist/chunk-5IU5ZFVO.js.map +0 -1
- package/dist/chunk-7LO6KLKD.js.map +0 -1
- package/dist/chunk-H7APH5UG.js.map +0 -1
- package/dist/chunk-OBNKOYOE.js.map +0 -1
- package/dist/chunk-VYUWWIVR.js +0 -7
- package/dist/chunk-VYUWWIVR.js.map +0 -1
- package/dist/server-PBBRSLBP.js.map +0 -1
- package/dist/worker-BXJQ7J4N.js.map +0 -1
- /package/dist/{chunk-JJ5RMEGX.js.map → chunk-E5J4HJBO.js.map} +0 -0
- /package/dist/{chunk-XDRHBGJR.js.map → chunk-PNBYYAEK.js.map} +0 -0
- /package/dist/{chunk-2P5VFIJM.js.map → chunk-SDHPQTFM.js.map} +0 -0
- /package/dist/{db-Z35S224E.js.map → db-WRILVOFY.js.map} +0 -0
- /package/dist/{extract-bundle-4RO3YU4Q.js.map → extract-bundle-NFFVMUMT.js.map} +0 -0
- /package/dist/{site-extract-repository-FDH6ZXKT.js.map → site-extract-repository-2QQSL2QC.js.map} +0 -0
package/dist/bin/api-server.cjs
CHANGED
|
@@ -3671,12 +3671,27 @@ __export(db_exports, {
|
|
|
3671
3671
|
setStripeCustomerId: () => setStripeCustomerId,
|
|
3672
3672
|
setSubscriptionTier: () => setSubscriptionTier,
|
|
3673
3673
|
setUserMemoryCreds: () => setUserMemoryCreds,
|
|
3674
|
+
settleDebitMcIdempotent: () => settleDebitMcIdempotent,
|
|
3674
3675
|
startHarvestAttempt: () => startHarvestAttempt,
|
|
3675
3676
|
stripeEventAlreadyProcessed: () => stripeEventAlreadyProcessed,
|
|
3676
3677
|
updateKpoJobState: () => updateKpoJobState,
|
|
3677
3678
|
upsertPageSnapshot: () => upsertPageSnapshot,
|
|
3678
3679
|
verifyPassword: () => verifyPassword
|
|
3679
3680
|
});
|
|
3681
|
+
function sqlStringList(values) {
|
|
3682
|
+
return values.map((value) => `'${value.replaceAll("'", "''")}'`).join(", ");
|
|
3683
|
+
}
|
|
3684
|
+
async function hasCurrentCoreSchema(db) {
|
|
3685
|
+
const checks = [
|
|
3686
|
+
`(SELECT COUNT(*) FROM sqlite_master WHERE type = 'table' AND name IN (${sqlStringList(CORE_SCHEMA_TABLES)})) = ${CORE_SCHEMA_TABLES.length}`,
|
|
3687
|
+
`(SELECT COUNT(*) FROM sqlite_master WHERE type = 'index' AND name IN (${sqlStringList(CORE_SCHEMA_INDEXES)})) = ${CORE_SCHEMA_INDEXES.length}`,
|
|
3688
|
+
...Object.entries(CORE_SCHEMA_COLUMNS).map(
|
|
3689
|
+
([table, columns]) => `(SELECT COUNT(*) FROM pragma_table_info('${table}') WHERE name IN (${sqlStringList(columns)})) = ${columns.length}`
|
|
3690
|
+
)
|
|
3691
|
+
];
|
|
3692
|
+
const result = await db.execute(`SELECT ${checks.join(" AND ")} AS ready`);
|
|
3693
|
+
return Number(result.rows[0]?.ready ?? 0) === 1;
|
|
3694
|
+
}
|
|
3680
3695
|
function getDb() {
|
|
3681
3696
|
if (!_db) _db = (0, import_http.createClient)({ url: DB_URL, authToken: DB_TOKEN });
|
|
3682
3697
|
return _db;
|
|
@@ -3686,6 +3701,31 @@ function setDbForTesting(client2) {
|
|
|
3686
3701
|
}
|
|
3687
3702
|
async function migrate() {
|
|
3688
3703
|
const db = getDb();
|
|
3704
|
+
let applied;
|
|
3705
|
+
try {
|
|
3706
|
+
applied = await db.execute({
|
|
3707
|
+
sql: "SELECT 1 AS applied FROM schema_migrations WHERE version = ? LIMIT 1",
|
|
3708
|
+
args: [CORE_SCHEMA_VERSION]
|
|
3709
|
+
});
|
|
3710
|
+
} catch (error) {
|
|
3711
|
+
const message = error instanceof Error ? error.message.toLowerCase() : String(error).toLowerCase();
|
|
3712
|
+
if (!message.includes("no such table") || !message.includes("schema_migrations")) throw error;
|
|
3713
|
+
await db.execute(`
|
|
3714
|
+
CREATE TABLE IF NOT EXISTS schema_migrations (
|
|
3715
|
+
version TEXT PRIMARY KEY,
|
|
3716
|
+
applied_at TEXT NOT NULL DEFAULT (datetime('now'))
|
|
3717
|
+
)
|
|
3718
|
+
`);
|
|
3719
|
+
applied = { rows: [] };
|
|
3720
|
+
}
|
|
3721
|
+
if (applied.rows.length > 0) return;
|
|
3722
|
+
if (await hasCurrentCoreSchema(db)) {
|
|
3723
|
+
await db.execute({
|
|
3724
|
+
sql: "INSERT OR IGNORE INTO schema_migrations (version) VALUES (?)",
|
|
3725
|
+
args: [CORE_SCHEMA_VERSION]
|
|
3726
|
+
});
|
|
3727
|
+
return;
|
|
3728
|
+
}
|
|
3689
3729
|
await db.execute(`
|
|
3690
3730
|
CREATE TABLE IF NOT EXISTS users (
|
|
3691
3731
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
@@ -4179,6 +4219,13 @@ async function migrate() {
|
|
|
4179
4219
|
)
|
|
4180
4220
|
`);
|
|
4181
4221
|
await db.execute(`CREATE INDEX IF NOT EXISTS page_snapshots_user_checked_at ON page_snapshots(user_id, checked_at DESC)`);
|
|
4222
|
+
if (!await hasCurrentCoreSchema(db)) {
|
|
4223
|
+
throw new Error("core schema migration finished without all required tables, indexes, and columns");
|
|
4224
|
+
}
|
|
4225
|
+
await db.execute({
|
|
4226
|
+
sql: "INSERT OR IGNORE INTO schema_migrations (version) VALUES (?)",
|
|
4227
|
+
args: [CORE_SCHEMA_VERSION]
|
|
4228
|
+
});
|
|
4182
4229
|
}
|
|
4183
4230
|
async function registerClient(clientId, redirectUris, name) {
|
|
4184
4231
|
await getDb().execute({
|
|
@@ -4641,6 +4688,17 @@ async function upsertPageSnapshot(userId, url, snapshot) {
|
|
|
4641
4688
|
args: [userId, url, snapshot.contentHash, snapshot.content, snapshot.title, snapshot.contentBytes, snapshot.truncated ? 1 : 0]
|
|
4642
4689
|
});
|
|
4643
4690
|
}
|
|
4691
|
+
function serializeRequestEventResult(result) {
|
|
4692
|
+
if (result == null) return null;
|
|
4693
|
+
const serialized = JSON.stringify(result);
|
|
4694
|
+
const bytes = Buffer.byteLength(serialized, "utf8");
|
|
4695
|
+
if (bytes <= REQUEST_EVENT_RESULT_MAX_BYTES) return serialized;
|
|
4696
|
+
return JSON.stringify({
|
|
4697
|
+
truncated: true,
|
|
4698
|
+
originalBytes: bytes,
|
|
4699
|
+
sha256: (0, import_node_crypto.createHash)("sha256").update(serialized).digest("hex")
|
|
4700
|
+
});
|
|
4701
|
+
}
|
|
4644
4702
|
function rowToRequestEvent(row) {
|
|
4645
4703
|
const rawResult = row.result != null ? String(row.result) : null;
|
|
4646
4704
|
return {
|
|
@@ -4671,7 +4729,7 @@ async function logRequestEvent(input) {
|
|
|
4671
4729
|
input.query,
|
|
4672
4730
|
input.location ?? null,
|
|
4673
4731
|
input.resultCount ?? null,
|
|
4674
|
-
|
|
4732
|
+
serializeRequestEventResult(input.result),
|
|
4675
4733
|
input.error ?? null
|
|
4676
4734
|
]
|
|
4677
4735
|
});
|
|
@@ -4680,7 +4738,7 @@ async function logRequestEvent(input) {
|
|
|
4680
4738
|
async function listRequestEvents(userId, limit = 100) {
|
|
4681
4739
|
const res = await getDb().execute({
|
|
4682
4740
|
sql: `
|
|
4683
|
-
SELECT id, user_id, source, status, query, location, result_count, result, error, created_at
|
|
4741
|
+
SELECT id, user_id, source, status, query, location, result_count, NULL AS result, error, created_at
|
|
4684
4742
|
FROM request_events
|
|
4685
4743
|
WHERE user_id = ?
|
|
4686
4744
|
ORDER BY created_at DESC
|
|
@@ -5308,58 +5366,75 @@ async function debitMcIdempotent(userId, mc, operation, description, idempotency
|
|
|
5308
5366
|
if (!idempotencyKey || idempotencyKey.length > 500) throw new Error("idempotent debit key is required and must be bounded");
|
|
5309
5367
|
const db = getDb();
|
|
5310
5368
|
await ensureMigrated(userId);
|
|
5311
|
-
|
|
5312
|
-
|
|
5313
|
-
|
|
5314
|
-
|
|
5315
|
-
|
|
5316
|
-
|
|
5317
|
-
|
|
5318
|
-
|
|
5319
|
-
|
|
5320
|
-
|
|
5321
|
-
|
|
5322
|
-
|
|
5323
|
-
|
|
5324
|
-
|
|
5325
|
-
|
|
5326
|
-
|
|
5327
|
-
|
|
5328
|
-
|
|
5329
|
-
|
|
5330
|
-
|
|
5331
|
-
|
|
5332
|
-
|
|
5333
|
-
|
|
5334
|
-
|
|
5335
|
-
|
|
5336
|
-
|
|
5337
|
-
|
|
5338
|
-
|
|
5339
|
-
|
|
5340
|
-
|
|
5341
|
-
|
|
5342
|
-
|
|
5343
|
-
|
|
5344
|
-
|
|
5345
|
-
|
|
5346
|
-
|
|
5347
|
-
|
|
5348
|
-
|
|
5349
|
-
|
|
5350
|
-
|
|
5351
|
-
|
|
5352
|
-
|
|
5353
|
-
|
|
5354
|
-
|
|
5369
|
+
let results;
|
|
5370
|
+
try {
|
|
5371
|
+
results = await db.batch([
|
|
5372
|
+
{
|
|
5373
|
+
sql: "UPDATE billing_debits SET amount_mc = ? WHERE idempotency_key = ? AND user_id = ? AND operation = ? AND description = ? AND status = 'claimed'",
|
|
5374
|
+
args: [mc, idempotencyKey, userId, operation, description]
|
|
5375
|
+
},
|
|
5376
|
+
{
|
|
5377
|
+
sql: "INSERT OR IGNORE INTO billing_debits (idempotency_key, user_id, amount_mc, operation, description, status) VALUES (?, ?, ?, ?, ?, 'claimed')",
|
|
5378
|
+
args: [idempotencyKey, userId, mc, operation, description]
|
|
5379
|
+
},
|
|
5380
|
+
{
|
|
5381
|
+
sql: "UPDATE users SET balance_mc = balance_mc - ? WHERE id = ? AND balance_mc >= ? AND EXISTS (SELECT 1 FROM billing_debits WHERE idempotency_key = ? AND user_id = ? AND amount_mc = ? AND operation = ? AND description = ? AND status = 'claimed')",
|
|
5382
|
+
args: [mc, userId, mc, idempotencyKey, userId, mc, operation, description]
|
|
5383
|
+
},
|
|
5384
|
+
{
|
|
5385
|
+
sql: "UPDATE billing_debits SET status = 'applying' WHERE idempotency_key = ? AND user_id = ? AND amount_mc = ? AND operation = ? AND description = ? AND status = 'claimed' AND changes() = 1",
|
|
5386
|
+
args: [idempotencyKey, userId, mc, operation, description]
|
|
5387
|
+
},
|
|
5388
|
+
{
|
|
5389
|
+
sql: `WITH ordered AS MATERIALIZED (
|
|
5390
|
+
SELECT id,
|
|
5391
|
+
remaining_mc,
|
|
5392
|
+
COALESCE(SUM(remaining_mc) OVER (
|
|
5393
|
+
ORDER BY expires_at ASC, granted_at ASC, id ASC
|
|
5394
|
+
ROWS BETWEEN UNBOUNDED PRECEDING AND 1 PRECEDING
|
|
5395
|
+
), 0) AS prior_mc
|
|
5396
|
+
FROM credit_lots
|
|
5397
|
+
WHERE user_id = ? AND remaining_mc > 0 AND expires_at > datetime('now')
|
|
5398
|
+
), allocation AS (
|
|
5399
|
+
SELECT id, MIN(remaining_mc, MAX(0, ? - prior_mc)) AS take_mc
|
|
5400
|
+
FROM ordered
|
|
5401
|
+
)
|
|
5402
|
+
UPDATE credit_lots
|
|
5403
|
+
SET remaining_mc = remaining_mc - COALESCE((SELECT take_mc FROM allocation WHERE allocation.id = credit_lots.id), 0)
|
|
5404
|
+
WHERE id IN (SELECT id FROM allocation WHERE take_mc > 0)
|
|
5405
|
+
AND EXISTS (SELECT 1 FROM billing_debits WHERE idempotency_key = ? AND status = 'applying')`,
|
|
5406
|
+
args: [userId, mc, idempotencyKey]
|
|
5407
|
+
},
|
|
5408
|
+
{
|
|
5409
|
+
sql: "INSERT INTO ledger (user_id, amount_mc, operation, description) SELECT user_id, -amount_mc, operation, description FROM billing_debits WHERE idempotency_key = ? AND status = 'applying'",
|
|
5410
|
+
args: [idempotencyKey]
|
|
5411
|
+
},
|
|
5412
|
+
{
|
|
5413
|
+
sql: "UPDATE billing_debits SET status = 'applied', applied_at = datetime('now') WHERE idempotency_key = ? AND status = 'applying'",
|
|
5414
|
+
args: [idempotencyKey]
|
|
5415
|
+
}
|
|
5416
|
+
], "write");
|
|
5417
|
+
} catch (error) {
|
|
5418
|
+
try {
|
|
5419
|
+
const recovered = await db.execute({
|
|
5420
|
+
sql: "SELECT user_id, amount_mc, operation, description, status FROM billing_debits WHERE idempotency_key = ? AND status = 'applied' LIMIT 1",
|
|
5421
|
+
args: [idempotencyKey]
|
|
5422
|
+
});
|
|
5423
|
+
const row2 = recovered.rows[0];
|
|
5424
|
+
if (row2 && Number(row2.user_id) === userId && Number(row2.amount_mc) === mc && String(row2.operation) === operation && String(row2.description) === description) {
|
|
5425
|
+
const balance2 = await db.execute({ sql: "SELECT balance_mc FROM users WHERE id = ?", args: [userId] });
|
|
5426
|
+
return { ok: true, balance_mc: Number(balance2.rows[0]?.balance_mc ?? 0), duplicate: true };
|
|
5427
|
+
}
|
|
5428
|
+
} catch {
|
|
5355
5429
|
}
|
|
5356
|
-
|
|
5430
|
+
throw error;
|
|
5431
|
+
}
|
|
5357
5432
|
const claim = await db.execute({
|
|
5358
|
-
sql: "SELECT user_id, amount_mc, operation, status FROM billing_debits WHERE idempotency_key = ? LIMIT 1",
|
|
5433
|
+
sql: "SELECT user_id, amount_mc, operation, description, status FROM billing_debits WHERE idempotency_key = ? LIMIT 1",
|
|
5359
5434
|
args: [idempotencyKey]
|
|
5360
5435
|
});
|
|
5361
5436
|
const row = claim.rows[0];
|
|
5362
|
-
if (!row || Number(row.user_id) !== userId || Number(row.amount_mc) !== mc || String(row.operation) !== operation) {
|
|
5437
|
+
if (!row || Number(row.user_id) !== userId || Number(row.amount_mc) !== mc || String(row.operation) !== operation || String(row.description) !== description) {
|
|
5363
5438
|
throw new Error("idempotent debit key conflicts with another debit");
|
|
5364
5439
|
}
|
|
5365
5440
|
const balance = await db.execute({ sql: "SELECT balance_mc FROM users WHERE id = ?", args: [userId] });
|
|
@@ -5369,6 +5444,158 @@ async function debitMcIdempotent(userId, mc, operation, description, idempotency
|
|
|
5369
5444
|
duplicate: results[2]?.rowsAffected === 0 && String(row.status) === "applied"
|
|
5370
5445
|
};
|
|
5371
5446
|
}
|
|
5447
|
+
async function settleDebitMcIdempotent(userId, debitIdempotencyKey, finalAmountMc, refundOperation, description, sourceSurface) {
|
|
5448
|
+
if (!Number.isSafeInteger(finalAmountMc) || finalAmountMc < 0) {
|
|
5449
|
+
throw new Error("idempotent debit settlement amount must be a non-negative integer");
|
|
5450
|
+
}
|
|
5451
|
+
if (!debitIdempotencyKey || debitIdempotencyKey.length > 500) {
|
|
5452
|
+
throw new Error("idempotent debit settlement key is required and must be bounded");
|
|
5453
|
+
}
|
|
5454
|
+
if (!refundOperation || !sourceSurface) {
|
|
5455
|
+
throw new Error("idempotent debit settlement operation and source are required");
|
|
5456
|
+
}
|
|
5457
|
+
const db = getDb();
|
|
5458
|
+
await ensureMigrated(userId);
|
|
5459
|
+
const settlementKey = `debit-settlement:${(0, import_node_crypto.createHash)("sha256").update(debitIdempotencyKey).digest("hex")}`;
|
|
5460
|
+
const billingClass = "wallet_hold_settlement";
|
|
5461
|
+
const eventId = (0, import_node_crypto.randomUUID)();
|
|
5462
|
+
const metadata = JSON.stringify({ debitIdempotencyKey, refundOperation, description });
|
|
5463
|
+
const readReceipt = async () => {
|
|
5464
|
+
const receipt2 = await db.execute({
|
|
5465
|
+
sql: `SELECT e.user_id,
|
|
5466
|
+
e.status,
|
|
5467
|
+
e.amount_mc AS final_amount_mc,
|
|
5468
|
+
e.billing_class,
|
|
5469
|
+
e.source_surface,
|
|
5470
|
+
d.amount_mc AS held_amount_mc
|
|
5471
|
+
FROM billing_events e
|
|
5472
|
+
JOIN billing_debits d ON d.idempotency_key = ?
|
|
5473
|
+
WHERE e.idempotency_key = ?
|
|
5474
|
+
AND d.user_id = e.user_id
|
|
5475
|
+
AND d.status = 'applied'
|
|
5476
|
+
LIMIT 1`,
|
|
5477
|
+
args: [debitIdempotencyKey, settlementKey]
|
|
5478
|
+
});
|
|
5479
|
+
const row = receipt2.rows[0];
|
|
5480
|
+
if (!row) return null;
|
|
5481
|
+
if (Number(row.user_id) !== userId || String(row.billing_class) !== billingClass || String(row.source_surface) !== sourceSurface) {
|
|
5482
|
+
throw new Error("idempotent debit settlement key conflicts with another settlement");
|
|
5483
|
+
}
|
|
5484
|
+
if (String(row.status) !== "settled") return null;
|
|
5485
|
+
const heldAmountMc = Number(row.held_amount_mc);
|
|
5486
|
+
const durableFinalAmountMc = Number(row.final_amount_mc);
|
|
5487
|
+
if (!Number.isSafeInteger(heldAmountMc) || !Number.isSafeInteger(durableFinalAmountMc) || durableFinalAmountMc < 0 || durableFinalAmountMc > heldAmountMc) {
|
|
5488
|
+
throw new Error("idempotent debit settlement receipt is invalid");
|
|
5489
|
+
}
|
|
5490
|
+
const balance = await db.execute({ sql: "SELECT balance_mc FROM users WHERE id = ?", args: [userId] });
|
|
5491
|
+
return {
|
|
5492
|
+
balance_mc: Number(balance.rows[0]?.balance_mc ?? 0),
|
|
5493
|
+
final_amount_mc: durableFinalAmountMc,
|
|
5494
|
+
refund_mc: heldAmountMc - durableFinalAmountMc,
|
|
5495
|
+
duplicate: true
|
|
5496
|
+
};
|
|
5497
|
+
};
|
|
5498
|
+
let results;
|
|
5499
|
+
try {
|
|
5500
|
+
results = await db.batch([
|
|
5501
|
+
{
|
|
5502
|
+
sql: `INSERT OR IGNORE INTO billing_events
|
|
5503
|
+
(id, user_id, idempotency_key, billing_class, source_surface, status, amount_mc, metadata)
|
|
5504
|
+
VALUES (?, ?, ?, ?, ?, 'settling', ?, ?)`,
|
|
5505
|
+
args: [eventId, userId, settlementKey, billingClass, sourceSurface, finalAmountMc, metadata]
|
|
5506
|
+
},
|
|
5507
|
+
{
|
|
5508
|
+
sql: `UPDATE users
|
|
5509
|
+
SET balance_mc = balance_mc + (
|
|
5510
|
+
SELECT d.amount_mc - e.amount_mc
|
|
5511
|
+
FROM billing_events e
|
|
5512
|
+
JOIN billing_debits d ON d.idempotency_key = ?
|
|
5513
|
+
WHERE e.idempotency_key = ?
|
|
5514
|
+
)
|
|
5515
|
+
WHERE id = ?
|
|
5516
|
+
AND EXISTS (
|
|
5517
|
+
SELECT 1
|
|
5518
|
+
FROM billing_events e
|
|
5519
|
+
JOIN billing_debits d ON d.idempotency_key = ?
|
|
5520
|
+
WHERE e.idempotency_key = ?
|
|
5521
|
+
AND e.user_id = ?
|
|
5522
|
+
AND e.billing_class = ?
|
|
5523
|
+
AND e.source_surface = ?
|
|
5524
|
+
AND e.status = 'settling'
|
|
5525
|
+
AND d.user_id = e.user_id
|
|
5526
|
+
AND d.status = 'applied'
|
|
5527
|
+
AND e.amount_mc BETWEEN 0 AND d.amount_mc
|
|
5528
|
+
)`,
|
|
5529
|
+
args: [
|
|
5530
|
+
debitIdempotencyKey,
|
|
5531
|
+
settlementKey,
|
|
5532
|
+
userId,
|
|
5533
|
+
debitIdempotencyKey,
|
|
5534
|
+
settlementKey,
|
|
5535
|
+
userId,
|
|
5536
|
+
billingClass,
|
|
5537
|
+
sourceSurface
|
|
5538
|
+
]
|
|
5539
|
+
},
|
|
5540
|
+
{
|
|
5541
|
+
sql: `UPDATE billing_events
|
|
5542
|
+
SET status = 'applying'
|
|
5543
|
+
WHERE idempotency_key = ?
|
|
5544
|
+
AND user_id = ?
|
|
5545
|
+
AND billing_class = ?
|
|
5546
|
+
AND source_surface = ?
|
|
5547
|
+
AND status = 'settling'
|
|
5548
|
+
AND changes() = 1`,
|
|
5549
|
+
args: [settlementKey, userId, billingClass, sourceSurface]
|
|
5550
|
+
},
|
|
5551
|
+
{
|
|
5552
|
+
sql: `INSERT INTO credit_lots
|
|
5553
|
+
(user_id, amount_mc, remaining_mc, source, expires_at)
|
|
5554
|
+
SELECT e.user_id,
|
|
5555
|
+
d.amount_mc - e.amount_mc,
|
|
5556
|
+
d.amount_mc - e.amount_mc,
|
|
5557
|
+
?,
|
|
5558
|
+
datetime('now', ?)
|
|
5559
|
+
FROM billing_events e
|
|
5560
|
+
JOIN billing_debits d ON d.idempotency_key = ?
|
|
5561
|
+
WHERE e.idempotency_key = ?
|
|
5562
|
+
AND e.status = 'applying'
|
|
5563
|
+
AND d.amount_mc > e.amount_mc`,
|
|
5564
|
+
args: [refundOperation, CREDIT_LOT_TTL, debitIdempotencyKey, settlementKey]
|
|
5565
|
+
},
|
|
5566
|
+
{
|
|
5567
|
+
sql: `INSERT INTO ledger (user_id, amount_mc, operation, description)
|
|
5568
|
+
SELECT e.user_id,
|
|
5569
|
+
d.amount_mc - e.amount_mc,
|
|
5570
|
+
?,
|
|
5571
|
+
?
|
|
5572
|
+
FROM billing_events e
|
|
5573
|
+
JOIN billing_debits d ON d.idempotency_key = ?
|
|
5574
|
+
WHERE e.idempotency_key = ?
|
|
5575
|
+
AND e.status = 'applying'
|
|
5576
|
+
AND d.amount_mc > e.amount_mc`,
|
|
5577
|
+
args: [refundOperation, description, debitIdempotencyKey, settlementKey]
|
|
5578
|
+
},
|
|
5579
|
+
{
|
|
5580
|
+
sql: `UPDATE billing_events
|
|
5581
|
+
SET status = 'settled', settled_at = datetime('now')
|
|
5582
|
+
WHERE idempotency_key = ?
|
|
5583
|
+
AND status = 'applying'`,
|
|
5584
|
+
args: [settlementKey]
|
|
5585
|
+
}
|
|
5586
|
+
], "write");
|
|
5587
|
+
} catch (error) {
|
|
5588
|
+
try {
|
|
5589
|
+
const recovered = await readReceipt();
|
|
5590
|
+
if (recovered) return recovered;
|
|
5591
|
+
} catch {
|
|
5592
|
+
}
|
|
5593
|
+
throw error;
|
|
5594
|
+
}
|
|
5595
|
+
const receipt = await readReceipt();
|
|
5596
|
+
if (!receipt) throw new Error("idempotent debit settlement did not reach a durable settled state");
|
|
5597
|
+
return { ...receipt, duplicate: results[0]?.rowsAffected === 0 };
|
|
5598
|
+
}
|
|
5372
5599
|
async function getLedger(userId, limit = 50) {
|
|
5373
5600
|
const res = await getDb().execute({
|
|
5374
5601
|
sql: "SELECT * FROM ledger WHERE user_id = ? ORDER BY created_at DESC LIMIT ?",
|
|
@@ -5417,7 +5644,7 @@ async function reconcileBalanceMc(userId, knownBalanceMc) {
|
|
|
5417
5644
|
}
|
|
5418
5645
|
return balanceMc;
|
|
5419
5646
|
}
|
|
5420
|
-
var import_http, import_node_crypto, import_zod, DB_URL, DB_TOKEN, _db, _rateLimitSchemaReady, CREDIT_LOT_TTL, SiteAuditJobRowSchema, SiteAuditPhaseLogRowSchema;
|
|
5647
|
+
var import_http, import_node_crypto, import_zod, DB_URL, DB_TOKEN, _db, _rateLimitSchemaReady, CORE_SCHEMA_VERSION, CORE_SCHEMA_TABLES, CORE_SCHEMA_INDEXES, CORE_SCHEMA_COLUMNS, REQUEST_EVENT_RESULT_MAX_BYTES, CREDIT_LOT_TTL, SiteAuditJobRowSchema, SiteAuditPhaseLogRowSchema;
|
|
5421
5648
|
var init_db = __esm({
|
|
5422
5649
|
"src/api/db.ts"() {
|
|
5423
5650
|
"use strict";
|
|
@@ -5428,6 +5655,95 @@ var init_db = __esm({
|
|
|
5428
5655
|
DB_TOKEN = process.env.TURSO_AUTH_TOKEN;
|
|
5429
5656
|
_db = null;
|
|
5430
5657
|
_rateLimitSchemaReady = false;
|
|
5658
|
+
CORE_SCHEMA_VERSION = "2026-07-14.1";
|
|
5659
|
+
CORE_SCHEMA_TABLES = [
|
|
5660
|
+
"billing_authorizations",
|
|
5661
|
+
"billing_debits",
|
|
5662
|
+
"billing_events",
|
|
5663
|
+
"concurrency_locks",
|
|
5664
|
+
"credit_lots",
|
|
5665
|
+
"free_credit_refreshes",
|
|
5666
|
+
"harvest_attempts",
|
|
5667
|
+
"jobs",
|
|
5668
|
+
"kpo_jobs",
|
|
5669
|
+
"kpo_phase_log",
|
|
5670
|
+
"ledger",
|
|
5671
|
+
"oauth_clients",
|
|
5672
|
+
"oauth_codes",
|
|
5673
|
+
"oauth_tokens",
|
|
5674
|
+
"page_snapshots",
|
|
5675
|
+
"password_reset_tokens",
|
|
5676
|
+
"rate_limits",
|
|
5677
|
+
"request_events",
|
|
5678
|
+
"site_audit_jobs",
|
|
5679
|
+
"site_audit_phase_log",
|
|
5680
|
+
"site_extract_jobs",
|
|
5681
|
+
"site_extract_pages",
|
|
5682
|
+
"stripe_events",
|
|
5683
|
+
"users",
|
|
5684
|
+
"workflow_artifacts",
|
|
5685
|
+
"workflow_runs",
|
|
5686
|
+
"workflow_schedules",
|
|
5687
|
+
"workflow_webhook_deliveries"
|
|
5688
|
+
];
|
|
5689
|
+
CORE_SCHEMA_INDEXES = [
|
|
5690
|
+
"billing_authorizations_user_status",
|
|
5691
|
+
"billing_debits_user_status",
|
|
5692
|
+
"billing_events_authorization",
|
|
5693
|
+
"billing_events_user_created_at",
|
|
5694
|
+
"concurrency_locks_operation",
|
|
5695
|
+
"concurrency_locks_user_status",
|
|
5696
|
+
"credit_lots_user_active",
|
|
5697
|
+
"harvest_attempts_job_id",
|
|
5698
|
+
"harvest_attempts_user_created_at",
|
|
5699
|
+
"jobs_status",
|
|
5700
|
+
"jobs_user_id",
|
|
5701
|
+
"kpo_jobs_status",
|
|
5702
|
+
"kpo_jobs_user_id",
|
|
5703
|
+
"kpo_phase_log_job_id",
|
|
5704
|
+
"ledger_stripe_pi_unique",
|
|
5705
|
+
"ledger_user_id",
|
|
5706
|
+
"oauth_tokens_identity",
|
|
5707
|
+
"page_snapshots_user_checked_at",
|
|
5708
|
+
"prt_user_id",
|
|
5709
|
+
"request_events_user_created_at",
|
|
5710
|
+
"site_audit_jobs_status",
|
|
5711
|
+
"site_audit_jobs_user_id",
|
|
5712
|
+
"site_audit_phase_log_job_id",
|
|
5713
|
+
"site_extract_jobs_status",
|
|
5714
|
+
"site_extract_jobs_user_id",
|
|
5715
|
+
"site_extract_pages_job_id",
|
|
5716
|
+
"workflow_artifacts_run",
|
|
5717
|
+
"workflow_runs_idempotency",
|
|
5718
|
+
"workflow_runs_schedule",
|
|
5719
|
+
"workflow_runs_user_created",
|
|
5720
|
+
"workflow_schedules_due",
|
|
5721
|
+
"workflow_schedules_user_status",
|
|
5722
|
+
"workflow_webhook_deliveries_run"
|
|
5723
|
+
];
|
|
5724
|
+
CORE_SCHEMA_COLUMNS = {
|
|
5725
|
+
harvest_attempts: ["debug_json"],
|
|
5726
|
+
oauth_tokens: ["replaced_by", "rotated_at"],
|
|
5727
|
+
users: [
|
|
5728
|
+
"balance_mc",
|
|
5729
|
+
"concurrency_stripe_sub_id",
|
|
5730
|
+
"extra_concurrency_slots",
|
|
5731
|
+
"key_active",
|
|
5732
|
+
"memory_identity",
|
|
5733
|
+
"memory_key",
|
|
5734
|
+
"memory_plan",
|
|
5735
|
+
"memory_provisioned",
|
|
5736
|
+
"memory_subscription_id",
|
|
5737
|
+
"password_hash",
|
|
5738
|
+
"stripe_customer_id",
|
|
5739
|
+
"subscription_concurrency",
|
|
5740
|
+
"subscription_id",
|
|
5741
|
+
"subscription_tier"
|
|
5742
|
+
],
|
|
5743
|
+
workflow_artifacts: ["content"],
|
|
5744
|
+
workflow_runs: ["step_index", "step_log_json", "step_state_json", "total_steps", "webhook_url"]
|
|
5745
|
+
};
|
|
5746
|
+
REQUEST_EVENT_RESULT_MAX_BYTES = 32 * 1024;
|
|
5431
5747
|
CREDIT_LOT_TTL = "+3 months";
|
|
5432
5748
|
SiteAuditJobRowSchema = import_zod.z.object({
|
|
5433
5749
|
id: import_zod.z.string(),
|
|
@@ -5499,7 +5815,26 @@ var init_cost_rates = __esm({
|
|
|
5499
5815
|
// src/api/cost-telemetry.ts
|
|
5500
5816
|
async function migrateCostTelemetry() {
|
|
5501
5817
|
if (_ready) return;
|
|
5818
|
+
if (_migrationPromise) return _migrationPromise;
|
|
5819
|
+
_migrationPromise = runCostTelemetryMigration().finally(() => {
|
|
5820
|
+
_migrationPromise = null;
|
|
5821
|
+
});
|
|
5822
|
+
return _migrationPromise;
|
|
5823
|
+
}
|
|
5824
|
+
async function runCostTelemetryMigration() {
|
|
5502
5825
|
const db = getDb();
|
|
5826
|
+
const existing = await db.execute(`
|
|
5827
|
+
SELECT
|
|
5828
|
+
(SELECT COUNT(*) FROM sqlite_master WHERE type = 'table' AND name IN ('kernel_session_log', 'vendor_usage_log', 'cost_probe_runs')) = 3
|
|
5829
|
+
AND (SELECT COUNT(*) FROM pragma_table_info('kernel_session_log') WHERE name IN ('proxy_source', 'proxy_type', 'method')) = 3
|
|
5830
|
+
AND (SELECT COUNT(*) FROM pragma_table_info('vendor_usage_log') WHERE name = 'method') = 1
|
|
5831
|
+
AND (SELECT COUNT(*) FROM pragma_table_info('cost_probe_runs') WHERE name IN ('units', 'unit_type', 'mode')) = 3
|
|
5832
|
+
AS ready
|
|
5833
|
+
`);
|
|
5834
|
+
if (Number(existing.rows[0]?.ready ?? 0) === 1) {
|
|
5835
|
+
_ready = true;
|
|
5836
|
+
return;
|
|
5837
|
+
}
|
|
5503
5838
|
await db.execute(`
|
|
5504
5839
|
CREATE TABLE IF NOT EXISTS kernel_session_log (
|
|
5505
5840
|
id TEXT PRIMARY KEY,
|
|
@@ -5663,7 +5998,7 @@ function boolToInt(v) {
|
|
|
5663
5998
|
if (v === null || v === void 0) return null;
|
|
5664
5999
|
return v ? 1 : 0;
|
|
5665
6000
|
}
|
|
5666
|
-
var import_node_crypto2, _ready;
|
|
6001
|
+
var import_node_crypto2, _ready, _migrationPromise;
|
|
5667
6002
|
var init_cost_telemetry = __esm({
|
|
5668
6003
|
"src/api/cost-telemetry.ts"() {
|
|
5669
6004
|
"use strict";
|
|
@@ -5672,6 +6007,7 @@ var init_cost_telemetry = __esm({
|
|
|
5672
6007
|
init_cost_context();
|
|
5673
6008
|
init_cost_rates();
|
|
5674
6009
|
_ready = false;
|
|
6010
|
+
_migrationPromise = null;
|
|
5675
6011
|
}
|
|
5676
6012
|
});
|
|
5677
6013
|
|
|
@@ -8563,6 +8899,9 @@ var init_schemas = __esm({
|
|
|
8563
8899
|
});
|
|
8564
8900
|
|
|
8565
8901
|
// src/api/rates.ts
|
|
8902
|
+
function serpActualCostMc(headlessSent) {
|
|
8903
|
+
return headlessSent === true ? MC_COSTS.serp_headless : MC_COSTS.serp_headful;
|
|
8904
|
+
}
|
|
8566
8905
|
function browserActiveCostMc(activeMs) {
|
|
8567
8906
|
return Math.round(activeMs * MC_PER_BROWSER_MS);
|
|
8568
8907
|
}
|
|
@@ -8617,12 +8956,12 @@ function insufficientBalanceResponse(balanceMc, requiredMc) {
|
|
|
8617
8956
|
topup_url: topupUrl
|
|
8618
8957
|
};
|
|
8619
8958
|
}
|
|
8620
|
-
var MC_COSTS, MC_PER_BROWSER_MS, BROWSER_OPEN_MIN_BALANCE_MC, MC_PER_CREDIT, BILLING_MC_PER_USD_NUMERATOR, BILLING_MC_PER_USD_DENOMINATOR, BILLING_BASIS_POINTS, USD_NANOS, SCHEDULED_RUN_BASE_CREDITS, SCHEDULED_RUN_BASE_MC, SCHEDULED_AGENT_LLM_MARKUP_BPS, SCHEDULED_AGENT_LLM_MARKUP, SCHEDULED_RUN_DEFAULT_MAX_CREDITS, SCHEDULED_ACTIONS_RATE_POLICY, CREDIT_COST_CATALOG, CONCURRENCY_PRICE_ID, SUBSCRIPTION_TIERS, SUBSCRIPTION_TIER_BY_KEY, CONCURRENCY_SLOT_PRICE_USD, CONCURRENCY_SLOT_PRICE_CURRENCY, CONCURRENCY_SLOT_PRICE_INTERVAL, CONCURRENCY_SLOT_PRICE_LABEL, CONCURRENCY_SLOT_TERMINAL_COMMAND,
|
|
8959
|
+
var MC_COSTS, MC_PER_BROWSER_MS, BROWSER_OPEN_MIN_BALANCE_MC, MC_PER_CREDIT, BILLING_MC_PER_USD_NUMERATOR, BILLING_MC_PER_USD_DENOMINATOR, BILLING_BASIS_POINTS, USD_NANOS, SCHEDULED_RUN_BASE_CREDITS, SCHEDULED_RUN_BASE_MC, SCHEDULED_AGENT_LLM_MARKUP_BPS, SCHEDULED_AGENT_LLM_MARKUP, SCHEDULED_RUN_DEFAULT_MAX_CREDITS, SCHEDULED_ACTIONS_RATE_POLICY, CREDIT_COST_CATALOG, CONCURRENCY_PRICE_ID, SUBSCRIPTION_TIERS, SUBSCRIPTION_TIER_BY_KEY, CONCURRENCY_SLOT_PRICE_USD, CONCURRENCY_SLOT_PRICE_CURRENCY, CONCURRENCY_SLOT_PRICE_INTERVAL, CONCURRENCY_SLOT_PRICE_LABEL, CONCURRENCY_SLOT_TERMINAL_COMMAND, MEMORY_FREE_QUOTA_BYTES, MEMORY_PLANS, MEMORY_PLAN_QUOTA, MEMORY_MARGIN_MULTIPLE, MEMORY_PRO_COST_BUDGET_USD, LedgerOperation, MEMORY_AI_MARGIN_MULTIPLE, SITE_AUDIT_LLM_MARGIN_MULTIPLE, VIDEO_ANALYSIS_LLM_MARGIN_MULTIPLE, MEDIA_TRANSCRIBE_MARGIN_MULTIPLE, MEDIA_TRANSCRIBE_HOLD_MC, MC_PER_USD;
|
|
8621
8960
|
var init_rates = __esm({
|
|
8622
8961
|
"src/api/rates.ts"() {
|
|
8623
8962
|
"use strict";
|
|
8624
8963
|
MC_COSTS = {
|
|
8625
|
-
serp:
|
|
8964
|
+
serp: 400,
|
|
8626
8965
|
serp_headless: 400,
|
|
8627
8966
|
serp_headful: 1400,
|
|
8628
8967
|
paa: 300,
|
|
@@ -8696,7 +9035,7 @@ var init_rates = __esm({
|
|
|
8696
9035
|
aliases: ["search_serp", "serp", "google search", "organic results"],
|
|
8697
9036
|
credits: mcToCredits(MC_COSTS.serp_headless),
|
|
8698
9037
|
unit: "per search (headless); escalates to headful only when needed",
|
|
8699
|
-
notes: `Returns AI Overview, PAA snippet, videos, forums, and local pack. Billed ${mcToCredits(MC_COSTS.serp_headless)} credits when served headless (the common case), ${mcToCredits(MC_COSTS.serp_headful)} credits if an anti-bot escalation forces headful mode (overestimate refunded).`
|
|
9038
|
+
notes: `Returns AI Overview, PAA snippet, videos, forums, and local pack. Billed ${mcToCredits(MC_COSTS.serp_headless)} credits when served headless (the common case), ${mcToCredits(MC_COSTS.serp_headful)} credits if an anti-bot escalation forces headful mode (overestimate refunded). Optional SERP Intelligence page snapshots add ${mcToCredits(MC_COSTS.page_scrape)} credit per attempted URL.`
|
|
8700
9039
|
},
|
|
8701
9040
|
{
|
|
8702
9041
|
key: "paa",
|
|
@@ -8919,7 +9258,6 @@ var init_rates = __esm({
|
|
|
8919
9258
|
CONCURRENCY_SLOT_PRICE_INTERVAL = "month";
|
|
8920
9259
|
CONCURRENCY_SLOT_PRICE_LABEL = "$5/month";
|
|
8921
9260
|
CONCURRENCY_SLOT_TERMINAL_COMMAND = "npx -y -p mcp-scraper@latest mcp-scraper-cli billing concurrency checkout";
|
|
8922
|
-
FREE_SIGNUP_MC = 1e5;
|
|
8923
9261
|
MEMORY_FREE_QUOTA_BYTES = 1e7 * 1;
|
|
8924
9262
|
MEMORY_PLANS = {
|
|
8925
9263
|
"price_1TnMSTS8aAcsk3TGBgiwuvqL": { plan: "pro", label: "Pro", price_id: "price_1TnMSTS8aAcsk3TGBgiwuvqL", interval: "month", monthly_usd: 19, quota_bytes: 5e9 },
|
|
@@ -16624,6 +16962,15 @@ function buildPageIntelUrl(body, country) {
|
|
|
16624
16962
|
if (body.pageId?.trim()) return `https://www.facebook.com/ads/library/?active_status=all&ad_type=all&country=${country}&is_targeted_country=false&media_type=all&search_type=page&view_all_page_id=${body.pageId.trim()}`;
|
|
16625
16963
|
return `https://www.facebook.com/ads/library/?active_status=all&ad_type=all&country=${country}&q=${encodeURIComponent(body.query.trim())}&search_type=keyword_unordered`;
|
|
16626
16964
|
}
|
|
16965
|
+
function pageIntelRequestContext(body) {
|
|
16966
|
+
const country = body.country?.trim().toUpperCase() ?? "US";
|
|
16967
|
+
return {
|
|
16968
|
+
country,
|
|
16969
|
+
maxAds: Math.min(200, Math.max(1, body.maxAds ?? 50)),
|
|
16970
|
+
listingUrl: buildPageIntelUrl(body, country),
|
|
16971
|
+
subject: body.pageId ?? body.query ?? body.libraryId ?? ""
|
|
16972
|
+
};
|
|
16973
|
+
}
|
|
16627
16974
|
function normalizeAdvertiserName(value) {
|
|
16628
16975
|
return value.toLowerCase().normalize("NFKD").replace(/[^a-z0-9]+/g, " ").trim().replace(/\s+/g, " ");
|
|
16629
16976
|
}
|
|
@@ -16788,16 +17135,14 @@ var init_facebook_ad_routes = __esm({
|
|
|
16788
17135
|
const parsed = FacebookPageIntelBodySchema.safeParse(raw);
|
|
16789
17136
|
if (!parsed.success) return c.json(invalidRequest(parsed.error.issues[0]?.message ?? "Invalid request"), 400);
|
|
16790
17137
|
const body = parsed.data;
|
|
16791
|
-
const
|
|
16792
|
-
const country = body.country?.trim().toUpperCase() ?? "US";
|
|
16793
|
-
const listingUrl = buildPageIntelUrl(body, country);
|
|
17138
|
+
const { country, maxAds, listingUrl, subject } = pageIntelRequestContext(body);
|
|
16794
17139
|
const fbUser = c.get("user");
|
|
16795
17140
|
const gate = await acquireConcurrencyGate(fbUser, "facebook_page_intel", { reuseLockId: c.req.header("x-mcp-scraper-concurrency-lock"), metadata: { pageId: body.pageId, query: body.query, libraryId: body.libraryId, country } });
|
|
16796
17141
|
if (!gate.ok) return c.json(concurrencyLimitExceededResponse(gate), 429, { "Retry-After": String(gate.retryAfterSeconds) });
|
|
16797
17142
|
const driver = new BrowserDriver();
|
|
16798
17143
|
let refunded = false, debited = false;
|
|
16799
17144
|
try {
|
|
16800
|
-
const { ok: fbOk, balance_mc: fbBal } = await debitMc(fbUser.id, MC_COSTS.fb_ad, LedgerOperation.FB_AD,
|
|
17145
|
+
const { ok: fbOk, balance_mc: fbBal } = await debitMc(fbUser.id, MC_COSTS.fb_ad, LedgerOperation.FB_AD, subject);
|
|
16801
17146
|
if (!fbOk) return c.json(insufficientBalanceResponse(fbBal, MC_COSTS.fb_ad), 402);
|
|
16802
17147
|
debited = true;
|
|
16803
17148
|
await driver.launch(await kernelLaunchOptsResidential());
|
|
@@ -16807,17 +17152,17 @@ var init_facebook_ad_routes = __esm({
|
|
|
16807
17152
|
if (result.ads.length === 0 && await detectSoftBlock(driver)) {
|
|
16808
17153
|
await creditMc(fbUser.id, MC_COSTS.fb_ad, LedgerOperation.FB_AD_REFUND, "soft-block empty result");
|
|
16809
17154
|
refunded = true;
|
|
16810
|
-
await logRequestEvent({ userId: fbUser.id, source: "facebook_page_intel", status: "failed", query:
|
|
17155
|
+
await logRequestEvent({ userId: fbUser.id, source: "facebook_page_intel", status: "failed", query: subject, error: "soft-block: empty result refunded" });
|
|
16811
17156
|
return c.json({ error: "soft-block: no ads returned (refunded)" }, 503);
|
|
16812
17157
|
}
|
|
16813
17158
|
const match = assessPageIntelMatch(body, result);
|
|
16814
17159
|
const output = { ...result, ...match };
|
|
16815
|
-
await logRequestEvent({ userId: fbUser.id, source: "facebook_page_intel", status: "done", query:
|
|
17160
|
+
await logRequestEvent({ userId: fbUser.id, source: "facebook_page_intel", status: "done", query: subject, resultCount: result.ads.length, result: output });
|
|
16816
17161
|
return c.json(output);
|
|
16817
17162
|
} catch (err) {
|
|
16818
17163
|
const msg = err instanceof Error ? err.message : String(err);
|
|
16819
17164
|
if (debited && !refunded) await creditMc(fbUser.id, MC_COSTS.fb_ad, LedgerOperation.FB_AD_REFUND, "failed call");
|
|
16820
|
-
await logRequestEvent({ userId: fbUser.id, source: "facebook_page_intel", status: "failed", query:
|
|
17165
|
+
await logRequestEvent({ userId: fbUser.id, source: "facebook_page_intel", status: "failed", query: subject, error: msg });
|
|
16821
17166
|
if (msg.toLowerCase().includes("blocked") || msg.toLowerCase().includes("captcha")) {
|
|
16822
17167
|
return c.json({ error: msg }, 503);
|
|
16823
17168
|
}
|
|
@@ -22567,21 +22912,33 @@ function formatCaptureSerpSnapshot(raw, input) {
|
|
|
22567
22912
|
const parsed = parseData(raw);
|
|
22568
22913
|
if ("error" in parsed) return { content: [{ type: "text", text: parsed.error }], isError: true };
|
|
22569
22914
|
const d = parsed.data;
|
|
22915
|
+
const harvestResult = d.harvestResult && typeof d.harvestResult === "object" ? d.harvestResult : d;
|
|
22570
22916
|
const providerPayload = d;
|
|
22571
|
-
const organic = Array.isArray(
|
|
22572
|
-
const localPack = Array.isArray(
|
|
22573
|
-
const artifacts = Array.isArray(d.artifacts) ? d.artifacts : [];
|
|
22574
|
-
const
|
|
22575
|
-
const
|
|
22576
|
-
const
|
|
22577
|
-
const
|
|
22917
|
+
const organic = Array.isArray(harvestResult.organicResults) ? harvestResult.organicResults : Array.isArray(harvestResult.organic) ? harvestResult.organic : [];
|
|
22918
|
+
const localPack = Array.isArray(harvestResult.localPack) ? harvestResult.localPack : [];
|
|
22919
|
+
const artifacts = Array.isArray(d.pageSnapshotArtifacts) ? d.pageSnapshotArtifacts : Array.isArray(d.artifacts) ? d.artifacts : [];
|
|
22920
|
+
const attempts = Array.isArray(d.attempts) ? d.attempts : [];
|
|
22921
|
+
const diagnostics = harvestResult.diagnostics && typeof harvestResult.diagnostics === "object" ? harvestResult.diagnostics : d.diagnostics && typeof d.diagnostics === "object" ? d.diagnostics : null;
|
|
22922
|
+
const completionStatus = diagnostics && typeof diagnostics.completionStatus === "string" ? diagnostics.completionStatus : null;
|
|
22923
|
+
const status = String(d.status ?? d.captureStatus ?? completionStatus ?? "captured");
|
|
22924
|
+
const query = typeof harvestResult.seed === "string" ? harvestResult.seed : typeof harvestResult.query === "string" ? harvestResult.query : typeof input.query === "string" ? input.query : null;
|
|
22925
|
+
const location2 = typeof harvestResult.location === "string" ? harvestResult.location : typeof input.location === "string" ? input.location : null;
|
|
22926
|
+
const capturedAt = typeof harvestResult.extractedAt === "string" ? harvestResult.extractedAt : typeof harvestResult.capturedAt === "string" ? harvestResult.capturedAt : null;
|
|
22578
22927
|
const resultCount = organic.length + localPack.length;
|
|
22928
|
+
const pageSnapshotFailedCount = artifacts.filter((artifact) => artifact.status === "failed" || Boolean(artifact.error)).length;
|
|
22929
|
+
const rawBilling = d.billing && typeof d.billing === "object" ? d.billing : null;
|
|
22930
|
+
const billing = rawBilling ? {
|
|
22931
|
+
creditsUsed: typeof rawBilling.creditsUsed === "number" ? rawBilling.creditsUsed : null,
|
|
22932
|
+
requestId: typeof rawBilling.requestId === "string" ? rawBilling.requestId : null,
|
|
22933
|
+
jobId: typeof rawBilling.jobId === "string" ? rawBilling.jobId : null
|
|
22934
|
+
} : null;
|
|
22579
22935
|
const full = [
|
|
22580
22936
|
`# SERP Intelligence Snapshot: ${query ?? "query"}`,
|
|
22581
22937
|
`**Status:** ${status}`,
|
|
22582
22938
|
location2 ? `**Location:** ${location2}` : "",
|
|
22583
|
-
`**
|
|
22584
|
-
|
|
22939
|
+
`**SERP results:** ${resultCount} (${organic.length} organic, ${localPack.length} local pack)`,
|
|
22940
|
+
`**Page snapshots:** ${artifacts.length} (${pageSnapshotFailedCount} failed)`,
|
|
22941
|
+
billing?.creditsUsed !== null && billing?.creditsUsed !== void 0 ? `**Credits used:** ${billing.creditsUsed}` : "",
|
|
22585
22942
|
"",
|
|
22586
22943
|
"Use `capture_serp_page_snapshots` when you need page-level evidence for ranking URLs."
|
|
22587
22944
|
].filter(Boolean).join("\n");
|
|
@@ -22594,6 +22951,10 @@ function formatCaptureSerpSnapshot(raw, input) {
|
|
|
22594
22951
|
location: location2,
|
|
22595
22952
|
capturedAt,
|
|
22596
22953
|
resultCount,
|
|
22954
|
+
organicResultCount: organic.length,
|
|
22955
|
+
localPackResultCount: localPack.length,
|
|
22956
|
+
pageSnapshotCount: artifacts.length,
|
|
22957
|
+
pageSnapshotFailedCount,
|
|
22597
22958
|
snapshotId: typeof d.snapshotId === "string" ? d.snapshotId : typeof d.snapshot_id === "string" ? d.snapshot_id : typeof d.id === "string" ? d.id : null,
|
|
22598
22959
|
resolvedInputs: {
|
|
22599
22960
|
query: input.query ?? null,
|
|
@@ -22606,7 +22967,9 @@ function formatCaptureSerpSnapshot(raw, input) {
|
|
|
22606
22967
|
pages: input.pages ?? null
|
|
22607
22968
|
},
|
|
22608
22969
|
artifacts,
|
|
22609
|
-
|
|
22970
|
+
attempts,
|
|
22971
|
+
billing,
|
|
22972
|
+
diagnostics,
|
|
22610
22973
|
providerPayload
|
|
22611
22974
|
}
|
|
22612
22975
|
};
|
|
@@ -22615,9 +22978,13 @@ function formatCaptureSerpPageSnapshots(raw, input) {
|
|
|
22615
22978
|
const parsed = parseData(raw);
|
|
22616
22979
|
if ("error" in parsed) return { content: [{ type: "text", text: parsed.error }], isError: true };
|
|
22617
22980
|
const d = parsed.data;
|
|
22618
|
-
const captures = Array.isArray(d.captures) ? d.captures : Array.isArray(d.results) ? d.results : [];
|
|
22981
|
+
const captures = Array.isArray(d.pageSnapshotArtifacts) ? d.pageSnapshotArtifacts : Array.isArray(d.captures) ? d.captures : Array.isArray(d.results) ? d.results : [];
|
|
22619
22982
|
const failedCount = captures.filter((c) => c.status === "failed" || c.error).length;
|
|
22620
|
-
const
|
|
22983
|
+
const capturedCount = captures.filter((c) => c.status === "captured" || c.status === "ok").length;
|
|
22984
|
+
const skippedCount = captures.filter((c) => c.status === "skipped").length;
|
|
22985
|
+
const attempts = Array.isArray(d.attempts) ? d.attempts : [];
|
|
22986
|
+
const inferredStatus = captures.length === 0 ? "empty" : failedCount === captures.length ? "failed" : failedCount > 0 || skippedCount > 0 ? "partial" : "captured";
|
|
22987
|
+
const status = String(d.status ?? inferredStatus);
|
|
22621
22988
|
const rows = captures.slice(0, 25).map((capture, i) => {
|
|
22622
22989
|
const url = typeof capture.url === "string" ? capture.url : "";
|
|
22623
22990
|
const sourceKind = typeof capture.sourceKind === "string" ? capture.sourceKind : typeof capture.source_kind === "string" ? capture.source_kind : "configured_target";
|
|
@@ -22626,8 +22993,10 @@ function formatCaptureSerpPageSnapshots(raw, input) {
|
|
|
22626
22993
|
const full = [
|
|
22627
22994
|
"# SERP Intelligence Page Snapshots",
|
|
22628
22995
|
`**Status:** ${status}`,
|
|
22629
|
-
`**
|
|
22996
|
+
`**Attempted:** ${captures.length}`,
|
|
22997
|
+
`**Captured:** ${capturedCount}`,
|
|
22630
22998
|
`**Failed:** ${failedCount}`,
|
|
22999
|
+
skippedCount ? `**Skipped:** ${skippedCount}` : "",
|
|
22631
23000
|
captures.length ? `
|
|
22632
23001
|
| # | URL | Source | Status |
|
|
22633
23002
|
|---|-----|--------|--------|
|
|
@@ -22639,8 +23008,11 @@ ${rows}` : ""
|
|
|
22639
23008
|
schemaVersion: "serp-intelligence.page-snapshots.v1",
|
|
22640
23009
|
status,
|
|
22641
23010
|
count: captures.length,
|
|
23011
|
+
capturedCount,
|
|
22642
23012
|
failedCount,
|
|
23013
|
+
skippedCount,
|
|
22643
23014
|
captures,
|
|
23015
|
+
attempts,
|
|
22644
23016
|
resolvedInputs: {
|
|
22645
23017
|
urls: input.urls ?? null,
|
|
22646
23018
|
targets: input.targets ?? null,
|
|
@@ -28180,12 +28552,22 @@ var init_schemas4 = __esm({
|
|
|
28180
28552
|
sourcePosition: import_zod33.z.number().int().min(1).optional()
|
|
28181
28553
|
}).strict();
|
|
28182
28554
|
SerpIntelligencePageSnapshotsBodySchema = import_zod33.z.object({
|
|
28183
|
-
urls: import_zod33.z.array(SerpIntelligencePublicHttpUrlSchema).min(1).max(25),
|
|
28555
|
+
urls: import_zod33.z.array(SerpIntelligencePublicHttpUrlSchema).min(1).max(25).optional(),
|
|
28184
28556
|
targets: import_zod33.z.array(SerpIntelligencePageSnapshotRequestSchema).min(1).max(25).optional(),
|
|
28185
28557
|
maxConcurrency: import_zod33.z.number().int().min(1).max(5).default(2),
|
|
28186
28558
|
timeoutMs: import_zod33.z.number().int().min(1e3).max(6e4).default(15e3),
|
|
28187
28559
|
debug: import_zod33.z.boolean().default(false)
|
|
28188
|
-
}).strict()
|
|
28560
|
+
}).strict().superRefine((body, ctx) => {
|
|
28561
|
+
const hasUrls = Boolean(body.urls?.length);
|
|
28562
|
+
const hasTargets = Boolean(body.targets?.length);
|
|
28563
|
+
if (hasUrls === hasTargets) {
|
|
28564
|
+
ctx.addIssue({
|
|
28565
|
+
code: import_zod33.z.ZodIssueCode.custom,
|
|
28566
|
+
message: "provide exactly one of urls or targets",
|
|
28567
|
+
path: hasUrls ? ["targets"] : ["urls"]
|
|
28568
|
+
});
|
|
28569
|
+
}
|
|
28570
|
+
});
|
|
28189
28571
|
SerpIntelligenceAICitationSchema = import_zod33.z.object({
|
|
28190
28572
|
text: import_zod33.z.string(),
|
|
28191
28573
|
href: import_zod33.z.string()
|
|
@@ -28445,14 +28827,24 @@ async function captureSerpIntelligenceSnapshot(rawInput, runtimeOptions = {}) {
|
|
|
28445
28827
|
});
|
|
28446
28828
|
const pageSnapshotLimit = normalizePageSnapshotLimit(parsedInput);
|
|
28447
28829
|
const pageSnapshotTargets = collectPageSnapshotTargets(harvestResult, pageSnapshotLimit);
|
|
28448
|
-
const pageSnapshotArtifacts = pageSnapshotTargets.length > 0 ? (await
|
|
28449
|
-
|
|
28450
|
-
|
|
28451
|
-
|
|
28452
|
-
|
|
28453
|
-
|
|
28454
|
-
|
|
28455
|
-
|
|
28830
|
+
const pageSnapshotArtifacts = pageSnapshotTargets.length > 0 ? (await runWithCostContext(
|
|
28831
|
+
{
|
|
28832
|
+
...currentCostContext() ?? { op: "serp" },
|
|
28833
|
+
subOp: "page_snapshot",
|
|
28834
|
+
// Page snapshot launches have their own fixed product rate. They
|
|
28835
|
+
// must not overwrite the headless/headful mode used to settle the
|
|
28836
|
+
// preceding SERP harvest.
|
|
28837
|
+
headlessSentOut: void 0
|
|
28838
|
+
},
|
|
28839
|
+
() => capturePageSnapshotsFn(pageSnapshotTargets, {
|
|
28840
|
+
kernelApiKey: runtimeOptions.kernelApiKey ?? browserServiceApiKey(),
|
|
28841
|
+
timeoutMs: runtimeOptions.pageSnapshotTimeoutMs,
|
|
28842
|
+
maxConcurrency: runtimeOptions.pageSnapshotMaxConcurrency,
|
|
28843
|
+
debug,
|
|
28844
|
+
includeContent: runtimeOptions.includePageSnapshotContent,
|
|
28845
|
+
maxContentChars: runtimeOptions.maxPageSnapshotContentChars
|
|
28846
|
+
})
|
|
28847
|
+
)).pageSnapshotArtifacts.map(stripPageSnapshotArtifact) : [];
|
|
28456
28848
|
return normalizeHarvestResultForSerpIntelligence({
|
|
28457
28849
|
harvestResult,
|
|
28458
28850
|
attempts,
|
|
@@ -28469,6 +28861,7 @@ var init_serp_capture_service = __esm({
|
|
|
28469
28861
|
"use strict";
|
|
28470
28862
|
init_harvest();
|
|
28471
28863
|
init_browser_service_env();
|
|
28864
|
+
init_cost_context();
|
|
28472
28865
|
init_harvest_problems();
|
|
28473
28866
|
init_page_snapshot_extractor();
|
|
28474
28867
|
init_schemas4();
|
|
@@ -28542,17 +28935,35 @@ async function enforceSerpIntelligenceRateLimit(c, userId) {
|
|
|
28542
28935
|
);
|
|
28543
28936
|
}
|
|
28544
28937
|
function pageSnapshotTargetsFromBody(body) {
|
|
28545
|
-
return body.targets?.length ? body.targets : body.urls.map((url) => ({
|
|
28938
|
+
return body.targets?.length ? body.targets : (body.urls ?? []).map((url) => ({
|
|
28546
28939
|
url,
|
|
28547
28940
|
sourceKind: "configured_target",
|
|
28548
28941
|
sourcePosition: null
|
|
28549
28942
|
}));
|
|
28550
28943
|
}
|
|
28551
|
-
|
|
28944
|
+
async function logRequestEventBestEffort(input) {
|
|
28945
|
+
try {
|
|
28946
|
+
await logRequestEvent(input);
|
|
28947
|
+
} catch (error) {
|
|
28948
|
+
console.warn("[serp-intelligence] request audit log failed:", error instanceof Error ? error.message : String(error));
|
|
28949
|
+
}
|
|
28950
|
+
}
|
|
28951
|
+
function captureBillingKeys(userId, suppliedKey, body) {
|
|
28952
|
+
const responseKey = suppliedKey?.trim() || (0, import_node_crypto12.randomUUID)();
|
|
28953
|
+
const requestFingerprint = (0, import_node_crypto12.createHash)("sha256").update(JSON.stringify(body)).digest("hex");
|
|
28954
|
+
const keyDigest = (0, import_node_crypto12.createHash)("sha256").update(String(userId)).update("\0").update(responseKey).digest("hex");
|
|
28955
|
+
return {
|
|
28956
|
+
responseKey,
|
|
28957
|
+
debitKey: `serp-capture:${userId}:${keyDigest}`,
|
|
28958
|
+
debitDescription: `${body.query} [request:${requestFingerprint}]`
|
|
28959
|
+
};
|
|
28960
|
+
}
|
|
28961
|
+
var import_hono16, import_node_crypto12, SERP_INTELLIGENCE_RATE_LIMIT, SERP_INTELLIGENCE_RATE_WINDOW_SECONDS, POST_CAPTURE_ROUTE_LABEL, POST_PAGE_SNAPSHOTS_ROUTE_LABEL, SERP_CAPTURE_BILLING_SOURCE, serpIntelligenceApp;
|
|
28552
28962
|
var init_serp_intelligence_routes = __esm({
|
|
28553
28963
|
"src/api/serp-intelligence-routes.ts"() {
|
|
28554
28964
|
"use strict";
|
|
28555
28965
|
import_hono16 = require("hono");
|
|
28966
|
+
import_node_crypto12 = require("crypto");
|
|
28556
28967
|
init_browser_service_env();
|
|
28557
28968
|
init_page_snapshot_extractor();
|
|
28558
28969
|
init_serp_capture_service();
|
|
@@ -28560,11 +28971,13 @@ var init_serp_intelligence_routes = __esm({
|
|
|
28560
28971
|
init_api_auth();
|
|
28561
28972
|
init_db();
|
|
28562
28973
|
init_rates();
|
|
28974
|
+
init_cost_context();
|
|
28563
28975
|
init_concurrency_gates();
|
|
28564
28976
|
SERP_INTELLIGENCE_RATE_LIMIT = 60;
|
|
28565
28977
|
SERP_INTELLIGENCE_RATE_WINDOW_SECONDS = 60;
|
|
28566
28978
|
POST_CAPTURE_ROUTE_LABEL = "POST /capture";
|
|
28567
28979
|
POST_PAGE_SNAPSHOTS_ROUTE_LABEL = "POST /page-snapshots";
|
|
28980
|
+
SERP_CAPTURE_BILLING_SOURCE = "serp_intelligence_capture";
|
|
28568
28981
|
serpIntelligenceApp = new import_hono16.Hono();
|
|
28569
28982
|
serpIntelligenceApp.use("*", createApiKeyAuth());
|
|
28570
28983
|
serpIntelligenceApp.post("/capture", async (c) => {
|
|
@@ -28573,6 +28986,17 @@ var init_serp_intelligence_routes = __esm({
|
|
|
28573
28986
|
const raw = await c.req.json().catch(() => ({}));
|
|
28574
28987
|
const parsed = SerpIntelligenceCaptureBodySchema.safeParse(raw);
|
|
28575
28988
|
if (!parsed.success) return c.json(formatZodError(parsed.error), 400);
|
|
28989
|
+
const suppliedIdempotencyKey = c.req.header("idempotency-key") ?? c.req.header("x-idempotency-key");
|
|
28990
|
+
if (suppliedIdempotencyKey && suppliedIdempotencyKey.trim().length > 500) {
|
|
28991
|
+
return c.json(structuredError({
|
|
28992
|
+
error_code: "invalid_request",
|
|
28993
|
+
error_type: "validation_error",
|
|
28994
|
+
message: "Idempotency-Key must be 500 characters or fewer.",
|
|
28995
|
+
retryable: false
|
|
28996
|
+
}), 400);
|
|
28997
|
+
}
|
|
28998
|
+
const billingKeys = captureBillingKeys(user.id, suppliedIdempotencyKey, parsed.data);
|
|
28999
|
+
c.header("Idempotency-Key", billingKeys.responseKey);
|
|
28576
29000
|
const limited = await enforceSerpIntelligenceRateLimit(c, user.id);
|
|
28577
29001
|
if (limited) return limited;
|
|
28578
29002
|
const gate = await acquireConcurrencyGate(user, "serp_intelligence_capture", {
|
|
@@ -28580,51 +29004,130 @@ var init_serp_intelligence_routes = __esm({
|
|
|
28580
29004
|
metadata: { query: parsed.data.query, location: parsed.data.location }
|
|
28581
29005
|
});
|
|
28582
29006
|
if (!gate.ok) return c.json(concurrencyLimitExceededResponse(gate), 429, { "Retry-After": String(gate.retryAfterSeconds) });
|
|
28583
|
-
const
|
|
28584
|
-
|
|
29007
|
+
const pageSnapshotHold = parsed.data.includePageSnapshots ? parsed.data.pageSnapshotLimit * MC_COSTS.page_scrape : 0;
|
|
29008
|
+
const cost = MC_COSTS.serp_headful + pageSnapshotHold;
|
|
29009
|
+
const headlessSentOut = { value: null };
|
|
28585
29010
|
try {
|
|
28586
|
-
const { ok, balance_mc } = await
|
|
29011
|
+
const { ok, balance_mc } = await debitMcIdempotent(
|
|
28587
29012
|
user.id,
|
|
28588
29013
|
cost,
|
|
28589
29014
|
LedgerOperation.SERP,
|
|
28590
|
-
|
|
29015
|
+
billingKeys.debitDescription,
|
|
29016
|
+
billingKeys.debitKey
|
|
28591
29017
|
);
|
|
28592
|
-
if (!ok)
|
|
28593
|
-
|
|
28594
|
-
|
|
28595
|
-
|
|
28596
|
-
|
|
28597
|
-
|
|
28598
|
-
|
|
29018
|
+
if (!ok) {
|
|
29019
|
+
await releaseConcurrencyGate(gate.lockId);
|
|
29020
|
+
return c.json(insufficientBalanceResponse(balance_mc, cost), 402);
|
|
29021
|
+
}
|
|
29022
|
+
} catch (error) {
|
|
29023
|
+
const idempotencyConflict = error instanceof Error && error.message === "idempotent debit key conflicts with another debit";
|
|
29024
|
+
const body = structuredError({
|
|
29025
|
+
error_code: idempotencyConflict ? "idempotency_conflict" : "capture_failed",
|
|
29026
|
+
error_type: idempotencyConflict ? "validation_error" : "capture_error",
|
|
29027
|
+
message: error instanceof Error ? error.message : String(error),
|
|
29028
|
+
retryable: !idempotencyConflict
|
|
28599
29029
|
});
|
|
28600
|
-
await
|
|
29030
|
+
await logRequestEventBestEffort({
|
|
28601
29031
|
userId: user.id,
|
|
28602
|
-
source:
|
|
28603
|
-
status: "
|
|
29032
|
+
source: SERP_CAPTURE_BILLING_SOURCE,
|
|
29033
|
+
status: "failed",
|
|
28604
29034
|
query: parsed.data.query,
|
|
28605
29035
|
location: parsed.data.location,
|
|
28606
|
-
|
|
28607
|
-
result
|
|
29036
|
+
error: body.message
|
|
28608
29037
|
});
|
|
28609
|
-
|
|
29038
|
+
await releaseConcurrencyGate(gate.lockId);
|
|
29039
|
+
return c.json(body, idempotencyConflict ? 409 : 500);
|
|
29040
|
+
}
|
|
29041
|
+
let result;
|
|
29042
|
+
try {
|
|
29043
|
+
const parentCostContext = currentCostContext();
|
|
29044
|
+
result = await runWithCostContext(
|
|
29045
|
+
{
|
|
29046
|
+
...parentCostContext ?? {},
|
|
29047
|
+
op: "serp",
|
|
29048
|
+
userId: Number(user.id),
|
|
29049
|
+
headlessSentOut
|
|
29050
|
+
},
|
|
29051
|
+
() => captureSerpIntelligenceSnapshot(parsed.data, {
|
|
29052
|
+
kernelApiKey: browserServiceApiKey(),
|
|
29053
|
+
kernelProxyId: browserServiceProxyId(),
|
|
29054
|
+
signal: c.req.raw.signal
|
|
29055
|
+
})
|
|
29056
|
+
);
|
|
28610
29057
|
} catch (error) {
|
|
28611
|
-
|
|
29058
|
+
try {
|
|
29059
|
+
await settleDebitMcIdempotent(
|
|
29060
|
+
user.id,
|
|
29061
|
+
billingKeys.debitKey,
|
|
29062
|
+
0,
|
|
29063
|
+
LedgerOperation.SERP_REFUND,
|
|
29064
|
+
"failed serp intelligence capture",
|
|
29065
|
+
SERP_CAPTURE_BILLING_SOURCE
|
|
29066
|
+
);
|
|
29067
|
+
} catch (settlementError) {
|
|
29068
|
+
console.warn(
|
|
29069
|
+
"[serp-intelligence] failed capture refund remains recoverable:",
|
|
29070
|
+
settlementError instanceof Error ? settlementError.message : String(settlementError)
|
|
29071
|
+
);
|
|
29072
|
+
}
|
|
28612
29073
|
const body = error instanceof SerpIntelligenceCaptureError ? error.toJSON() : structuredError({
|
|
28613
29074
|
error_code: "capture_failed",
|
|
28614
29075
|
error_type: "capture_error",
|
|
28615
29076
|
message: error instanceof Error ? error.message : String(error),
|
|
28616
29077
|
retryable: true
|
|
28617
29078
|
});
|
|
28618
|
-
await
|
|
29079
|
+
await logRequestEventBestEffort({
|
|
28619
29080
|
userId: user.id,
|
|
28620
|
-
source:
|
|
29081
|
+
source: SERP_CAPTURE_BILLING_SOURCE,
|
|
28621
29082
|
status: "failed",
|
|
28622
29083
|
query: parsed.data.query,
|
|
28623
29084
|
location: parsed.data.location,
|
|
28624
29085
|
error: "message" in body && typeof body.message === "string" ? body.message : "SERP Intelligence capture failed"
|
|
28625
29086
|
});
|
|
28626
29087
|
const status = error instanceof SerpIntelligenceCaptureError ? error.httpStatus : 500;
|
|
29088
|
+
await releaseConcurrencyGate(gate.lockId);
|
|
28627
29089
|
return c.json(body, status);
|
|
29090
|
+
}
|
|
29091
|
+
try {
|
|
29092
|
+
const requestedActualCost = serpActualCostMc(headlessSentOut.value) + result.pageSnapshotArtifacts.length * MC_COSTS.page_scrape;
|
|
29093
|
+
const settlement = await settleDebitMcIdempotent(
|
|
29094
|
+
user.id,
|
|
29095
|
+
billingKeys.debitKey,
|
|
29096
|
+
requestedActualCost,
|
|
29097
|
+
LedgerOperation.SERP_REFUND,
|
|
29098
|
+
"SERP capture pricing settle",
|
|
29099
|
+
SERP_CAPTURE_BILLING_SOURCE
|
|
29100
|
+
);
|
|
29101
|
+
const settledResult = {
|
|
29102
|
+
...result,
|
|
29103
|
+
billing: { creditsUsed: settlement.final_amount_mc / MC_PER_CREDIT }
|
|
29104
|
+
};
|
|
29105
|
+
await logRequestEventBestEffort({
|
|
29106
|
+
userId: user.id,
|
|
29107
|
+
source: SERP_CAPTURE_BILLING_SOURCE,
|
|
29108
|
+
status: "done",
|
|
29109
|
+
query: parsed.data.query,
|
|
29110
|
+
location: parsed.data.location,
|
|
29111
|
+
resultCount: settledResult.harvestResult.organicResults.length,
|
|
29112
|
+
result: settledResult
|
|
29113
|
+
});
|
|
29114
|
+
return c.json(settledResult);
|
|
29115
|
+
} catch (error) {
|
|
29116
|
+
const body = structuredError({
|
|
29117
|
+
error_code: "capture_failed",
|
|
29118
|
+
error_type: "capture_error",
|
|
29119
|
+
message: error instanceof Error ? error.message : String(error),
|
|
29120
|
+
retryable: true
|
|
29121
|
+
});
|
|
29122
|
+
await logRequestEventBestEffort({
|
|
29123
|
+
userId: user.id,
|
|
29124
|
+
source: SERP_CAPTURE_BILLING_SOURCE,
|
|
29125
|
+
status: "failed",
|
|
29126
|
+
query: parsed.data.query,
|
|
29127
|
+
location: parsed.data.location,
|
|
29128
|
+
error: body.message
|
|
29129
|
+
});
|
|
29130
|
+
return c.json(body, 500);
|
|
28628
29131
|
} finally {
|
|
28629
29132
|
await releaseConcurrencyGate(gate.lockId);
|
|
28630
29133
|
}
|
|
@@ -28660,7 +29163,7 @@ var init_serp_intelligence_routes = __esm({
|
|
|
28660
29163
|
maxConcurrency: parsed.data.maxConcurrency,
|
|
28661
29164
|
debug: parsed.data.debug
|
|
28662
29165
|
});
|
|
28663
|
-
await
|
|
29166
|
+
await logRequestEventBestEffort({
|
|
28664
29167
|
userId: user.id,
|
|
28665
29168
|
source: "serp_intelligence_page_snapshots",
|
|
28666
29169
|
status: "done",
|
|
@@ -28677,7 +29180,7 @@ var init_serp_intelligence_routes = __esm({
|
|
|
28677
29180
|
message: error instanceof Error ? error.message : String(error),
|
|
28678
29181
|
retryable: true
|
|
28679
29182
|
});
|
|
28680
|
-
await
|
|
29183
|
+
await logRequestEventBestEffort({
|
|
28681
29184
|
userId: user.id,
|
|
28682
29185
|
source: "serp_intelligence_page_snapshots",
|
|
28683
29186
|
status: "failed",
|
|
@@ -28697,7 +29200,7 @@ var PACKAGE_VERSION;
|
|
|
28697
29200
|
var init_version = __esm({
|
|
28698
29201
|
"src/version.ts"() {
|
|
28699
29202
|
"use strict";
|
|
28700
|
-
PACKAGE_VERSION = "0.21.
|
|
29203
|
+
PACKAGE_VERSION = "0.21.4";
|
|
28701
29204
|
}
|
|
28702
29205
|
});
|
|
28703
29206
|
|
|
@@ -29983,9 +30486,19 @@ var init_mcp_tool_schemas = __esm({
|
|
|
29983
30486
|
location: NullableString,
|
|
29984
30487
|
capturedAt: NullableString,
|
|
29985
30488
|
resultCount: import_zod34.z.number().int().min(0).nullable(),
|
|
30489
|
+
organicResultCount: import_zod34.z.number().int().min(0),
|
|
30490
|
+
localPackResultCount: import_zod34.z.number().int().min(0),
|
|
30491
|
+
pageSnapshotCount: import_zod34.z.number().int().min(0),
|
|
30492
|
+
pageSnapshotFailedCount: import_zod34.z.number().int().min(0),
|
|
29986
30493
|
snapshotId: NullableString,
|
|
29987
30494
|
resolvedInputs: import_zod34.z.record(import_zod34.z.unknown()),
|
|
29988
30495
|
artifacts: import_zod34.z.array(import_zod34.z.record(import_zod34.z.unknown())),
|
|
30496
|
+
attempts: import_zod34.z.array(import_zod34.z.record(import_zod34.z.unknown())),
|
|
30497
|
+
billing: import_zod34.z.object({
|
|
30498
|
+
creditsUsed: import_zod34.z.number().min(0).nullable(),
|
|
30499
|
+
requestId: NullableString,
|
|
30500
|
+
jobId: NullableString
|
|
30501
|
+
}).nullable(),
|
|
29989
30502
|
diagnostics: import_zod34.z.record(import_zod34.z.unknown()).nullable(),
|
|
29990
30503
|
providerPayload: import_zod34.z.record(import_zod34.z.unknown())
|
|
29991
30504
|
};
|
|
@@ -29993,8 +30506,11 @@ var init_mcp_tool_schemas = __esm({
|
|
|
29993
30506
|
schemaVersion: import_zod34.z.literal("serp-intelligence.page-snapshots.v1"),
|
|
29994
30507
|
status: import_zod34.z.string(),
|
|
29995
30508
|
count: import_zod34.z.number().int().min(0),
|
|
30509
|
+
capturedCount: import_zod34.z.number().int().min(0),
|
|
29996
30510
|
failedCount: import_zod34.z.number().int().min(0),
|
|
30511
|
+
skippedCount: import_zod34.z.number().int().min(0),
|
|
29997
30512
|
captures: import_zod34.z.array(import_zod34.z.record(import_zod34.z.unknown())),
|
|
30513
|
+
attempts: import_zod34.z.array(import_zod34.z.record(import_zod34.z.unknown())),
|
|
29998
30514
|
resolvedInputs: import_zod34.z.record(import_zod34.z.unknown()),
|
|
29999
30515
|
diagnostics: import_zod34.z.record(import_zod34.z.unknown()).nullable(),
|
|
30000
30516
|
providerPayload: import_zod34.z.record(import_zod34.z.unknown())
|
|
@@ -30119,8 +30635,8 @@ var init_mcp_tool_schemas = __esm({
|
|
|
30119
30635
|
proxyZip: import_zod34.z.string().regex(/^\d{5}$/).optional().describe('US ZIP for residential geo-IP targeting. Only meaningful with proxyMode "location".'),
|
|
30120
30636
|
pages: import_zod34.z.number().int().min(1).max(2).default(1).describe("Google result pages to capture. Use 2 only for deeper ranking evidence."),
|
|
30121
30637
|
debug: import_zod34.z.boolean().default(false).describe("Include sanitized browser/proxy/location diagnostics."),
|
|
30122
|
-
includePageSnapshots: import_zod34.z.boolean().default(false).describe("Also capture ranking-page snapshots for selected SERP URLs."),
|
|
30123
|
-
pageSnapshotLimit: import_zod34.z.number().int().min(0).max(10).default(0).describe("Maximum ranking-page snapshots when includePageSnapshots is true.")
|
|
30638
|
+
includePageSnapshots: import_zod34.z.boolean().default(false).describe("Also capture ranking-page snapshots for selected SERP URLs. Each attempted snapshot adds 1 Credit."),
|
|
30639
|
+
pageSnapshotLimit: import_zod34.z.number().int().min(0).max(10).default(0).describe("Maximum ranking-page snapshots when includePageSnapshots is true. This capacity is held up front and unused capacity is refunded.")
|
|
30124
30640
|
};
|
|
30125
30641
|
ScreenshotInputSchema = {
|
|
30126
30642
|
url: import_zod34.z.string().url().describe("URL to capture as a full-page screenshot. Use http or https. Pass allowLocal: true to capture localhost or private-network URLs during development."),
|
|
@@ -30128,12 +30644,12 @@ var init_mcp_tool_schemas = __esm({
|
|
|
30128
30644
|
allowLocal: import_zod34.z.boolean().default(false).describe("Allow localhost and private-network URLs (127.x, 192.168.x, 10.x, etc.). For local development only \u2014 not for production use.")
|
|
30129
30645
|
};
|
|
30130
30646
|
CaptureSerpPageSnapshotsInputSchema = {
|
|
30131
|
-
urls: import_zod34.z.array(import_zod34.z.string().url()).min(1).max(25).describe("Public HTTP/HTTPS URLs to capture. Do not pass localhost, private IPs, file URLs, or internal admin URLs."),
|
|
30647
|
+
urls: import_zod34.z.array(import_zod34.z.string().url()).min(1).max(25).optional().describe("Public HTTP/HTTPS URLs to capture. Provide exactly one of urls or targets. Do not pass localhost, private IPs, file URLs, or internal admin URLs."),
|
|
30132
30648
|
targets: import_zod34.z.array(import_zod34.z.object({
|
|
30133
30649
|
url: import_zod34.z.string().url().describe("Public HTTP/HTTPS URL to capture."),
|
|
30134
30650
|
sourceKind: import_zod34.z.enum(["organic", "ai_citation", "local_pack_website", "configured_target", "site_subject"]).default("configured_target").describe("Why this page is being captured."),
|
|
30135
30651
|
sourcePosition: import_zod34.z.number().int().min(1).optional().describe("Ranking or citation position when the page came from SERP evidence.")
|
|
30136
|
-
}).strict()).min(1).max(25).optional().describe("Structured targets.
|
|
30652
|
+
}).strict()).min(1).max(25).optional().describe("Structured targets. Provide exactly one of targets or urls; use targets when source kind or position should be preserved."),
|
|
30137
30653
|
maxConcurrency: import_zod34.z.number().int().min(1).max(5).default(2).describe("Parallel page captures."),
|
|
30138
30654
|
timeoutMs: import_zod34.z.number().int().min(1e3).max(6e4).default(15e3).describe("Per-page capture timeout in milliseconds; timeouts return as structured capture failures."),
|
|
30139
30655
|
debug: import_zod34.z.boolean().default(false).describe("Include sanitized browser/proxy diagnostics.")
|
|
@@ -30769,7 +31285,7 @@ var init_rank_tracker_blueprint = __esm({
|
|
|
30769
31285
|
|
|
30770
31286
|
// src/mcp/paa-mcp-server.ts
|
|
30771
31287
|
function hashOwnerId(callerKey) {
|
|
30772
|
-
return (0,
|
|
31288
|
+
return (0, import_node_crypto13.createHash)("sha256").update(callerKey).digest("hex").slice(0, 24);
|
|
30773
31289
|
}
|
|
30774
31290
|
function liveWebToolAnnotations(title) {
|
|
30775
31291
|
return {
|
|
@@ -30783,14 +31299,14 @@ function liveWebToolAnnotations(title) {
|
|
|
30783
31299
|
function registerSerpIntelligenceCaptureTools(server, executor) {
|
|
30784
31300
|
server.registerTool("capture_serp_snapshot", {
|
|
30785
31301
|
title: "SERP Intelligence Snapshot",
|
|
30786
|
-
description: "Capture a structured SERP Intelligence snapshot of a Google query \u2014 the persistent evidence format used by rank-tracking and comparison pipelines. Split query from location; leave proxyMode unset.",
|
|
31302
|
+
description: "Capture a structured SERP Intelligence snapshot of a Google query \u2014 the persistent evidence format used by rank-tracking and comparison pipelines. Split query from location; leave proxyMode unset. Costs 4 Credits when headless or 14 if anti-bot escalation requires headful mode; the 14-Credit hold is settled to the mode used. Optional page snapshots add 1 Credit per attempted URL.",
|
|
30787
31303
|
inputSchema: CaptureSerpSnapshotInputSchema,
|
|
30788
31304
|
outputSchema: recordOutputSchema("capture_serp_snapshot", CaptureSerpSnapshotOutputSchema),
|
|
30789
31305
|
annotations: liveWebToolAnnotations("SERP Intelligence Snapshot")
|
|
30790
31306
|
}, async (input) => formatCaptureSerpSnapshot(await executor.captureSerpSnapshot(input), input));
|
|
30791
31307
|
server.registerTool("capture_serp_page_snapshots", {
|
|
30792
31308
|
title: "SERP Intelligence Page Snapshots",
|
|
30793
|
-
description: "Capture public ranking pages as SERP Intelligence page snapshots \u2014 persistent page evidence linked to a captured SERP. Provide urls
|
|
31309
|
+
description: "Capture public ranking pages as SERP Intelligence page snapshots \u2014 persistent page evidence linked to a captured SERP. Provide exactly one of urls or targets; use targets to preserve source metadata. Private IPs, localhost, file, and internal URLs are rejected. Costs 1 Credit per attempted URL.",
|
|
30794
31310
|
inputSchema: CaptureSerpPageSnapshotsInputSchema,
|
|
30795
31311
|
outputSchema: recordOutputSchema("capture_serp_page_snapshots", CaptureSerpPageSnapshotsOutputSchema),
|
|
30796
31312
|
annotations: liveWebToolAnnotations("SERP Intelligence Page Snapshots")
|
|
@@ -31213,14 +31729,14 @@ function registerPaaExtractorMcpTools(server, executor, options = {}) {
|
|
|
31213
31729
|
annotations: { title: "Set Scheduled Action Connections", readOnlyHint: false, destructiveHint: false, idempotentHint: true, openWorldHint: false }
|
|
31214
31730
|
}, async (input) => executor.setScheduledActionConnections(input));
|
|
31215
31731
|
}
|
|
31216
|
-
var import_mcp, import_node_fs9, import_node_path13,
|
|
31732
|
+
var import_mcp, import_node_fs9, import_node_path13, import_node_crypto13;
|
|
31217
31733
|
var init_paa_mcp_server = __esm({
|
|
31218
31734
|
"src/mcp/paa-mcp-server.ts"() {
|
|
31219
31735
|
"use strict";
|
|
31220
31736
|
import_mcp = require("@modelcontextprotocol/sdk/server/mcp.js");
|
|
31221
31737
|
import_node_fs9 = require("fs");
|
|
31222
31738
|
import_node_path13 = require("path");
|
|
31223
|
-
|
|
31739
|
+
import_node_crypto13 = require("crypto");
|
|
31224
31740
|
init_version();
|
|
31225
31741
|
init_mcp_response_formatter();
|
|
31226
31742
|
init_server_instructions();
|
|
@@ -31290,10 +31806,11 @@ function withDefaultHostedProfile(input) {
|
|
|
31290
31806
|
...typeof saveProfileChanges === "boolean" ? { saveProfileChanges } : {}
|
|
31291
31807
|
};
|
|
31292
31808
|
}
|
|
31293
|
-
var HttpMcpToolExecutor;
|
|
31809
|
+
var import_node_crypto14, HttpMcpToolExecutor;
|
|
31294
31810
|
var init_http_mcp_tool_executor = __esm({
|
|
31295
31811
|
"src/mcp/http-mcp-tool-executor.ts"() {
|
|
31296
31812
|
"use strict";
|
|
31813
|
+
import_node_crypto14 = require("crypto");
|
|
31297
31814
|
init_harvest_timeout();
|
|
31298
31815
|
init_browser_service_env();
|
|
31299
31816
|
HttpMcpToolExecutor = class {
|
|
@@ -31302,6 +31819,7 @@ var init_http_mcp_tool_executor = __esm({
|
|
|
31302
31819
|
timeoutMs;
|
|
31303
31820
|
httpTimeoutOverrideMs;
|
|
31304
31821
|
serpIntelligenceTimeoutMs;
|
|
31822
|
+
pendingSerpCaptureBillingKeys = /* @__PURE__ */ new Map();
|
|
31305
31823
|
constructor(baseUrl, apiKey) {
|
|
31306
31824
|
this.baseUrl = baseUrl.replace(/\/$/, "");
|
|
31307
31825
|
this.apiKey = apiKey;
|
|
@@ -31312,13 +31830,14 @@ var init_http_mcp_tool_executor = __esm({
|
|
|
31312
31830
|
const configuredSerpIntelligenceTimeoutMs = Number(process.env.MCP_SCRAPER_SERP_INTELLIGENCE_HTTP_TIMEOUT_MS ?? this.timeoutMs);
|
|
31313
31831
|
this.serpIntelligenceTimeoutMs = Number.isFinite(configuredSerpIntelligenceTimeoutMs) && configuredSerpIntelligenceTimeoutMs > 0 ? configuredSerpIntelligenceTimeoutMs : this.timeoutMs;
|
|
31314
31832
|
}
|
|
31315
|
-
async call(path6, body, timeoutMs = this.timeoutMs, method = "POST") {
|
|
31833
|
+
async call(path6, body, timeoutMs = this.timeoutMs, method = "POST", requestHeaders = {}) {
|
|
31316
31834
|
try {
|
|
31317
31835
|
const res = await fetch(`${this.baseUrl}${path6}`, {
|
|
31318
31836
|
method,
|
|
31319
31837
|
headers: {
|
|
31320
31838
|
"Content-Type": "application/json",
|
|
31321
|
-
"x-api-key": this.apiKey
|
|
31839
|
+
"x-api-key": this.apiKey,
|
|
31840
|
+
...requestHeaders
|
|
31322
31841
|
},
|
|
31323
31842
|
body: JSON.stringify(body),
|
|
31324
31843
|
signal: AbortSignal.timeout(timeoutMs)
|
|
@@ -31566,8 +32085,27 @@ var init_http_mcp_tool_executor = __esm({
|
|
|
31566
32085
|
connections: input.connections
|
|
31567
32086
|
}, void 0, "PUT");
|
|
31568
32087
|
}
|
|
31569
|
-
captureSerpSnapshot(input) {
|
|
31570
|
-
|
|
32088
|
+
async captureSerpSnapshot(input) {
|
|
32089
|
+
const fingerprint = (0, import_node_crypto14.createHash)("sha256").update(JSON.stringify(input)).digest("hex");
|
|
32090
|
+
const now = Date.now();
|
|
32091
|
+
for (const [pendingFingerprint, pendingEntry] of this.pendingSerpCaptureBillingKeys) {
|
|
32092
|
+
if (pendingEntry.expiresAt <= now) this.pendingSerpCaptureBillingKeys.delete(pendingFingerprint);
|
|
32093
|
+
}
|
|
32094
|
+
const pending = this.pendingSerpCaptureBillingKeys.get(fingerprint);
|
|
32095
|
+
const idempotencyKey = pending && pending.expiresAt > now ? pending.key : (0, import_node_crypto14.randomUUID)();
|
|
32096
|
+
this.pendingSerpCaptureBillingKeys.set(fingerprint, {
|
|
32097
|
+
key: idempotencyKey,
|
|
32098
|
+
expiresAt: now + 15 * 6e4
|
|
32099
|
+
});
|
|
32100
|
+
const result = await this.call(
|
|
32101
|
+
"/serp-intelligence/capture",
|
|
32102
|
+
input,
|
|
32103
|
+
this.serpIntelligenceTimeoutMs,
|
|
32104
|
+
"POST",
|
|
32105
|
+
{ "Idempotency-Key": idempotencyKey }
|
|
32106
|
+
);
|
|
32107
|
+
if (!result.isError) this.pendingSerpCaptureBillingKeys.delete(fingerprint);
|
|
32108
|
+
return result;
|
|
31571
32109
|
}
|
|
31572
32110
|
captureSerpPageSnapshots(input) {
|
|
31573
32111
|
return this.call("/serp-intelligence/page-snapshots", input, this.serpIntelligenceTimeoutMs);
|
|
@@ -36261,7 +36799,33 @@ var init_mcp_routes = __esm({
|
|
|
36261
36799
|
// src/api/browser-agent-db.ts
|
|
36262
36800
|
async function migrateBrowserAgent() {
|
|
36263
36801
|
if (_ready2) return;
|
|
36802
|
+
if (_migrationPromise2) return _migrationPromise2;
|
|
36803
|
+
_migrationPromise2 = runBrowserAgentMigration().finally(() => {
|
|
36804
|
+
_migrationPromise2 = null;
|
|
36805
|
+
});
|
|
36806
|
+
return _migrationPromise2;
|
|
36807
|
+
}
|
|
36808
|
+
async function runBrowserAgentMigration() {
|
|
36264
36809
|
const db = getDb();
|
|
36810
|
+
const existing = await db.execute(`
|
|
36811
|
+
SELECT
|
|
36812
|
+
(SELECT COUNT(*) FROM sqlite_master WHERE type = 'table' AND name IN (
|
|
36813
|
+
'browser_agent_sessions',
|
|
36814
|
+
'browser_agent_actions',
|
|
36815
|
+
'browser_agent_replays',
|
|
36816
|
+
'browser_auth_connections',
|
|
36817
|
+
'browser_agent_extensions'
|
|
36818
|
+
)) = 5
|
|
36819
|
+
AND (SELECT COUNT(*) FROM pragma_table_info('browser_agent_sessions') WHERE name = 'concurrency_lock_id') = 1
|
|
36820
|
+
AND (SELECT COUNT(*) FROM pragma_table_info('browser_auth_connections') WHERE name IN (
|
|
36821
|
+
'account_email', 'note', 'updated_at', 'status', 'browser_agent_session_id'
|
|
36822
|
+
)) = 5
|
|
36823
|
+
AS ready
|
|
36824
|
+
`);
|
|
36825
|
+
if (Number(existing.rows[0]?.ready ?? 0) === 1) {
|
|
36826
|
+
_ready2 = true;
|
|
36827
|
+
return;
|
|
36828
|
+
}
|
|
36265
36829
|
await db.execute(`
|
|
36266
36830
|
CREATE TABLE IF NOT EXISTS browser_agent_sessions (
|
|
36267
36831
|
id TEXT PRIMARY KEY,
|
|
@@ -36361,7 +36925,7 @@ async function migrateBrowserAgent() {
|
|
|
36361
36925
|
}
|
|
36362
36926
|
async function createExtensionRow(input) {
|
|
36363
36927
|
const db = getDb();
|
|
36364
|
-
const id = `bext_${(0,
|
|
36928
|
+
const id = `bext_${(0, import_node_crypto15.randomUUID)().replace(/-/g, "").slice(0, 20)}`;
|
|
36365
36929
|
await db.execute({
|
|
36366
36930
|
sql: `INSERT INTO browser_agent_extensions (id, user_id, name, backend_id, backend_name, source, source_url, size_bytes)
|
|
36367
36931
|
VALUES (?, ?, ?, ?, ?, ?, ?, ?)`,
|
|
@@ -36396,7 +36960,7 @@ async function deleteExtensionRow(userId, name) {
|
|
|
36396
36960
|
}
|
|
36397
36961
|
async function createAuthConnectionRow(input) {
|
|
36398
36962
|
const db = getDb();
|
|
36399
|
-
const connectionId = `authc_${(0,
|
|
36963
|
+
const connectionId = `authc_${(0, import_node_crypto15.randomUUID)().replace(/-/g, "").slice(0, 20)}`;
|
|
36400
36964
|
await db.execute({
|
|
36401
36965
|
sql: `INSERT INTO browser_auth_connections (connection_id, domain, profile, account_email, note, status, browser_agent_session_id)
|
|
36402
36966
|
VALUES (?, ?, ?, ?, ?, 'NEEDS_AUTH', ?)`,
|
|
@@ -36430,7 +36994,7 @@ async function listAuthConnectionsByProfile(profile) {
|
|
|
36430
36994
|
}
|
|
36431
36995
|
async function createSessionRow(input) {
|
|
36432
36996
|
const db = getDb();
|
|
36433
|
-
const id = `bas_${(0,
|
|
36997
|
+
const id = `bas_${(0, import_node_crypto15.randomUUID)().replace(/-/g, "").slice(0, 20)}`;
|
|
36434
36998
|
await db.execute({
|
|
36435
36999
|
sql: `INSERT INTO browser_agent_sessions (id, runtime_session_id, live_view_url, cdp_ws_url, status, label, user_id, concurrency_lock_id, last_action_at)
|
|
36436
37000
|
VALUES (?, ?, ?, ?, 'open', ?, ?, ?, datetime('now'))`,
|
|
@@ -36499,7 +37063,7 @@ async function recordAction(input) {
|
|
|
36499
37063
|
sql: `INSERT INTO browser_agent_actions (id, session_id, type, params_json, ok, error)
|
|
36500
37064
|
VALUES (?, ?, ?, ?, ?, ?)`,
|
|
36501
37065
|
args: [
|
|
36502
|
-
`baa_${(0,
|
|
37066
|
+
`baa_${(0, import_node_crypto15.randomUUID)().replace(/-/g, "").slice(0, 20)}`,
|
|
36503
37067
|
input.sessionId,
|
|
36504
37068
|
input.type,
|
|
36505
37069
|
input.params == null ? null : JSON.stringify(input.params),
|
|
@@ -36539,13 +37103,14 @@ async function listReplayRows(sessionId) {
|
|
|
36539
37103
|
});
|
|
36540
37104
|
return res.rows;
|
|
36541
37105
|
}
|
|
36542
|
-
var
|
|
37106
|
+
var import_node_crypto15, _ready2, _migrationPromise2;
|
|
36543
37107
|
var init_browser_agent_db = __esm({
|
|
36544
37108
|
"src/api/browser-agent-db.ts"() {
|
|
36545
37109
|
"use strict";
|
|
36546
|
-
|
|
37110
|
+
import_node_crypto15 = require("crypto");
|
|
36547
37111
|
init_db();
|
|
36548
37112
|
_ready2 = false;
|
|
37113
|
+
_migrationPromise2 = null;
|
|
36549
37114
|
}
|
|
36550
37115
|
});
|
|
36551
37116
|
|
|
@@ -38255,7 +38820,7 @@ function buildBrowserAgentRoutes() {
|
|
|
38255
38820
|
}
|
|
38256
38821
|
const existing = await getExtensionRow(user.id, name);
|
|
38257
38822
|
if (existing) return c.json({ error: `an extension named "${name}" already exists \u2014 delete it first or pick another name` }, 409);
|
|
38258
|
-
const backendName = `u${user.id}_${(0,
|
|
38823
|
+
const backendName = `u${user.id}_${(0, import_node_crypto16.randomUUID)().replace(/-/g, "")}`;
|
|
38259
38824
|
try {
|
|
38260
38825
|
const imported = await importExtensionFromStore(storeUrl, backendName);
|
|
38261
38826
|
const row = await createExtensionRow({
|
|
@@ -38314,11 +38879,11 @@ function buildBrowserAgentRoutes() {
|
|
|
38314
38879
|
});
|
|
38315
38880
|
return app2;
|
|
38316
38881
|
}
|
|
38317
|
-
var
|
|
38882
|
+
var import_node_crypto16, import_hono18, auth, DEFAULT_BROWSER_SESSION_LOCK_TTL_SECONDS, EXTENSION_NAME_RE;
|
|
38318
38883
|
var init_browser_agent_routes = __esm({
|
|
38319
38884
|
"src/api/browser-agent-routes.ts"() {
|
|
38320
38885
|
"use strict";
|
|
38321
|
-
|
|
38886
|
+
import_node_crypto16 = require("crypto");
|
|
38322
38887
|
import_hono18 = require("hono");
|
|
38323
38888
|
init_api_auth();
|
|
38324
38889
|
init_errors();
|
|
@@ -38728,7 +39293,7 @@ async function getKeys() {
|
|
|
38728
39293
|
const privateKey = await (0, import_jose2.importPKCS8)(pem, "RS256", { extractable: true });
|
|
38729
39294
|
const full = await (0, import_jose2.exportJWK)(privateKey);
|
|
38730
39295
|
const publicJwk = { kty: full.kty, n: full.n, e: full.e };
|
|
38731
|
-
const kid = (0,
|
|
39296
|
+
const kid = (0, import_node_crypto17.createHash)("sha256").update(JSON.stringify({ e: publicJwk.e, kty: publicJwk.kty, n: publicJwk.n })).digest("base64url").slice(0, 16);
|
|
38732
39297
|
publicJwk.kid = kid;
|
|
38733
39298
|
publicJwk.alg = "RS256";
|
|
38734
39299
|
publicJwk.use = "sig";
|
|
@@ -38907,23 +39472,23 @@ async function validateAuthRequest(p) {
|
|
|
38907
39472
|
}
|
|
38908
39473
|
function pkceMatches(verifier, challenge) {
|
|
38909
39474
|
if (!verifier) return false;
|
|
38910
|
-
const computed = (0,
|
|
39475
|
+
const computed = (0, import_node_crypto17.createHash)("sha256").update(verifier).digest("base64url");
|
|
38911
39476
|
return computed === challenge;
|
|
38912
39477
|
}
|
|
38913
39478
|
async function mintAccessToken(identity, scope, plan, audience) {
|
|
38914
39479
|
const { privateKey, kid } = await getKeys();
|
|
38915
|
-
return new import_jose2.SignJWT({ scope, plan }).setProtectedHeader({ alg: "RS256", kid }).setIssuer(ISSUER).setSubject(identity).setAudience(audience).setIssuedAt().setJti((0,
|
|
39480
|
+
return new import_jose2.SignJWT({ scope, plan }).setProtectedHeader({ alg: "RS256", kid }).setIssuer(ISSUER).setSubject(identity).setAudience(audience).setIssuedAt().setJti((0, import_node_crypto17.randomUUID)()).setExpirationTime(`${ACCESS_TTL_SECONDS}s`).sign(privateKey);
|
|
38916
39481
|
}
|
|
38917
39482
|
function tokenErrorResponse(c, error, description, status) {
|
|
38918
39483
|
return c.json({ error, error_description: description }, status);
|
|
38919
39484
|
}
|
|
38920
|
-
var import_hono20, import_cookie,
|
|
39485
|
+
var import_hono20, import_cookie, import_node_crypto17, import_jose2, ISSUER, RESOURCE, SCRAPER_RESOURCE, MEMORY_SCOPES, SCRAPER_SCOPES, SUPPORTED_SCOPES, ACCESS_TTL_SECONDS, REFRESH_TTL_SECONDS, CODE_TTL_SECONDS, ROTATION_GRACE_SECONDS, secureCookies, sessionCookieOptions, cachedKeys, oauthApp;
|
|
38921
39486
|
var init_oauth_routes = __esm({
|
|
38922
39487
|
"src/api/oauth-routes.ts"() {
|
|
38923
39488
|
"use strict";
|
|
38924
39489
|
import_hono20 = require("hono");
|
|
38925
39490
|
import_cookie = require("hono/cookie");
|
|
38926
|
-
|
|
39491
|
+
import_node_crypto17 = require("crypto");
|
|
38927
39492
|
import_jose2 = require("jose");
|
|
38928
39493
|
init_session();
|
|
38929
39494
|
init_db();
|
|
@@ -39009,7 +39574,7 @@ var init_oauth_routes = __esm({
|
|
|
39009
39574
|
}
|
|
39010
39575
|
}
|
|
39011
39576
|
const clientName = typeof body.client_name === "string" ? body.client_name : null;
|
|
39012
|
-
const clientId = `client_${(0,
|
|
39577
|
+
const clientId = `client_${(0, import_node_crypto17.randomBytes)(16).toString("hex")}`;
|
|
39013
39578
|
await registerClient(clientId, redirectUris, clientName);
|
|
39014
39579
|
console.log("[oauth-dcr] register OK client_id=%s redirect_uris=%s", clientId, JSON.stringify(redirectUris));
|
|
39015
39580
|
return c.json({
|
|
@@ -39062,7 +39627,7 @@ var init_oauth_routes = __esm({
|
|
|
39062
39627
|
if (action === "deny") return redirectWithError(p.redirect_uri, p.state, "access_denied");
|
|
39063
39628
|
if (action !== "approve") return c.text("unsupported action", 400);
|
|
39064
39629
|
const scope = negotiateScope(p.scope, user, p.resource);
|
|
39065
|
-
const code = `code_${(0,
|
|
39630
|
+
const code = `code_${(0, import_node_crypto17.randomBytes)(32).toString("base64url")}`;
|
|
39066
39631
|
const expiresAt = new Date(Date.now() + CODE_TTL_SECONDS * 1e3).toISOString();
|
|
39067
39632
|
await putCode({
|
|
39068
39633
|
code,
|
|
@@ -39101,7 +39666,7 @@ var init_oauth_routes = __esm({
|
|
|
39101
39666
|
const plan = user ? resolvePlan(user) : "free";
|
|
39102
39667
|
const audience = record.resource ?? RESOURCE();
|
|
39103
39668
|
const accessToken = await mintAccessToken(record.identity, record.scope, plan, audience);
|
|
39104
|
-
const refreshToken = `rt_${(0,
|
|
39669
|
+
const refreshToken = `rt_${(0, import_node_crypto17.randomBytes)(40).toString("base64url")}`;
|
|
39105
39670
|
await putRefresh({
|
|
39106
39671
|
refresh_token: refreshToken,
|
|
39107
39672
|
client_id: clientId,
|
|
@@ -39131,7 +39696,7 @@ var init_oauth_routes = __esm({
|
|
|
39131
39696
|
const plan = user ? resolvePlan(user) : "free";
|
|
39132
39697
|
const audience = record.resource ?? RESOURCE();
|
|
39133
39698
|
const accessToken = await mintAccessToken(record.identity, record.scope, plan, audience);
|
|
39134
|
-
const nextRefresh = `rt_${(0,
|
|
39699
|
+
const nextRefresh = `rt_${(0, import_node_crypto17.randomBytes)(40).toString("base64url")}`;
|
|
39135
39700
|
await rotateRefresh(refreshToken, {
|
|
39136
39701
|
refresh_token: nextRefresh,
|
|
39137
39702
|
client_id: record.client_id,
|
|
@@ -40448,7 +41013,7 @@ var init_site_audit_worker = __esm({
|
|
|
40448
41013
|
|
|
40449
41014
|
// src/api/page-diff.ts
|
|
40450
41015
|
function sha256Hex(value) {
|
|
40451
|
-
return (0,
|
|
41016
|
+
return (0, import_node_crypto18.createHash)("sha256").update(value).digest("hex");
|
|
40452
41017
|
}
|
|
40453
41018
|
function truncateForStorage(value, maxChars = MAX_SNAPSHOT_CONTENT_CHARS) {
|
|
40454
41019
|
if (value.length <= maxChars) return { value, truncated: false };
|
|
@@ -40505,11 +41070,11 @@ function diffPageContent(oldContent, newContent) {
|
|
|
40505
41070
|
totalChangedLineCount
|
|
40506
41071
|
};
|
|
40507
41072
|
}
|
|
40508
|
-
var
|
|
41073
|
+
var import_node_crypto18, import_diff, MAX_SNAPSHOT_CONTENT_CHARS, MAX_DIFF_HUNKS, MAX_DIFF_LINES_PER_RESPONSE;
|
|
40509
41074
|
var init_page_diff = __esm({
|
|
40510
41075
|
"src/api/page-diff.ts"() {
|
|
40511
41076
|
"use strict";
|
|
40512
|
-
|
|
41077
|
+
import_node_crypto18 = require("crypto");
|
|
40513
41078
|
import_diff = require("diff");
|
|
40514
41079
|
MAX_SNAPSHOT_CONTENT_CHARS = 25e4;
|
|
40515
41080
|
MAX_DIFF_HUNKS = 200;
|
|
@@ -40586,7 +41151,7 @@ function canonicalJson(value) {
|
|
|
40586
41151
|
return JSON.stringify(value);
|
|
40587
41152
|
}
|
|
40588
41153
|
function sha2562(value) {
|
|
40589
|
-
return (0,
|
|
41154
|
+
return (0, import_node_crypto19.createHash)("sha256").update(value).digest("hex");
|
|
40590
41155
|
}
|
|
40591
41156
|
function sensitiveKey(key) {
|
|
40592
41157
|
const normalized = key.toLowerCase().replace(/[^a-z0-9]/g, "");
|
|
@@ -40801,11 +41366,11 @@ async function importServiceConnectionToMemory(identity, input, dependencies) {
|
|
|
40801
41366
|
...!searchReady ? { warning: "The snapshot was stored, but no search chunks were indexed yet." } : {}
|
|
40802
41367
|
};
|
|
40803
41368
|
}
|
|
40804
|
-
var
|
|
41369
|
+
var import_node_crypto19, CONNECTION_MEMORY_IMPORT_MAX_ARGS_BYTES, CONNECTION_MEMORY_IMPORT_MAX_RESULT_BYTES, CONNECTION_MEMORY_IMPORT_MAX_STRING_CHARS, CONNECTION_MEMORY_IMPORT_MAX_DEPTH, ConnectionMemoryImportError;
|
|
40805
41370
|
var init_connection_memory_import = __esm({
|
|
40806
41371
|
"src/api/connection-memory-import.ts"() {
|
|
40807
41372
|
"use strict";
|
|
40808
|
-
|
|
41373
|
+
import_node_crypto19 = require("crypto");
|
|
40809
41374
|
init_slugify();
|
|
40810
41375
|
CONNECTION_MEMORY_IMPORT_MAX_ARGS_BYTES = 64 * 1024;
|
|
40811
41376
|
CONNECTION_MEMORY_IMPORT_MAX_RESULT_BYTES = 1e6;
|
|
@@ -40979,7 +41544,7 @@ function finitePositive(value, fallback) {
|
|
|
40979
41544
|
return Number.isFinite(value) && value > 0 ? value : fallback;
|
|
40980
41545
|
}
|
|
40981
41546
|
async function collectConnectedDataExport(args) {
|
|
40982
|
-
const exportId = (0,
|
|
41547
|
+
const exportId = (0, import_node_crypto20.randomUUID)();
|
|
40983
41548
|
const now = args.now ?? Date.now;
|
|
40984
41549
|
const startedAt = now();
|
|
40985
41550
|
const budgetMs = finitePositive(CONNECTED_DATA_EXPORT_BUDGET_MS, 24e4);
|
|
@@ -41089,11 +41654,11 @@ ${lines.length ? `${lines.join("\n")}
|
|
|
41089
41654
|
untrustedContent: true
|
|
41090
41655
|
};
|
|
41091
41656
|
}
|
|
41092
|
-
var
|
|
41657
|
+
var import_node_crypto20, CONNECTED_DATA_INLINE_BUDGET_BYTES, CONNECTED_DATA_MAX_EXPORT_BYTES, CONNECTED_DATA_EXPORT_BUDGET_MS, CONNECTED_DATA_PAGE_START_HEADROOM_MS, CONNECTED_DATA_DATASETS, ConnectedDataExportValidationError;
|
|
41093
41658
|
var init_connected_data_export = __esm({
|
|
41094
41659
|
"src/api/connected-data-export.ts"() {
|
|
41095
41660
|
"use strict";
|
|
41096
|
-
|
|
41661
|
+
import_node_crypto20 = require("crypto");
|
|
41097
41662
|
CONNECTED_DATA_INLINE_BUDGET_BYTES = Number(
|
|
41098
41663
|
process.env.MCP_SCRAPER_CONNECTED_DATA_INLINE_BUDGET_BYTES ?? 5e4
|
|
41099
41664
|
);
|
|
@@ -41131,12 +41696,6 @@ var init_connected_data_export = __esm({
|
|
|
41131
41696
|
});
|
|
41132
41697
|
|
|
41133
41698
|
// src/api/credit-operations.ts
|
|
41134
|
-
async function grantSignupCredit(userId) {
|
|
41135
|
-
if (!FREE_SIGNUP_MC || FREE_SIGNUP_MC <= 0) return;
|
|
41136
|
-
const already = await ledgerExistsForOperation(userId, LedgerOperation.SIGNUP_GRANT);
|
|
41137
|
-
if (already) return;
|
|
41138
|
-
await creditMc(userId, FREE_SIGNUP_MC, LedgerOperation.SIGNUP_GRANT, "Welcome credits");
|
|
41139
|
-
}
|
|
41140
41699
|
async function applyMonthlyFreeRefresh(user) {
|
|
41141
41700
|
return user;
|
|
41142
41701
|
}
|
|
@@ -41903,7 +42462,7 @@ function canonicalJson2(value) {
|
|
|
41903
42462
|
return JSON.stringify(value);
|
|
41904
42463
|
}
|
|
41905
42464
|
function projectedToolSchemaHash(tool) {
|
|
41906
|
-
return (0,
|
|
42465
|
+
return (0, import_node_crypto21.createHash)("sha256").update(canonicalJson2(tool)).digest("hex");
|
|
41907
42466
|
}
|
|
41908
42467
|
async function describeNangoTool(identity, connectionId, tool, fresh) {
|
|
41909
42468
|
const body = await controlRequest("/api/internal/nango/connections/actions/describe", {
|
|
@@ -42033,11 +42592,11 @@ async function callScheduleConnectionExportPage(identity, input) {
|
|
|
42033
42592
|
untrustedContent: true
|
|
42034
42593
|
};
|
|
42035
42594
|
}
|
|
42036
|
-
var
|
|
42595
|
+
var import_node_crypto21, DEFAULT_NANGO_CONTROL_URL, DISABLED_NANGO_TOOLS, CONNECTION_SYNC_REQUIRED_TOOLS, CONNECTION_SYNC_OPTIONAL_TOOLS, NangoControlError, ScheduleConnectionValidationError, SAFE_CONTROL_ERROR_CODES, CONTROL_ERROR_CODE_ALIASES;
|
|
42037
42596
|
var init_nango_control = __esm({
|
|
42038
42597
|
"src/api/nango-control.ts"() {
|
|
42039
42598
|
"use strict";
|
|
42040
|
-
|
|
42599
|
+
import_node_crypto21 = require("crypto");
|
|
42041
42600
|
init_connected_data_export();
|
|
42042
42601
|
DEFAULT_NANGO_CONTROL_URL = "https://mcp-scraper-scheduler.vercel.app";
|
|
42043
42602
|
DISABLED_NANGO_TOOLS = {
|
|
@@ -42525,7 +43084,7 @@ async function processJob(job) {
|
|
|
42525
43084
|
if (diff > 0) await creditMc(job.user_id, diff, "paa_refund", "overestimate refund");
|
|
42526
43085
|
else if (diff < 0) await debitMc(job.user_id, -diff, "paa", opts.query ?? job.query);
|
|
42527
43086
|
} else if (opts.serpOnly && typeof opts.billingHoldMc === "number") {
|
|
42528
|
-
const actualCost = headlessSentOut.value
|
|
43087
|
+
const actualCost = serpActualCostMc(headlessSentOut.value);
|
|
42529
43088
|
const diff = opts.billingHoldMc - actualCost;
|
|
42530
43089
|
if (diff > 0) await creditMc(job.user_id, diff, "serp_refund", "headless-mode pricing settle");
|
|
42531
43090
|
else if (diff < 0) await debitMc(job.user_id, -diff, "serp", opts.query ?? job.query);
|
|
@@ -42882,7 +43441,6 @@ var init_server = __esm({
|
|
|
42882
43441
|
init_browser_agent_console();
|
|
42883
43442
|
init_browser_agent_db();
|
|
42884
43443
|
init_cost_context();
|
|
42885
|
-
init_cost_telemetry();
|
|
42886
43444
|
init_stripe_routes();
|
|
42887
43445
|
init_oauth_routes();
|
|
42888
43446
|
init_chat_routes();
|
|
@@ -42988,7 +43546,6 @@ var init_server = __esm({
|
|
|
42988
43546
|
const forceHeadless = c.req.header("x-kernel-headless") === "true";
|
|
42989
43547
|
return runWithCostContext({ op, probeRunId, forceHeadless }, () => next());
|
|
42990
43548
|
});
|
|
42991
|
-
void migrateCostTelemetry();
|
|
42992
43549
|
app.post("/auth/register", requireAllowedOrigin, async (c) => {
|
|
42993
43550
|
const { email, password } = await c.req.json();
|
|
42994
43551
|
const normalizedEmail = email?.trim().toLowerCase();
|
|
@@ -43019,7 +43576,6 @@ var init_server = __esm({
|
|
|
43019
43576
|
}
|
|
43020
43577
|
}
|
|
43021
43578
|
const newKey = await rotateApiKey(user.id);
|
|
43022
|
-
await grantSignupCredit(user.id);
|
|
43023
43579
|
void (async () => {
|
|
43024
43580
|
try {
|
|
43025
43581
|
const full = await getUserById(user.id);
|
|
@@ -44112,7 +44668,7 @@ var init_server = __esm({
|
|
|
44112
44668
|
if (diff > 0) await creditMc(user.id, diff, LedgerOperation.PAA_REFUND, "overestimate refund");
|
|
44113
44669
|
else if (diff < 0) await debitMc(user.id, -diff, LedgerOperation.PAA, options.query);
|
|
44114
44670
|
} else {
|
|
44115
|
-
const actualCost = headlessSentOut.value
|
|
44671
|
+
const actualCost = serpActualCostMc(headlessSentOut.value);
|
|
44116
44672
|
const diff = syncCost - actualCost;
|
|
44117
44673
|
if (diff > 0) await creditMc(user.id, diff, LedgerOperation.SERP_REFUND, "headless-mode pricing settle");
|
|
44118
44674
|
else if (diff < 0) await debitMc(user.id, -diff, LedgerOperation.SERP, options.query);
|
|
@@ -44191,22 +44747,13 @@ var init_server = __esm({
|
|
|
44191
44747
|
});
|
|
44192
44748
|
app.post("/admin/backfill-signup-credits", adminAuth, async (c) => {
|
|
44193
44749
|
const users = await listUsers();
|
|
44194
|
-
|
|
44195
|
-
|
|
44196
|
-
|
|
44197
|
-
|
|
44198
|
-
|
|
44199
|
-
|
|
44200
|
-
|
|
44201
|
-
if (alreadyHasGrant) {
|
|
44202
|
-
skipped++;
|
|
44203
|
-
} else {
|
|
44204
|
-
await grantSignupCredit(user.id);
|
|
44205
|
-
credited++;
|
|
44206
|
-
users_credited.push({ id: user.id, email: user.email });
|
|
44207
|
-
}
|
|
44208
|
-
}
|
|
44209
|
-
return c.json({ processed, credited, skipped, users_credited });
|
|
44750
|
+
return c.json({
|
|
44751
|
+
processed: users.length,
|
|
44752
|
+
credited: 0,
|
|
44753
|
+
skipped: users.length,
|
|
44754
|
+
users_credited: [],
|
|
44755
|
+
retired: true
|
|
44756
|
+
});
|
|
44210
44757
|
});
|
|
44211
44758
|
app.post("/extract-url", auth2, async (c) => {
|
|
44212
44759
|
const raw = await c.req.json().catch(() => ({}));
|