mcp-scraper 0.21.2 → 0.21.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/dist/bin/api-server.cjs +1061 -308
- package/dist/bin/api-server.cjs.map +1 -1
- package/dist/bin/api-server.js +3 -3
- package/dist/bin/mcp-scraper-cli.cjs +1 -1
- package/dist/bin/mcp-scraper-cli.cjs.map +1 -1
- package/dist/bin/mcp-scraper-cli.js +1 -1
- package/dist/bin/mcp-scraper-install.cjs +1 -1
- package/dist/bin/mcp-scraper-install.cjs.map +1 -1
- package/dist/bin/mcp-scraper-install.js +1 -1
- package/dist/bin/mcp-stdio-server.cjs +101 -34
- package/dist/bin/mcp-stdio-server.cjs.map +1 -1
- package/dist/bin/mcp-stdio-server.js +5 -5
- package/dist/bin/paa-harvest.cjs +22 -0
- package/dist/bin/paa-harvest.cjs.map +1 -1
- package/dist/bin/paa-harvest.js +3 -3
- package/dist/{chunk-ZJDANZTQ.js → chunk-E5J4HJBO.js} +3 -2
- package/dist/chunk-E5J4HJBO.js.map +1 -0
- package/dist/{chunk-DZBGULWW.js → chunk-ICT7DDHL.js} +399 -3
- package/dist/chunk-ICT7DDHL.js.map +1 -0
- package/dist/{chunk-2FF4GBQ4.js → chunk-JHI373VB.js} +49 -7
- package/dist/chunk-JHI373VB.js.map +1 -0
- package/dist/{chunk-GMZFQRH2.js → chunk-MTSBI7ZH.js} +23 -6
- package/dist/chunk-MTSBI7ZH.js.map +1 -0
- package/dist/{chunk-ZNNYPE2U.js → chunk-N4JHXPW6.js} +89 -27
- package/dist/chunk-N4JHXPW6.js.map +1 -0
- package/dist/{chunk-X3NIXHHG.js → chunk-PNBYYAEK.js} +3 -3
- package/dist/{chunk-JBME5WJR.js → chunk-SDHPQTFM.js} +2 -2
- package/dist/chunk-WQI7JIIN.js +7 -0
- package/dist/chunk-WQI7JIIN.js.map +1 -0
- package/dist/{db-EONGTNJM.js → db-WRILVOFY.js} +6 -2
- package/dist/{extract-bundle-HMCYUVBJ.js → extract-bundle-NFFVMUMT.js} +2 -2
- package/dist/index.cjs +22 -0
- package/dist/index.cjs.map +1 -1
- package/dist/index.js +3 -3
- package/dist/{server-CHTX6YQA.js → server-OLOOFN5W.js} +499 -271
- package/dist/server-OLOOFN5W.js.map +1 -0
- package/dist/{site-extract-repository-5V5EEQ72.js → site-extract-repository-2QQSL2QC.js} +4 -4
- package/dist/{worker-GE5JMHE6.js → worker-RV2TYXE3.js} +15 -8
- package/dist/worker-RV2TYXE3.js.map +1 -0
- package/docs/mcp-tool-manifest.generated.json +90 -11
- package/docs/specs/oauth-mcp-spec.md +2 -2
- package/package.json +1 -1
- package/dist/chunk-2FF4GBQ4.js.map +0 -1
- package/dist/chunk-DZBGULWW.js.map +0 -1
- package/dist/chunk-GMZFQRH2.js.map +0 -1
- package/dist/chunk-NBF4F4UT.js +0 -7
- package/dist/chunk-NBF4F4UT.js.map +0 -1
- package/dist/chunk-ZJDANZTQ.js.map +0 -1
- package/dist/chunk-ZNNYPE2U.js.map +0 -1
- package/dist/server-CHTX6YQA.js.map +0 -1
- package/dist/worker-GE5JMHE6.js.map +0 -1
- /package/dist/{chunk-X3NIXHHG.js.map → chunk-PNBYYAEK.js.map} +0 -0
- /package/dist/{chunk-JBME5WJR.js.map → chunk-SDHPQTFM.js.map} +0 -0
- /package/dist/{db-EONGTNJM.js.map → db-WRILVOFY.js.map} +0 -0
- /package/dist/{extract-bundle-HMCYUVBJ.js.map → extract-bundle-NFFVMUMT.js.map} +0 -0
- /package/dist/{site-extract-repository-5V5EEQ72.js.map → site-extract-repository-2QQSL2QC.js.map} +0 -0
package/dist/bin/api-server.cjs
CHANGED
|
@@ -3602,6 +3602,7 @@ __export(db_exports, {
|
|
|
3602
3602
|
creditMc: () => creditMc,
|
|
3603
3603
|
deactivateUser: () => deactivateUser,
|
|
3604
3604
|
debitMc: () => debitMc,
|
|
3605
|
+
debitMcIdempotent: () => debitMcIdempotent,
|
|
3605
3606
|
deleteWorkflowSchedule: () => deleteWorkflowSchedule,
|
|
3606
3607
|
expireOldLots: () => expireOldLots,
|
|
3607
3608
|
failJob: () => failJob,
|
|
@@ -3670,12 +3671,27 @@ __export(db_exports, {
|
|
|
3670
3671
|
setStripeCustomerId: () => setStripeCustomerId,
|
|
3671
3672
|
setSubscriptionTier: () => setSubscriptionTier,
|
|
3672
3673
|
setUserMemoryCreds: () => setUserMemoryCreds,
|
|
3674
|
+
settleDebitMcIdempotent: () => settleDebitMcIdempotent,
|
|
3673
3675
|
startHarvestAttempt: () => startHarvestAttempt,
|
|
3674
3676
|
stripeEventAlreadyProcessed: () => stripeEventAlreadyProcessed,
|
|
3675
3677
|
updateKpoJobState: () => updateKpoJobState,
|
|
3676
3678
|
upsertPageSnapshot: () => upsertPageSnapshot,
|
|
3677
3679
|
verifyPassword: () => verifyPassword
|
|
3678
3680
|
});
|
|
3681
|
+
function sqlStringList(values) {
|
|
3682
|
+
return values.map((value) => `'${value.replaceAll("'", "''")}'`).join(", ");
|
|
3683
|
+
}
|
|
3684
|
+
async function hasCurrentCoreSchema(db) {
|
|
3685
|
+
const checks = [
|
|
3686
|
+
`(SELECT COUNT(*) FROM sqlite_master WHERE type = 'table' AND name IN (${sqlStringList(CORE_SCHEMA_TABLES)})) = ${CORE_SCHEMA_TABLES.length}`,
|
|
3687
|
+
`(SELECT COUNT(*) FROM sqlite_master WHERE type = 'index' AND name IN (${sqlStringList(CORE_SCHEMA_INDEXES)})) = ${CORE_SCHEMA_INDEXES.length}`,
|
|
3688
|
+
...Object.entries(CORE_SCHEMA_COLUMNS).map(
|
|
3689
|
+
([table, columns]) => `(SELECT COUNT(*) FROM pragma_table_info('${table}') WHERE name IN (${sqlStringList(columns)})) = ${columns.length}`
|
|
3690
|
+
)
|
|
3691
|
+
];
|
|
3692
|
+
const result = await db.execute(`SELECT ${checks.join(" AND ")} AS ready`);
|
|
3693
|
+
return Number(result.rows[0]?.ready ?? 0) === 1;
|
|
3694
|
+
}
|
|
3679
3695
|
function getDb() {
|
|
3680
3696
|
if (!_db) _db = (0, import_http.createClient)({ url: DB_URL, authToken: DB_TOKEN });
|
|
3681
3697
|
return _db;
|
|
@@ -3685,6 +3701,31 @@ function setDbForTesting(client2) {
|
|
|
3685
3701
|
}
|
|
3686
3702
|
async function migrate() {
|
|
3687
3703
|
const db = getDb();
|
|
3704
|
+
let applied;
|
|
3705
|
+
try {
|
|
3706
|
+
applied = await db.execute({
|
|
3707
|
+
sql: "SELECT 1 AS applied FROM schema_migrations WHERE version = ? LIMIT 1",
|
|
3708
|
+
args: [CORE_SCHEMA_VERSION]
|
|
3709
|
+
});
|
|
3710
|
+
} catch (error) {
|
|
3711
|
+
const message = error instanceof Error ? error.message.toLowerCase() : String(error).toLowerCase();
|
|
3712
|
+
if (!message.includes("no such table") || !message.includes("schema_migrations")) throw error;
|
|
3713
|
+
await db.execute(`
|
|
3714
|
+
CREATE TABLE IF NOT EXISTS schema_migrations (
|
|
3715
|
+
version TEXT PRIMARY KEY,
|
|
3716
|
+
applied_at TEXT NOT NULL DEFAULT (datetime('now'))
|
|
3717
|
+
)
|
|
3718
|
+
`);
|
|
3719
|
+
applied = { rows: [] };
|
|
3720
|
+
}
|
|
3721
|
+
if (applied.rows.length > 0) return;
|
|
3722
|
+
if (await hasCurrentCoreSchema(db)) {
|
|
3723
|
+
await db.execute({
|
|
3724
|
+
sql: "INSERT OR IGNORE INTO schema_migrations (version) VALUES (?)",
|
|
3725
|
+
args: [CORE_SCHEMA_VERSION]
|
|
3726
|
+
});
|
|
3727
|
+
return;
|
|
3728
|
+
}
|
|
3688
3729
|
await db.execute(`
|
|
3689
3730
|
CREATE TABLE IF NOT EXISTS users (
|
|
3690
3731
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
@@ -3918,6 +3959,19 @@ async function migrate() {
|
|
|
3918
3959
|
`);
|
|
3919
3960
|
await db.execute(`CREATE INDEX IF NOT EXISTS billing_events_user_created_at ON billing_events(user_id, created_at DESC)`);
|
|
3920
3961
|
await db.execute(`CREATE INDEX IF NOT EXISTS billing_events_authorization ON billing_events(authorization_id)`);
|
|
3962
|
+
await db.execute(`
|
|
3963
|
+
CREATE TABLE IF NOT EXISTS billing_debits (
|
|
3964
|
+
idempotency_key TEXT PRIMARY KEY,
|
|
3965
|
+
user_id INTEGER NOT NULL REFERENCES users(id),
|
|
3966
|
+
amount_mc INTEGER NOT NULL,
|
|
3967
|
+
operation TEXT NOT NULL,
|
|
3968
|
+
description TEXT,
|
|
3969
|
+
status TEXT NOT NULL DEFAULT 'claimed',
|
|
3970
|
+
created_at TEXT NOT NULL DEFAULT (datetime('now')),
|
|
3971
|
+
applied_at TEXT
|
|
3972
|
+
)
|
|
3973
|
+
`);
|
|
3974
|
+
await db.execute(`CREATE INDEX IF NOT EXISTS billing_debits_user_status ON billing_debits(user_id, status)`);
|
|
3921
3975
|
await db.execute(`
|
|
3922
3976
|
CREATE TABLE IF NOT EXISTS site_audit_jobs (
|
|
3923
3977
|
id TEXT PRIMARY KEY,
|
|
@@ -4165,6 +4219,13 @@ async function migrate() {
|
|
|
4165
4219
|
)
|
|
4166
4220
|
`);
|
|
4167
4221
|
await db.execute(`CREATE INDEX IF NOT EXISTS page_snapshots_user_checked_at ON page_snapshots(user_id, checked_at DESC)`);
|
|
4222
|
+
if (!await hasCurrentCoreSchema(db)) {
|
|
4223
|
+
throw new Error("core schema migration finished without all required tables, indexes, and columns");
|
|
4224
|
+
}
|
|
4225
|
+
await db.execute({
|
|
4226
|
+
sql: "INSERT OR IGNORE INTO schema_migrations (version) VALUES (?)",
|
|
4227
|
+
args: [CORE_SCHEMA_VERSION]
|
|
4228
|
+
});
|
|
4168
4229
|
}
|
|
4169
4230
|
async function registerClient(clientId, redirectUris, name) {
|
|
4170
4231
|
await getDb().execute({
|
|
@@ -4627,6 +4688,17 @@ async function upsertPageSnapshot(userId, url, snapshot) {
|
|
|
4627
4688
|
args: [userId, url, snapshot.contentHash, snapshot.content, snapshot.title, snapshot.contentBytes, snapshot.truncated ? 1 : 0]
|
|
4628
4689
|
});
|
|
4629
4690
|
}
|
|
4691
|
+
function serializeRequestEventResult(result) {
|
|
4692
|
+
if (result == null) return null;
|
|
4693
|
+
const serialized = JSON.stringify(result);
|
|
4694
|
+
const bytes = Buffer.byteLength(serialized, "utf8");
|
|
4695
|
+
if (bytes <= REQUEST_EVENT_RESULT_MAX_BYTES) return serialized;
|
|
4696
|
+
return JSON.stringify({
|
|
4697
|
+
truncated: true,
|
|
4698
|
+
originalBytes: bytes,
|
|
4699
|
+
sha256: (0, import_node_crypto.createHash)("sha256").update(serialized).digest("hex")
|
|
4700
|
+
});
|
|
4701
|
+
}
|
|
4630
4702
|
function rowToRequestEvent(row) {
|
|
4631
4703
|
const rawResult = row.result != null ? String(row.result) : null;
|
|
4632
4704
|
return {
|
|
@@ -4657,7 +4729,7 @@ async function logRequestEvent(input) {
|
|
|
4657
4729
|
input.query,
|
|
4658
4730
|
input.location ?? null,
|
|
4659
4731
|
input.resultCount ?? null,
|
|
4660
|
-
|
|
4732
|
+
serializeRequestEventResult(input.result),
|
|
4661
4733
|
input.error ?? null
|
|
4662
4734
|
]
|
|
4663
4735
|
});
|
|
@@ -4666,7 +4738,7 @@ async function logRequestEvent(input) {
|
|
|
4666
4738
|
async function listRequestEvents(userId, limit = 100) {
|
|
4667
4739
|
const res = await getDb().execute({
|
|
4668
4740
|
sql: `
|
|
4669
|
-
SELECT id, user_id, source, status, query, location, result_count, result, error, created_at
|
|
4741
|
+
SELECT id, user_id, source, status, query, location, result_count, NULL AS result, error, created_at
|
|
4670
4742
|
FROM request_events
|
|
4671
4743
|
WHERE user_id = ?
|
|
4672
4744
|
ORDER BY created_at DESC
|
|
@@ -5289,6 +5361,241 @@ async function debitMc(userId, mc, operation, description) {
|
|
|
5289
5361
|
const res = await db.execute({ sql: "SELECT balance_mc FROM users WHERE id = ?", args: [userId] });
|
|
5290
5362
|
return { ok: true, balance_mc: Number(res.rows[0]?.balance_mc ?? 0) };
|
|
5291
5363
|
}
|
|
5364
|
+
async function debitMcIdempotent(userId, mc, operation, description, idempotencyKey) {
|
|
5365
|
+
if (!Number.isSafeInteger(mc) || mc <= 0) throw new Error("idempotent debit amount must be a positive integer");
|
|
5366
|
+
if (!idempotencyKey || idempotencyKey.length > 500) throw new Error("idempotent debit key is required and must be bounded");
|
|
5367
|
+
const db = getDb();
|
|
5368
|
+
await ensureMigrated(userId);
|
|
5369
|
+
let results;
|
|
5370
|
+
try {
|
|
5371
|
+
results = await db.batch([
|
|
5372
|
+
{
|
|
5373
|
+
sql: "UPDATE billing_debits SET amount_mc = ? WHERE idempotency_key = ? AND user_id = ? AND operation = ? AND description = ? AND status = 'claimed'",
|
|
5374
|
+
args: [mc, idempotencyKey, userId, operation, description]
|
|
5375
|
+
},
|
|
5376
|
+
{
|
|
5377
|
+
sql: "INSERT OR IGNORE INTO billing_debits (idempotency_key, user_id, amount_mc, operation, description, status) VALUES (?, ?, ?, ?, ?, 'claimed')",
|
|
5378
|
+
args: [idempotencyKey, userId, mc, operation, description]
|
|
5379
|
+
},
|
|
5380
|
+
{
|
|
5381
|
+
sql: "UPDATE users SET balance_mc = balance_mc - ? WHERE id = ? AND balance_mc >= ? AND EXISTS (SELECT 1 FROM billing_debits WHERE idempotency_key = ? AND user_id = ? AND amount_mc = ? AND operation = ? AND description = ? AND status = 'claimed')",
|
|
5382
|
+
args: [mc, userId, mc, idempotencyKey, userId, mc, operation, description]
|
|
5383
|
+
},
|
|
5384
|
+
{
|
|
5385
|
+
sql: "UPDATE billing_debits SET status = 'applying' WHERE idempotency_key = ? AND user_id = ? AND amount_mc = ? AND operation = ? AND description = ? AND status = 'claimed' AND changes() = 1",
|
|
5386
|
+
args: [idempotencyKey, userId, mc, operation, description]
|
|
5387
|
+
},
|
|
5388
|
+
{
|
|
5389
|
+
sql: `WITH ordered AS MATERIALIZED (
|
|
5390
|
+
SELECT id,
|
|
5391
|
+
remaining_mc,
|
|
5392
|
+
COALESCE(SUM(remaining_mc) OVER (
|
|
5393
|
+
ORDER BY expires_at ASC, granted_at ASC, id ASC
|
|
5394
|
+
ROWS BETWEEN UNBOUNDED PRECEDING AND 1 PRECEDING
|
|
5395
|
+
), 0) AS prior_mc
|
|
5396
|
+
FROM credit_lots
|
|
5397
|
+
WHERE user_id = ? AND remaining_mc > 0 AND expires_at > datetime('now')
|
|
5398
|
+
), allocation AS (
|
|
5399
|
+
SELECT id, MIN(remaining_mc, MAX(0, ? - prior_mc)) AS take_mc
|
|
5400
|
+
FROM ordered
|
|
5401
|
+
)
|
|
5402
|
+
UPDATE credit_lots
|
|
5403
|
+
SET remaining_mc = remaining_mc - COALESCE((SELECT take_mc FROM allocation WHERE allocation.id = credit_lots.id), 0)
|
|
5404
|
+
WHERE id IN (SELECT id FROM allocation WHERE take_mc > 0)
|
|
5405
|
+
AND EXISTS (SELECT 1 FROM billing_debits WHERE idempotency_key = ? AND status = 'applying')`,
|
|
5406
|
+
args: [userId, mc, idempotencyKey]
|
|
5407
|
+
},
|
|
5408
|
+
{
|
|
5409
|
+
sql: "INSERT INTO ledger (user_id, amount_mc, operation, description) SELECT user_id, -amount_mc, operation, description FROM billing_debits WHERE idempotency_key = ? AND status = 'applying'",
|
|
5410
|
+
args: [idempotencyKey]
|
|
5411
|
+
},
|
|
5412
|
+
{
|
|
5413
|
+
sql: "UPDATE billing_debits SET status = 'applied', applied_at = datetime('now') WHERE idempotency_key = ? AND status = 'applying'",
|
|
5414
|
+
args: [idempotencyKey]
|
|
5415
|
+
}
|
|
5416
|
+
], "write");
|
|
5417
|
+
} catch (error) {
|
|
5418
|
+
try {
|
|
5419
|
+
const recovered = await db.execute({
|
|
5420
|
+
sql: "SELECT user_id, amount_mc, operation, description, status FROM billing_debits WHERE idempotency_key = ? AND status = 'applied' LIMIT 1",
|
|
5421
|
+
args: [idempotencyKey]
|
|
5422
|
+
});
|
|
5423
|
+
const row2 = recovered.rows[0];
|
|
5424
|
+
if (row2 && Number(row2.user_id) === userId && Number(row2.amount_mc) === mc && String(row2.operation) === operation && String(row2.description) === description) {
|
|
5425
|
+
const balance2 = await db.execute({ sql: "SELECT balance_mc FROM users WHERE id = ?", args: [userId] });
|
|
5426
|
+
return { ok: true, balance_mc: Number(balance2.rows[0]?.balance_mc ?? 0), duplicate: true };
|
|
5427
|
+
}
|
|
5428
|
+
} catch {
|
|
5429
|
+
}
|
|
5430
|
+
throw error;
|
|
5431
|
+
}
|
|
5432
|
+
const claim = await db.execute({
|
|
5433
|
+
sql: "SELECT user_id, amount_mc, operation, description, status FROM billing_debits WHERE idempotency_key = ? LIMIT 1",
|
|
5434
|
+
args: [idempotencyKey]
|
|
5435
|
+
});
|
|
5436
|
+
const row = claim.rows[0];
|
|
5437
|
+
if (!row || Number(row.user_id) !== userId || Number(row.amount_mc) !== mc || String(row.operation) !== operation || String(row.description) !== description) {
|
|
5438
|
+
throw new Error("idempotent debit key conflicts with another debit");
|
|
5439
|
+
}
|
|
5440
|
+
const balance = await db.execute({ sql: "SELECT balance_mc FROM users WHERE id = ?", args: [userId] });
|
|
5441
|
+
return {
|
|
5442
|
+
ok: String(row.status) === "applied",
|
|
5443
|
+
balance_mc: Number(balance.rows[0]?.balance_mc ?? 0),
|
|
5444
|
+
duplicate: results[2]?.rowsAffected === 0 && String(row.status) === "applied"
|
|
5445
|
+
};
|
|
5446
|
+
}
|
|
5447
|
+
async function settleDebitMcIdempotent(userId, debitIdempotencyKey, finalAmountMc, refundOperation, description, sourceSurface) {
|
|
5448
|
+
if (!Number.isSafeInteger(finalAmountMc) || finalAmountMc < 0) {
|
|
5449
|
+
throw new Error("idempotent debit settlement amount must be a non-negative integer");
|
|
5450
|
+
}
|
|
5451
|
+
if (!debitIdempotencyKey || debitIdempotencyKey.length > 500) {
|
|
5452
|
+
throw new Error("idempotent debit settlement key is required and must be bounded");
|
|
5453
|
+
}
|
|
5454
|
+
if (!refundOperation || !sourceSurface) {
|
|
5455
|
+
throw new Error("idempotent debit settlement operation and source are required");
|
|
5456
|
+
}
|
|
5457
|
+
const db = getDb();
|
|
5458
|
+
await ensureMigrated(userId);
|
|
5459
|
+
const settlementKey = `debit-settlement:${(0, import_node_crypto.createHash)("sha256").update(debitIdempotencyKey).digest("hex")}`;
|
|
5460
|
+
const billingClass = "wallet_hold_settlement";
|
|
5461
|
+
const eventId = (0, import_node_crypto.randomUUID)();
|
|
5462
|
+
const metadata = JSON.stringify({ debitIdempotencyKey, refundOperation, description });
|
|
5463
|
+
const readReceipt = async () => {
|
|
5464
|
+
const receipt2 = await db.execute({
|
|
5465
|
+
sql: `SELECT e.user_id,
|
|
5466
|
+
e.status,
|
|
5467
|
+
e.amount_mc AS final_amount_mc,
|
|
5468
|
+
e.billing_class,
|
|
5469
|
+
e.source_surface,
|
|
5470
|
+
d.amount_mc AS held_amount_mc
|
|
5471
|
+
FROM billing_events e
|
|
5472
|
+
JOIN billing_debits d ON d.idempotency_key = ?
|
|
5473
|
+
WHERE e.idempotency_key = ?
|
|
5474
|
+
AND d.user_id = e.user_id
|
|
5475
|
+
AND d.status = 'applied'
|
|
5476
|
+
LIMIT 1`,
|
|
5477
|
+
args: [debitIdempotencyKey, settlementKey]
|
|
5478
|
+
});
|
|
5479
|
+
const row = receipt2.rows[0];
|
|
5480
|
+
if (!row) return null;
|
|
5481
|
+
if (Number(row.user_id) !== userId || String(row.billing_class) !== billingClass || String(row.source_surface) !== sourceSurface) {
|
|
5482
|
+
throw new Error("idempotent debit settlement key conflicts with another settlement");
|
|
5483
|
+
}
|
|
5484
|
+
if (String(row.status) !== "settled") return null;
|
|
5485
|
+
const heldAmountMc = Number(row.held_amount_mc);
|
|
5486
|
+
const durableFinalAmountMc = Number(row.final_amount_mc);
|
|
5487
|
+
if (!Number.isSafeInteger(heldAmountMc) || !Number.isSafeInteger(durableFinalAmountMc) || durableFinalAmountMc < 0 || durableFinalAmountMc > heldAmountMc) {
|
|
5488
|
+
throw new Error("idempotent debit settlement receipt is invalid");
|
|
5489
|
+
}
|
|
5490
|
+
const balance = await db.execute({ sql: "SELECT balance_mc FROM users WHERE id = ?", args: [userId] });
|
|
5491
|
+
return {
|
|
5492
|
+
balance_mc: Number(balance.rows[0]?.balance_mc ?? 0),
|
|
5493
|
+
final_amount_mc: durableFinalAmountMc,
|
|
5494
|
+
refund_mc: heldAmountMc - durableFinalAmountMc,
|
|
5495
|
+
duplicate: true
|
|
5496
|
+
};
|
|
5497
|
+
};
|
|
5498
|
+
let results;
|
|
5499
|
+
try {
|
|
5500
|
+
results = await db.batch([
|
|
5501
|
+
{
|
|
5502
|
+
sql: `INSERT OR IGNORE INTO billing_events
|
|
5503
|
+
(id, user_id, idempotency_key, billing_class, source_surface, status, amount_mc, metadata)
|
|
5504
|
+
VALUES (?, ?, ?, ?, ?, 'settling', ?, ?)`,
|
|
5505
|
+
args: [eventId, userId, settlementKey, billingClass, sourceSurface, finalAmountMc, metadata]
|
|
5506
|
+
},
|
|
5507
|
+
{
|
|
5508
|
+
sql: `UPDATE users
|
|
5509
|
+
SET balance_mc = balance_mc + (
|
|
5510
|
+
SELECT d.amount_mc - e.amount_mc
|
|
5511
|
+
FROM billing_events e
|
|
5512
|
+
JOIN billing_debits d ON d.idempotency_key = ?
|
|
5513
|
+
WHERE e.idempotency_key = ?
|
|
5514
|
+
)
|
|
5515
|
+
WHERE id = ?
|
|
5516
|
+
AND EXISTS (
|
|
5517
|
+
SELECT 1
|
|
5518
|
+
FROM billing_events e
|
|
5519
|
+
JOIN billing_debits d ON d.idempotency_key = ?
|
|
5520
|
+
WHERE e.idempotency_key = ?
|
|
5521
|
+
AND e.user_id = ?
|
|
5522
|
+
AND e.billing_class = ?
|
|
5523
|
+
AND e.source_surface = ?
|
|
5524
|
+
AND e.status = 'settling'
|
|
5525
|
+
AND d.user_id = e.user_id
|
|
5526
|
+
AND d.status = 'applied'
|
|
5527
|
+
AND e.amount_mc BETWEEN 0 AND d.amount_mc
|
|
5528
|
+
)`,
|
|
5529
|
+
args: [
|
|
5530
|
+
debitIdempotencyKey,
|
|
5531
|
+
settlementKey,
|
|
5532
|
+
userId,
|
|
5533
|
+
debitIdempotencyKey,
|
|
5534
|
+
settlementKey,
|
|
5535
|
+
userId,
|
|
5536
|
+
billingClass,
|
|
5537
|
+
sourceSurface
|
|
5538
|
+
]
|
|
5539
|
+
},
|
|
5540
|
+
{
|
|
5541
|
+
sql: `UPDATE billing_events
|
|
5542
|
+
SET status = 'applying'
|
|
5543
|
+
WHERE idempotency_key = ?
|
|
5544
|
+
AND user_id = ?
|
|
5545
|
+
AND billing_class = ?
|
|
5546
|
+
AND source_surface = ?
|
|
5547
|
+
AND status = 'settling'
|
|
5548
|
+
AND changes() = 1`,
|
|
5549
|
+
args: [settlementKey, userId, billingClass, sourceSurface]
|
|
5550
|
+
},
|
|
5551
|
+
{
|
|
5552
|
+
sql: `INSERT INTO credit_lots
|
|
5553
|
+
(user_id, amount_mc, remaining_mc, source, expires_at)
|
|
5554
|
+
SELECT e.user_id,
|
|
5555
|
+
d.amount_mc - e.amount_mc,
|
|
5556
|
+
d.amount_mc - e.amount_mc,
|
|
5557
|
+
?,
|
|
5558
|
+
datetime('now', ?)
|
|
5559
|
+
FROM billing_events e
|
|
5560
|
+
JOIN billing_debits d ON d.idempotency_key = ?
|
|
5561
|
+
WHERE e.idempotency_key = ?
|
|
5562
|
+
AND e.status = 'applying'
|
|
5563
|
+
AND d.amount_mc > e.amount_mc`,
|
|
5564
|
+
args: [refundOperation, CREDIT_LOT_TTL, debitIdempotencyKey, settlementKey]
|
|
5565
|
+
},
|
|
5566
|
+
{
|
|
5567
|
+
sql: `INSERT INTO ledger (user_id, amount_mc, operation, description)
|
|
5568
|
+
SELECT e.user_id,
|
|
5569
|
+
d.amount_mc - e.amount_mc,
|
|
5570
|
+
?,
|
|
5571
|
+
?
|
|
5572
|
+
FROM billing_events e
|
|
5573
|
+
JOIN billing_debits d ON d.idempotency_key = ?
|
|
5574
|
+
WHERE e.idempotency_key = ?
|
|
5575
|
+
AND e.status = 'applying'
|
|
5576
|
+
AND d.amount_mc > e.amount_mc`,
|
|
5577
|
+
args: [refundOperation, description, debitIdempotencyKey, settlementKey]
|
|
5578
|
+
},
|
|
5579
|
+
{
|
|
5580
|
+
sql: `UPDATE billing_events
|
|
5581
|
+
SET status = 'settled', settled_at = datetime('now')
|
|
5582
|
+
WHERE idempotency_key = ?
|
|
5583
|
+
AND status = 'applying'`,
|
|
5584
|
+
args: [settlementKey]
|
|
5585
|
+
}
|
|
5586
|
+
], "write");
|
|
5587
|
+
} catch (error) {
|
|
5588
|
+
try {
|
|
5589
|
+
const recovered = await readReceipt();
|
|
5590
|
+
if (recovered) return recovered;
|
|
5591
|
+
} catch {
|
|
5592
|
+
}
|
|
5593
|
+
throw error;
|
|
5594
|
+
}
|
|
5595
|
+
const receipt = await readReceipt();
|
|
5596
|
+
if (!receipt) throw new Error("idempotent debit settlement did not reach a durable settled state");
|
|
5597
|
+
return { ...receipt, duplicate: results[0]?.rowsAffected === 0 };
|
|
5598
|
+
}
|
|
5292
5599
|
async function getLedger(userId, limit = 50) {
|
|
5293
5600
|
const res = await getDb().execute({
|
|
5294
5601
|
sql: "SELECT * FROM ledger WHERE user_id = ? ORDER BY created_at DESC LIMIT ?",
|
|
@@ -5337,7 +5644,7 @@ async function reconcileBalanceMc(userId, knownBalanceMc) {
|
|
|
5337
5644
|
}
|
|
5338
5645
|
return balanceMc;
|
|
5339
5646
|
}
|
|
5340
|
-
var import_http, import_node_crypto, import_zod, DB_URL, DB_TOKEN, _db, _rateLimitSchemaReady, CREDIT_LOT_TTL, SiteAuditJobRowSchema, SiteAuditPhaseLogRowSchema;
|
|
5647
|
+
var import_http, import_node_crypto, import_zod, DB_URL, DB_TOKEN, _db, _rateLimitSchemaReady, CORE_SCHEMA_VERSION, CORE_SCHEMA_TABLES, CORE_SCHEMA_INDEXES, CORE_SCHEMA_COLUMNS, REQUEST_EVENT_RESULT_MAX_BYTES, CREDIT_LOT_TTL, SiteAuditJobRowSchema, SiteAuditPhaseLogRowSchema;
|
|
5341
5648
|
var init_db = __esm({
|
|
5342
5649
|
"src/api/db.ts"() {
|
|
5343
5650
|
"use strict";
|
|
@@ -5348,6 +5655,95 @@ var init_db = __esm({
|
|
|
5348
5655
|
DB_TOKEN = process.env.TURSO_AUTH_TOKEN;
|
|
5349
5656
|
_db = null;
|
|
5350
5657
|
_rateLimitSchemaReady = false;
|
|
5658
|
+
CORE_SCHEMA_VERSION = "2026-07-14.1";
|
|
5659
|
+
CORE_SCHEMA_TABLES = [
|
|
5660
|
+
"billing_authorizations",
|
|
5661
|
+
"billing_debits",
|
|
5662
|
+
"billing_events",
|
|
5663
|
+
"concurrency_locks",
|
|
5664
|
+
"credit_lots",
|
|
5665
|
+
"free_credit_refreshes",
|
|
5666
|
+
"harvest_attempts",
|
|
5667
|
+
"jobs",
|
|
5668
|
+
"kpo_jobs",
|
|
5669
|
+
"kpo_phase_log",
|
|
5670
|
+
"ledger",
|
|
5671
|
+
"oauth_clients",
|
|
5672
|
+
"oauth_codes",
|
|
5673
|
+
"oauth_tokens",
|
|
5674
|
+
"page_snapshots",
|
|
5675
|
+
"password_reset_tokens",
|
|
5676
|
+
"rate_limits",
|
|
5677
|
+
"request_events",
|
|
5678
|
+
"site_audit_jobs",
|
|
5679
|
+
"site_audit_phase_log",
|
|
5680
|
+
"site_extract_jobs",
|
|
5681
|
+
"site_extract_pages",
|
|
5682
|
+
"stripe_events",
|
|
5683
|
+
"users",
|
|
5684
|
+
"workflow_artifacts",
|
|
5685
|
+
"workflow_runs",
|
|
5686
|
+
"workflow_schedules",
|
|
5687
|
+
"workflow_webhook_deliveries"
|
|
5688
|
+
];
|
|
5689
|
+
CORE_SCHEMA_INDEXES = [
|
|
5690
|
+
"billing_authorizations_user_status",
|
|
5691
|
+
"billing_debits_user_status",
|
|
5692
|
+
"billing_events_authorization",
|
|
5693
|
+
"billing_events_user_created_at",
|
|
5694
|
+
"concurrency_locks_operation",
|
|
5695
|
+
"concurrency_locks_user_status",
|
|
5696
|
+
"credit_lots_user_active",
|
|
5697
|
+
"harvest_attempts_job_id",
|
|
5698
|
+
"harvest_attempts_user_created_at",
|
|
5699
|
+
"jobs_status",
|
|
5700
|
+
"jobs_user_id",
|
|
5701
|
+
"kpo_jobs_status",
|
|
5702
|
+
"kpo_jobs_user_id",
|
|
5703
|
+
"kpo_phase_log_job_id",
|
|
5704
|
+
"ledger_stripe_pi_unique",
|
|
5705
|
+
"ledger_user_id",
|
|
5706
|
+
"oauth_tokens_identity",
|
|
5707
|
+
"page_snapshots_user_checked_at",
|
|
5708
|
+
"prt_user_id",
|
|
5709
|
+
"request_events_user_created_at",
|
|
5710
|
+
"site_audit_jobs_status",
|
|
5711
|
+
"site_audit_jobs_user_id",
|
|
5712
|
+
"site_audit_phase_log_job_id",
|
|
5713
|
+
"site_extract_jobs_status",
|
|
5714
|
+
"site_extract_jobs_user_id",
|
|
5715
|
+
"site_extract_pages_job_id",
|
|
5716
|
+
"workflow_artifacts_run",
|
|
5717
|
+
"workflow_runs_idempotency",
|
|
5718
|
+
"workflow_runs_schedule",
|
|
5719
|
+
"workflow_runs_user_created",
|
|
5720
|
+
"workflow_schedules_due",
|
|
5721
|
+
"workflow_schedules_user_status",
|
|
5722
|
+
"workflow_webhook_deliveries_run"
|
|
5723
|
+
];
|
|
5724
|
+
CORE_SCHEMA_COLUMNS = {
|
|
5725
|
+
harvest_attempts: ["debug_json"],
|
|
5726
|
+
oauth_tokens: ["replaced_by", "rotated_at"],
|
|
5727
|
+
users: [
|
|
5728
|
+
"balance_mc",
|
|
5729
|
+
"concurrency_stripe_sub_id",
|
|
5730
|
+
"extra_concurrency_slots",
|
|
5731
|
+
"key_active",
|
|
5732
|
+
"memory_identity",
|
|
5733
|
+
"memory_key",
|
|
5734
|
+
"memory_plan",
|
|
5735
|
+
"memory_provisioned",
|
|
5736
|
+
"memory_subscription_id",
|
|
5737
|
+
"password_hash",
|
|
5738
|
+
"stripe_customer_id",
|
|
5739
|
+
"subscription_concurrency",
|
|
5740
|
+
"subscription_id",
|
|
5741
|
+
"subscription_tier"
|
|
5742
|
+
],
|
|
5743
|
+
workflow_artifacts: ["content"],
|
|
5744
|
+
workflow_runs: ["step_index", "step_log_json", "step_state_json", "total_steps", "webhook_url"]
|
|
5745
|
+
};
|
|
5746
|
+
REQUEST_EVENT_RESULT_MAX_BYTES = 32 * 1024;
|
|
5351
5747
|
CREDIT_LOT_TTL = "+3 months";
|
|
5352
5748
|
SiteAuditJobRowSchema = import_zod.z.object({
|
|
5353
5749
|
id: import_zod.z.string(),
|
|
@@ -5419,7 +5815,26 @@ var init_cost_rates = __esm({
|
|
|
5419
5815
|
// src/api/cost-telemetry.ts
|
|
5420
5816
|
async function migrateCostTelemetry() {
|
|
5421
5817
|
if (_ready) return;
|
|
5818
|
+
if (_migrationPromise) return _migrationPromise;
|
|
5819
|
+
_migrationPromise = runCostTelemetryMigration().finally(() => {
|
|
5820
|
+
_migrationPromise = null;
|
|
5821
|
+
});
|
|
5822
|
+
return _migrationPromise;
|
|
5823
|
+
}
|
|
5824
|
+
async function runCostTelemetryMigration() {
|
|
5422
5825
|
const db = getDb();
|
|
5826
|
+
const existing = await db.execute(`
|
|
5827
|
+
SELECT
|
|
5828
|
+
(SELECT COUNT(*) FROM sqlite_master WHERE type = 'table' AND name IN ('kernel_session_log', 'vendor_usage_log', 'cost_probe_runs')) = 3
|
|
5829
|
+
AND (SELECT COUNT(*) FROM pragma_table_info('kernel_session_log') WHERE name IN ('proxy_source', 'proxy_type', 'method')) = 3
|
|
5830
|
+
AND (SELECT COUNT(*) FROM pragma_table_info('vendor_usage_log') WHERE name = 'method') = 1
|
|
5831
|
+
AND (SELECT COUNT(*) FROM pragma_table_info('cost_probe_runs') WHERE name IN ('units', 'unit_type', 'mode')) = 3
|
|
5832
|
+
AS ready
|
|
5833
|
+
`);
|
|
5834
|
+
if (Number(existing.rows[0]?.ready ?? 0) === 1) {
|
|
5835
|
+
_ready = true;
|
|
5836
|
+
return;
|
|
5837
|
+
}
|
|
5423
5838
|
await db.execute(`
|
|
5424
5839
|
CREATE TABLE IF NOT EXISTS kernel_session_log (
|
|
5425
5840
|
id TEXT PRIMARY KEY,
|
|
@@ -5583,7 +5998,7 @@ function boolToInt(v) {
|
|
|
5583
5998
|
if (v === null || v === void 0) return null;
|
|
5584
5999
|
return v ? 1 : 0;
|
|
5585
6000
|
}
|
|
5586
|
-
var import_node_crypto2, _ready;
|
|
6001
|
+
var import_node_crypto2, _ready, _migrationPromise;
|
|
5587
6002
|
var init_cost_telemetry = __esm({
|
|
5588
6003
|
"src/api/cost-telemetry.ts"() {
|
|
5589
6004
|
"use strict";
|
|
@@ -5592,6 +6007,7 @@ var init_cost_telemetry = __esm({
|
|
|
5592
6007
|
init_cost_context();
|
|
5593
6008
|
init_cost_rates();
|
|
5594
6009
|
_ready = false;
|
|
6010
|
+
_migrationPromise = null;
|
|
5595
6011
|
}
|
|
5596
6012
|
});
|
|
5597
6013
|
|
|
@@ -8483,6 +8899,9 @@ var init_schemas = __esm({
|
|
|
8483
8899
|
});
|
|
8484
8900
|
|
|
8485
8901
|
// src/api/rates.ts
|
|
8902
|
+
function serpActualCostMc(headlessSent) {
|
|
8903
|
+
return headlessSent === true ? MC_COSTS.serp_headless : MC_COSTS.serp_headful;
|
|
8904
|
+
}
|
|
8486
8905
|
function browserActiveCostMc(activeMs) {
|
|
8487
8906
|
return Math.round(activeMs * MC_PER_BROWSER_MS);
|
|
8488
8907
|
}
|
|
@@ -8496,6 +8915,18 @@ function vendorCostUsdToMc(costUsd, markupBps) {
|
|
|
8496
8915
|
if (mc > BigInt(Number.MAX_SAFE_INTEGER)) throw new Error("calculated mc exceeds the safe integer range");
|
|
8497
8916
|
return Number(mc);
|
|
8498
8917
|
}
|
|
8918
|
+
function getPublicRatesPayload() {
|
|
8919
|
+
const starter = SUBSCRIPTION_TIER_BY_KEY.starter;
|
|
8920
|
+
const costs = CREDIT_COST_CATALOG.map(({ aliases, ...cost }) => ({
|
|
8921
|
+
...cost,
|
|
8922
|
+
...cost.notes ? { notes: cost.notes.replace(/ via fal\.ai/g, "") } : {}
|
|
8923
|
+
}));
|
|
8924
|
+
return {
|
|
8925
|
+
creditsPerDollarStarter: Math.round(mcToCredits(starter.credits_mc) / starter.monthly_usd),
|
|
8926
|
+
costs,
|
|
8927
|
+
scheduling: SCHEDULED_ACTIONS_RATE_POLICY
|
|
8928
|
+
};
|
|
8929
|
+
}
|
|
8499
8930
|
function concurrencySlotBillingInfo() {
|
|
8500
8931
|
const billingUrl = process.env.CONCURRENCY_BILLING_URL ?? "https://mcpscraper.dev/billing";
|
|
8501
8932
|
return {
|
|
@@ -8525,12 +8956,14 @@ function insufficientBalanceResponse(balanceMc, requiredMc) {
|
|
|
8525
8956
|
topup_url: topupUrl
|
|
8526
8957
|
};
|
|
8527
8958
|
}
|
|
8528
|
-
var MC_COSTS, MC_PER_BROWSER_MS, BROWSER_OPEN_MIN_BALANCE_MC, MC_PER_CREDIT, BILLING_MC_PER_USD_NUMERATOR, BILLING_MC_PER_USD_DENOMINATOR, BILLING_BASIS_POINTS, USD_NANOS, SCHEDULED_RUN_BASE_CREDITS, SCHEDULED_RUN_BASE_MC, SCHEDULED_AGENT_LLM_MARKUP_BPS, SCHEDULED_AGENT_LLM_MARKUP, SCHEDULED_RUN_DEFAULT_MAX_CREDITS, CREDIT_COST_CATALOG, CONCURRENCY_PRICE_ID, SUBSCRIPTION_TIERS, SUBSCRIPTION_TIER_BY_KEY, CONCURRENCY_SLOT_PRICE_USD, CONCURRENCY_SLOT_PRICE_CURRENCY, CONCURRENCY_SLOT_PRICE_INTERVAL, CONCURRENCY_SLOT_PRICE_LABEL, CONCURRENCY_SLOT_TERMINAL_COMMAND,
|
|
8959
|
+
var MC_COSTS, MC_PER_BROWSER_MS, BROWSER_OPEN_MIN_BALANCE_MC, MC_PER_CREDIT, BILLING_MC_PER_USD_NUMERATOR, BILLING_MC_PER_USD_DENOMINATOR, BILLING_BASIS_POINTS, USD_NANOS, SCHEDULED_RUN_BASE_CREDITS, SCHEDULED_RUN_BASE_MC, SCHEDULED_AGENT_LLM_MARKUP_BPS, SCHEDULED_AGENT_LLM_MARKUP, SCHEDULED_RUN_DEFAULT_MAX_CREDITS, SCHEDULED_ACTIONS_RATE_POLICY, CREDIT_COST_CATALOG, CONCURRENCY_PRICE_ID, SUBSCRIPTION_TIERS, SUBSCRIPTION_TIER_BY_KEY, CONCURRENCY_SLOT_PRICE_USD, CONCURRENCY_SLOT_PRICE_CURRENCY, CONCURRENCY_SLOT_PRICE_INTERVAL, CONCURRENCY_SLOT_PRICE_LABEL, CONCURRENCY_SLOT_TERMINAL_COMMAND, MEMORY_FREE_QUOTA_BYTES, MEMORY_PLANS, MEMORY_PLAN_QUOTA, MEMORY_MARGIN_MULTIPLE, MEMORY_PRO_COST_BUDGET_USD, LedgerOperation, MEMORY_AI_MARGIN_MULTIPLE, SITE_AUDIT_LLM_MARGIN_MULTIPLE, VIDEO_ANALYSIS_LLM_MARGIN_MULTIPLE, MEDIA_TRANSCRIBE_MARGIN_MULTIPLE, MEDIA_TRANSCRIBE_HOLD_MC, MC_PER_USD;
|
|
8529
8960
|
var init_rates = __esm({
|
|
8530
8961
|
"src/api/rates.ts"() {
|
|
8531
8962
|
"use strict";
|
|
8532
8963
|
MC_COSTS = {
|
|
8533
|
-
serp:
|
|
8964
|
+
serp: 400,
|
|
8965
|
+
serp_headless: 400,
|
|
8966
|
+
serp_headful: 1400,
|
|
8534
8967
|
paa: 300,
|
|
8535
8968
|
paa_base: 5500,
|
|
8536
8969
|
page_scrape: 100,
|
|
@@ -8571,14 +9004,38 @@ var init_rates = __esm({
|
|
|
8571
9004
|
SCHEDULED_AGENT_LLM_MARKUP_BPS = 15e3;
|
|
8572
9005
|
SCHEDULED_AGENT_LLM_MARKUP = SCHEDULED_AGENT_LLM_MARKUP_BPS / BILLING_BASIS_POINTS;
|
|
8573
9006
|
SCHEDULED_RUN_DEFAULT_MAX_CREDITS = 5e3;
|
|
9007
|
+
SCHEDULED_ACTIONS_RATE_POLICY = {
|
|
9008
|
+
billingMode: "credits",
|
|
9009
|
+
entitlement: "paid_plan_included",
|
|
9010
|
+
separateSubscriptionRequired: false,
|
|
9011
|
+
chargeEvent: "started_occurrence",
|
|
9012
|
+
runBaseCredits: SCHEDULED_RUN_BASE_CREDITS,
|
|
9013
|
+
runBaseMc: SCHEDULED_RUN_BASE_MC,
|
|
9014
|
+
llmCostMultiplier: SCHEDULED_AGENT_LLM_MARKUP,
|
|
9015
|
+
llmCostMultiplierBps: SCHEDULED_AGENT_LLM_MARKUP_BPS,
|
|
9016
|
+
modes: {
|
|
9017
|
+
connection_sync: {
|
|
9018
|
+
runBaseCredits: SCHEDULED_RUN_BASE_CREDITS,
|
|
9019
|
+
modelCostBilling: "none",
|
|
9020
|
+
modelCostMultiplier: 0
|
|
9021
|
+
},
|
|
9022
|
+
agent: {
|
|
9023
|
+
runBaseCredits: SCHEDULED_RUN_BASE_CREDITS,
|
|
9024
|
+
modelCostBilling: "actual_reported_cost",
|
|
9025
|
+
modelCostProvider: "openrouter",
|
|
9026
|
+
modelCostMultiplier: SCHEDULED_AGENT_LLM_MARKUP,
|
|
9027
|
+
modelCostMultiplierBps: SCHEDULED_AGENT_LLM_MARKUP_BPS
|
|
9028
|
+
}
|
|
9029
|
+
}
|
|
9030
|
+
};
|
|
8574
9031
|
CREDIT_COST_CATALOG = [
|
|
8575
9032
|
{
|
|
8576
9033
|
key: "serp",
|
|
8577
9034
|
label: "SERP search",
|
|
8578
9035
|
aliases: ["search_serp", "serp", "google search", "organic results"],
|
|
8579
|
-
credits: mcToCredits(MC_COSTS.
|
|
8580
|
-
unit: "per search",
|
|
8581
|
-
notes:
|
|
9036
|
+
credits: mcToCredits(MC_COSTS.serp_headless),
|
|
9037
|
+
unit: "per search (headless); escalates to headful only when needed",
|
|
9038
|
+
notes: `Returns AI Overview, PAA snippet, videos, forums, and local pack. Billed ${mcToCredits(MC_COSTS.serp_headless)} credits when served headless (the common case), ${mcToCredits(MC_COSTS.serp_headful)} credits if an anti-bot escalation forces headful mode (overestimate refunded). Optional SERP Intelligence page snapshots add ${mcToCredits(MC_COSTS.page_scrape)} credit per attempted URL.`
|
|
8582
9039
|
},
|
|
8583
9040
|
{
|
|
8584
9041
|
key: "paa",
|
|
@@ -8801,7 +9258,6 @@ var init_rates = __esm({
|
|
|
8801
9258
|
CONCURRENCY_SLOT_PRICE_INTERVAL = "month";
|
|
8802
9259
|
CONCURRENCY_SLOT_PRICE_LABEL = "$5/month";
|
|
8803
9260
|
CONCURRENCY_SLOT_TERMINAL_COMMAND = "npx -y -p mcp-scraper@latest mcp-scraper-cli billing concurrency checkout";
|
|
8804
|
-
FREE_SIGNUP_MC = 1e5;
|
|
8805
9261
|
MEMORY_FREE_QUOTA_BYTES = 1e7 * 1;
|
|
8806
9262
|
MEMORY_PLANS = {
|
|
8807
9263
|
"price_1TnMSTS8aAcsk3TGBgiwuvqL": { plan: "pro", label: "Pro", price_id: "price_1TnMSTS8aAcsk3TGBgiwuvqL", interval: "month", monthly_usd: 19, quota_bytes: 5e9 },
|
|
@@ -8818,6 +9274,7 @@ var init_rates = __esm({
|
|
|
8818
9274
|
PAA: "paa",
|
|
8819
9275
|
PAA_REFUND: "paa_refund",
|
|
8820
9276
|
SERP: "serp",
|
|
9277
|
+
SERP_REFUND: "serp_refund",
|
|
8821
9278
|
REFUND: "refund",
|
|
8822
9279
|
TRANSCRIPTION: "transcription",
|
|
8823
9280
|
TRANSCRIPTION_HOLD: "transcription_hold",
|
|
@@ -11901,8 +12358,17 @@ var init_internal_memory_routes = __esm({
|
|
|
11901
12358
|
});
|
|
11902
12359
|
|
|
11903
12360
|
// src/api/scheduling-access.ts
|
|
12361
|
+
function schedulingAccessOverrides() {
|
|
12362
|
+
return new Set(
|
|
12363
|
+
(process.env.SCHEDULING_ACCESS_OVERRIDE_IDENTITIES ?? "").split(",").map((identity) => identity.trim().toLowerCase()).filter(Boolean)
|
|
12364
|
+
);
|
|
12365
|
+
}
|
|
11904
12366
|
function hasPaidSchedulingAccess(user) {
|
|
11905
|
-
|
|
12367
|
+
const tier = typeof user.subscription_tier === "string" ? user.subscription_tier.trim() : "";
|
|
12368
|
+
const subscriptionId = typeof user.subscription_id === "string" ? user.subscription_id.trim() : "";
|
|
12369
|
+
if (tier && subscriptionId) return true;
|
|
12370
|
+
const identity = typeof user.email === "string" ? user.email.trim().toLowerCase() : "";
|
|
12371
|
+
return identity.length > 0 && schedulingAccessOverrides().has(identity);
|
|
11906
12372
|
}
|
|
11907
12373
|
function paidPlanRequiredPayload() {
|
|
11908
12374
|
return {
|
|
@@ -11995,6 +12461,15 @@ function safeMetadata(value) {
|
|
|
11995
12461
|
if (!value) return "{}";
|
|
11996
12462
|
return JSON.stringify(value).slice(0, 4e3);
|
|
11997
12463
|
}
|
|
12464
|
+
function metadataReason(row) {
|
|
12465
|
+
if (!row.metadata) return void 0;
|
|
12466
|
+
try {
|
|
12467
|
+
const metadata = JSON.parse(row.metadata);
|
|
12468
|
+
return typeof metadata.reason === "string" ? metadata.reason : void 0;
|
|
12469
|
+
} catch {
|
|
12470
|
+
return void 0;
|
|
12471
|
+
}
|
|
12472
|
+
}
|
|
11998
12473
|
function mapAuthorization(row, balanceMc) {
|
|
11999
12474
|
return {
|
|
12000
12475
|
id: row.id,
|
|
@@ -12023,23 +12498,28 @@ async function eventByKey(idempotencyKey) {
|
|
|
12023
12498
|
});
|
|
12024
12499
|
return result.rows[0] ? result.rows[0] : null;
|
|
12025
12500
|
}
|
|
12026
|
-
function
|
|
12027
|
-
|
|
12028
|
-
|
|
12029
|
-
|
|
12030
|
-
|
|
12031
|
-
|
|
12032
|
-
|
|
12033
|
-
|
|
12034
|
-
|
|
12501
|
+
function mapStartEvent(row, balanceMc, duplicate) {
|
|
12502
|
+
return {
|
|
12503
|
+
id: row.id,
|
|
12504
|
+
authorizationId: row.authorization_id,
|
|
12505
|
+
status: "started",
|
|
12506
|
+
amountMc: SCHEDULED_RUN_BASE_MC,
|
|
12507
|
+
credits: SCHEDULED_RUN_BASE_MC / MC_PER_CREDIT,
|
|
12508
|
+
balanceMc,
|
|
12509
|
+
duplicate
|
|
12510
|
+
};
|
|
12511
|
+
}
|
|
12512
|
+
function mapSettlementEvent(row, balanceMc, duplicate) {
|
|
12035
12513
|
const rawNanos = row.raw_cost_usd_nanos == null ? null : Number(row.raw_cost_usd_nanos);
|
|
12036
|
-
const modelMc = Math.max(0, Number(row.amount_mc)
|
|
12514
|
+
const modelMc = Math.max(0, Number(row.amount_mc));
|
|
12515
|
+
const amountMc = SCHEDULED_RUN_BASE_MC + modelMc;
|
|
12516
|
+
const reason = metadataReason(row);
|
|
12037
12517
|
return {
|
|
12038
12518
|
id: row.id,
|
|
12039
12519
|
authorizationId: row.authorization_id,
|
|
12040
12520
|
status: row.status === "settled" ? "settled" : "cost_pending",
|
|
12041
|
-
amountMc
|
|
12042
|
-
credits:
|
|
12521
|
+
amountMc,
|
|
12522
|
+
credits: amountMc / MC_PER_CREDIT,
|
|
12043
12523
|
baseMc: SCHEDULED_RUN_BASE_MC,
|
|
12044
12524
|
modelMc,
|
|
12045
12525
|
rawCostUsd: rawNanos == null ? null : rawNanos / USD_NANOS,
|
|
@@ -12061,7 +12541,7 @@ async function authorizeScheduledRun(args) {
|
|
|
12061
12541
|
args: [user.id, SCHEDULED_RUN_BILLING_CLASS]
|
|
12062
12542
|
});
|
|
12063
12543
|
if (pending.rows.length > 0) {
|
|
12064
|
-
throw new UnifiedBillingError("pending_cost", "a previous scheduled run has an unresolved cost", 409);
|
|
12544
|
+
throw new UnifiedBillingError("pending_cost", "a previous scheduled run has an unresolved model cost", 409);
|
|
12065
12545
|
}
|
|
12066
12546
|
const existing = await getDb().execute({
|
|
12067
12547
|
sql: "SELECT id, user_id, idempotency_key, status, minimum_amount_mc, maximum_amount_mc, token_hash, expires_at FROM billing_authorizations WHERE idempotency_key = ? LIMIT 1",
|
|
@@ -12070,14 +12550,9 @@ async function authorizeScheduledRun(args) {
|
|
|
12070
12550
|
if (existing.rows[0]) {
|
|
12071
12551
|
const row = existing.rows[0];
|
|
12072
12552
|
if (Number(row.user_id) !== user.id) {
|
|
12073
|
-
throw new UnifiedBillingError(
|
|
12074
|
-
"invalid_request",
|
|
12075
|
-
"the idempotency key is already in use by another billing identity",
|
|
12076
|
-
409
|
|
12077
|
-
);
|
|
12553
|
+
throw new UnifiedBillingError("invalid_request", "the idempotency key is already in use by another billing identity", 409);
|
|
12078
12554
|
}
|
|
12079
|
-
|
|
12080
|
-
return mapAuthorization(row, balanceMc2);
|
|
12555
|
+
return mapAuthorization(row, await reconcileBalanceMc(user.id, user.balance_mc));
|
|
12081
12556
|
}
|
|
12082
12557
|
const maximumCredits = args.maximumCredits ?? SCHEDULED_RUN_DEFAULT_MAX_CREDITS;
|
|
12083
12558
|
if (!Number.isFinite(maximumCredits) || maximumCredits < SCHEDULED_RUN_BASE_MC / MC_PER_CREDIT) {
|
|
@@ -12098,17 +12573,7 @@ async function authorizeScheduledRun(args) {
|
|
|
12098
12573
|
const expiresAt = new Date(Date.now() + AUTHORIZATION_TTL_MS).toISOString();
|
|
12099
12574
|
const inserted = await getDb().execute({
|
|
12100
12575
|
sql: "INSERT OR IGNORE INTO billing_authorizations (id, user_id, idempotency_key, billing_class, status, minimum_amount_mc, maximum_amount_mc, token_hash, metadata, expires_at) VALUES (?, ?, ?, ?, 'authorized', ?, ?, ?, ?, ?)",
|
|
12101
|
-
args: [
|
|
12102
|
-
id,
|
|
12103
|
-
user.id,
|
|
12104
|
-
idempotencyKey,
|
|
12105
|
-
SCHEDULED_RUN_BILLING_CLASS,
|
|
12106
|
-
SCHEDULED_RUN_BASE_MC,
|
|
12107
|
-
maximumMc,
|
|
12108
|
-
tokenHash(token),
|
|
12109
|
-
safeMetadata(args.metadata),
|
|
12110
|
-
expiresAt
|
|
12111
|
-
]
|
|
12576
|
+
args: [id, user.id, idempotencyKey, SCHEDULED_RUN_BILLING_CLASS, SCHEDULED_RUN_BASE_MC, maximumMc, tokenHash(token), safeMetadata(args.metadata), expiresAt]
|
|
12112
12577
|
});
|
|
12113
12578
|
if (inserted.rowsAffected === 0) {
|
|
12114
12579
|
const raced = await getDb().execute({
|
|
@@ -12118,11 +12583,7 @@ async function authorizeScheduledRun(args) {
|
|
|
12118
12583
|
if (!raced.rows[0]) throw new Error("billing authorization insert raced without a readable winner");
|
|
12119
12584
|
const row = raced.rows[0];
|
|
12120
12585
|
if (Number(row.user_id) !== user.id) {
|
|
12121
|
-
throw new UnifiedBillingError(
|
|
12122
|
-
"invalid_request",
|
|
12123
|
-
"the idempotency key is already in use by another billing identity",
|
|
12124
|
-
409
|
|
12125
|
-
);
|
|
12586
|
+
throw new UnifiedBillingError("invalid_request", "the idempotency key is already in use by another billing identity", 409);
|
|
12126
12587
|
}
|
|
12127
12588
|
return mapAuthorization(row, balanceMc);
|
|
12128
12589
|
}
|
|
@@ -12139,180 +12600,235 @@ async function authorizeScheduledRun(args) {
|
|
|
12139
12600
|
reservationMode: "balance_check"
|
|
12140
12601
|
};
|
|
12141
12602
|
}
|
|
12142
|
-
async function
|
|
12143
|
-
const
|
|
12144
|
-
|
|
12145
|
-
if (!user) throw new UnifiedBillingError("user_not_found", "billing identity was not found", 404);
|
|
12146
|
-
const authorization = await authorizationById(args.authorizationId);
|
|
12603
|
+
async function startScheduledRun(args) {
|
|
12604
|
+
const user = await resolveSchedulingUser(args.identity);
|
|
12605
|
+
let authorization = await authorizationById(args.authorizationId);
|
|
12147
12606
|
if (!authorization || Number(authorization.user_id) !== user.id) {
|
|
12148
12607
|
throw new UnifiedBillingError("authorization_not_found", "billing authorization was not found", 404);
|
|
12149
12608
|
}
|
|
12150
12609
|
verifyToken(authorization, args.authorizationToken);
|
|
12151
|
-
const
|
|
12152
|
-
const existing = await eventByKey(
|
|
12153
|
-
if (existing) {
|
|
12154
|
-
|
|
12155
|
-
|
|
12156
|
-
|
|
12157
|
-
}
|
|
12158
|
-
return
|
|
12610
|
+
const startKey = authorization.id + ":start";
|
|
12611
|
+
const existing = await eventByKey(startKey);
|
|
12612
|
+
if (existing?.status === "settled") {
|
|
12613
|
+
await getDb().execute({
|
|
12614
|
+
sql: "UPDATE billing_authorizations SET status = 'started' WHERE id = ? AND status IN ('authorized', 'starting')",
|
|
12615
|
+
args: [authorization.id]
|
|
12616
|
+
});
|
|
12617
|
+
return mapStartEvent(existing, await reconcileBalanceMc(user.id, user.balance_mc), true);
|
|
12159
12618
|
}
|
|
12160
|
-
if (
|
|
12161
|
-
throw new UnifiedBillingError(
|
|
12619
|
+
if (!hasPaidSchedulingAccess(user)) {
|
|
12620
|
+
throw new UnifiedBillingError(PAID_PLAN_REQUIRED_CODE, PAID_PLAN_REQUIRED_MESSAGE, 403);
|
|
12162
12621
|
}
|
|
12163
|
-
if (
|
|
12622
|
+
if (authorization.status === "authorized") {
|
|
12623
|
+
if (Date.parse(authorization.expires_at) <= Date.now()) {
|
|
12624
|
+
await getDb().execute({
|
|
12625
|
+
sql: "UPDATE billing_authorizations SET status = 'expired' WHERE id = ? AND status = 'authorized'",
|
|
12626
|
+
args: [authorization.id]
|
|
12627
|
+
});
|
|
12628
|
+
throw new UnifiedBillingError("authorization_expired", "billing authorization expired before execution started", 409);
|
|
12629
|
+
}
|
|
12164
12630
|
await getDb().execute({
|
|
12165
|
-
sql: "UPDATE billing_authorizations SET status = '
|
|
12631
|
+
sql: "UPDATE billing_authorizations SET status = 'starting' WHERE id = ? AND status = 'authorized'",
|
|
12166
12632
|
args: [authorization.id]
|
|
12167
12633
|
});
|
|
12168
|
-
|
|
12169
|
-
}
|
|
12170
|
-
const rawCostUsd = args.rawCostUsd ?? null;
|
|
12171
|
-
if (args.modelCostStatus === "reported" && (rawCostUsd == null || !Number.isFinite(rawCostUsd) || rawCostUsd < 0)) {
|
|
12172
|
-
throw new UnifiedBillingError("invalid_request", "reported model cost must be a finite non-negative number", 400);
|
|
12634
|
+
authorization = await authorizationById(authorization.id);
|
|
12173
12635
|
}
|
|
12174
|
-
if (
|
|
12175
|
-
throw new UnifiedBillingError("
|
|
12636
|
+
if (!authorization || !["starting", "started"].includes(authorization.status)) {
|
|
12637
|
+
throw new UnifiedBillingError("authorization_closed", "billing authorization is no longer startable", 409);
|
|
12176
12638
|
}
|
|
12177
|
-
const eventId = (0, import_node_crypto4.randomUUID)();
|
|
12178
|
-
|
|
12179
|
-
const modelMc = rawCostUsd == null ? 0 : vendorCostUsdToMc(rawCostUsd, SCHEDULED_AGENT_LLM_MARKUP_BPS);
|
|
12180
|
-
const totalMc = SCHEDULED_RUN_BASE_MC + modelMc;
|
|
12181
|
-
const pendingReason = args.modelCostStatus === "missing" ? "openrouter_cost_missing" : totalMc > Number(authorization.maximum_amount_mc) ? "max_credits_per_run_exceeded" : null;
|
|
12182
|
-
if (pendingReason) {
|
|
12639
|
+
const eventId = existing?.id ?? (0, import_node_crypto4.randomUUID)();
|
|
12640
|
+
if (!existing) {
|
|
12183
12641
|
await getDb().execute({
|
|
12184
|
-
sql: "INSERT INTO billing_events (id, user_id, authorization_id, idempotency_key, billing_class, source_surface, status, amount_mc,
|
|
12642
|
+
sql: "INSERT OR IGNORE INTO billing_events (id, user_id, authorization_id, idempotency_key, billing_class, source_surface, status, amount_mc, multiplier_bps, metadata) VALUES (?, ?, ?, ?, ?, ?, 'capturing', ?, ?, ?)",
|
|
12185
12643
|
args: [
|
|
12186
12644
|
eventId,
|
|
12187
12645
|
user.id,
|
|
12188
12646
|
authorization.id,
|
|
12189
|
-
|
|
12647
|
+
startKey,
|
|
12190
12648
|
SCHEDULED_RUN_BILLING_CLASS,
|
|
12191
12649
|
SCHEDULED_RUN_SOURCE_SURFACE,
|
|
12192
|
-
|
|
12193
|
-
rawCostNanos,
|
|
12650
|
+
SCHEDULED_RUN_BASE_MC,
|
|
12194
12651
|
SCHEDULED_AGENT_LLM_MARKUP_BPS,
|
|
12195
|
-
|
|
12196
|
-
Math.max(0, Math.round(args.outputTokens ?? 0)),
|
|
12197
|
-
Math.max(0, Math.round(args.cachedTokens ?? 0)),
|
|
12198
|
-
Math.max(0, Math.round(args.modelSteps ?? 0)),
|
|
12199
|
-
safeMetadata({ ...args.metadata, reason: pendingReason })
|
|
12652
|
+
safeMetadata({ ...args.metadata, billingPhase: "start", chargeMode: "base_capture" })
|
|
12200
12653
|
]
|
|
12201
12654
|
});
|
|
12202
|
-
await getDb().execute({
|
|
12203
|
-
sql: "UPDATE billing_authorizations SET status = 'cost_pending', settled_at = datetime('now') WHERE id = ?",
|
|
12204
|
-
args: [authorization.id]
|
|
12205
|
-
});
|
|
12206
|
-
return {
|
|
12207
|
-
id: eventId,
|
|
12208
|
-
authorizationId: authorization.id,
|
|
12209
|
-
status: "cost_pending",
|
|
12210
|
-
amountMc: totalMc,
|
|
12211
|
-
credits: totalMc / MC_PER_CREDIT,
|
|
12212
|
-
baseMc: SCHEDULED_RUN_BASE_MC,
|
|
12213
|
-
modelMc,
|
|
12214
|
-
rawCostUsd,
|
|
12215
|
-
multiplierBps: SCHEDULED_AGENT_LLM_MARKUP_BPS,
|
|
12216
|
-
balanceMc: await reconcileBalanceMc(user.id, user.balance_mc),
|
|
12217
|
-
reason: pendingReason,
|
|
12218
|
-
duplicate: false
|
|
12219
|
-
};
|
|
12220
|
-
}
|
|
12221
|
-
const claimed = await getDb().execute({
|
|
12222
|
-
sql: "INSERT OR IGNORE INTO billing_events (id, user_id, authorization_id, idempotency_key, billing_class, source_surface, status, amount_mc, raw_cost_usd_nanos, multiplier_bps, input_tokens, output_tokens, cached_tokens, model_steps, metadata) VALUES (?, ?, ?, ?, ?, ?, 'settling', ?, ?, ?, ?, ?, ?, ?, ?)",
|
|
12223
|
-
args: [
|
|
12224
|
-
eventId,
|
|
12225
|
-
user.id,
|
|
12226
|
-
authorization.id,
|
|
12227
|
-
settlementKey,
|
|
12228
|
-
SCHEDULED_RUN_BILLING_CLASS,
|
|
12229
|
-
SCHEDULED_RUN_SOURCE_SURFACE,
|
|
12230
|
-
totalMc,
|
|
12231
|
-
rawCostNanos,
|
|
12232
|
-
SCHEDULED_AGENT_LLM_MARKUP_BPS,
|
|
12233
|
-
Math.max(0, Math.round(args.inputTokens ?? 0)),
|
|
12234
|
-
Math.max(0, Math.round(args.outputTokens ?? 0)),
|
|
12235
|
-
Math.max(0, Math.round(args.cachedTokens ?? 0)),
|
|
12236
|
-
Math.max(0, Math.round(args.modelSteps ?? 0)),
|
|
12237
|
-
safeMetadata(args.metadata)
|
|
12238
|
-
]
|
|
12239
|
-
});
|
|
12240
|
-
if (claimed.rowsAffected === 0) {
|
|
12241
|
-
const raced = await eventByKey(settlementKey);
|
|
12242
|
-
if (!raced) throw new Error("billing settlement raced without a readable winner");
|
|
12243
|
-
if (raced.status === "settling") {
|
|
12244
|
-
throw new UnifiedBillingError("settlement_in_progress", "scheduled-run settlement is already in progress", 409);
|
|
12245
|
-
}
|
|
12246
|
-
return mapEvent(raced, await reconcileBalanceMc(user.id, user.balance_mc), true);
|
|
12247
12655
|
}
|
|
12248
|
-
const debit = await
|
|
12656
|
+
const debit = await debitMcIdempotent(
|
|
12249
12657
|
user.id,
|
|
12250
|
-
|
|
12658
|
+
SCHEDULED_RUN_BASE_MC,
|
|
12251
12659
|
LedgerOperation.SCHEDULED_RUN,
|
|
12252
|
-
"scheduled-run:" + authorization.id
|
|
12660
|
+
"scheduled-run:" + authorization.id + ":base",
|
|
12661
|
+
startKey
|
|
12253
12662
|
);
|
|
12254
12663
|
if (!debit.ok) {
|
|
12255
|
-
const metadata = safeMetadata({ ...args.metadata, reason: "insufficient_credits_at_settlement" });
|
|
12256
|
-
await getDb().execute({
|
|
12257
|
-
sql: "UPDATE billing_events SET status = 'cost_pending', metadata = ? WHERE id = ?",
|
|
12258
|
-
args: [metadata, eventId]
|
|
12259
|
-
});
|
|
12260
12664
|
await getDb().execute({
|
|
12261
|
-
sql: "UPDATE billing_authorizations SET status = '
|
|
12665
|
+
sql: "UPDATE billing_authorizations SET status = 'authorized' WHERE id = ? AND status = 'starting'",
|
|
12262
12666
|
args: [authorization.id]
|
|
12263
12667
|
});
|
|
12264
|
-
|
|
12265
|
-
|
|
12266
|
-
|
|
12267
|
-
|
|
12268
|
-
|
|
12269
|
-
|
|
12270
|
-
|
|
12668
|
+
throw new UnifiedBillingError(
|
|
12669
|
+
"insufficient_credits",
|
|
12670
|
+
"the wallet could not capture the scheduled-run base charge",
|
|
12671
|
+
402,
|
|
12672
|
+
{ balanceMc: debit.balance_mc, requiredMc: SCHEDULED_RUN_BASE_MC }
|
|
12673
|
+
);
|
|
12674
|
+
}
|
|
12675
|
+
await getDb().execute({
|
|
12676
|
+
sql: "UPDATE billing_events SET status = 'settled', settled_at = datetime('now') WHERE idempotency_key = ? AND status = 'capturing'",
|
|
12677
|
+
args: [startKey]
|
|
12678
|
+
});
|
|
12679
|
+
await getDb().execute({
|
|
12680
|
+
sql: "UPDATE billing_authorizations SET status = 'started' WHERE id = ? AND status IN ('authorized', 'starting', 'started')",
|
|
12681
|
+
args: [authorization.id]
|
|
12682
|
+
});
|
|
12683
|
+
const started = await eventByKey(startKey);
|
|
12684
|
+
if (!started) throw new Error("scheduled-run base capture completed without an event receipt");
|
|
12685
|
+
return mapStartEvent(started, debit.balance_mc, debit.duplicate);
|
|
12686
|
+
}
|
|
12687
|
+
function validateModelCost(args) {
|
|
12688
|
+
const rawCostUsd = args.rawCostUsd ?? null;
|
|
12689
|
+
if (args.modelCostStatus === "reported" && (rawCostUsd == null || !Number.isFinite(rawCostUsd) || rawCostUsd < 0)) {
|
|
12690
|
+
throw new UnifiedBillingError("invalid_request", "reported model cost must be a finite non-negative number", 400);
|
|
12691
|
+
}
|
|
12692
|
+
if (args.modelCostStatus === "not_applicable" && rawCostUsd != null && rawCostUsd !== 0) {
|
|
12693
|
+
throw new UnifiedBillingError("invalid_request", "not_applicable model cost cannot include a non-zero cost", 400);
|
|
12694
|
+
}
|
|
12695
|
+
return rawCostUsd;
|
|
12696
|
+
}
|
|
12697
|
+
async function recoverSettlement(row, user, authorization, duplicate) {
|
|
12698
|
+
const modelMc = Math.max(0, Number(row.amount_mc));
|
|
12699
|
+
let balanceMc = await reconcileBalanceMc(user.id, user.balance_mc);
|
|
12700
|
+
if (modelMc > 0) {
|
|
12701
|
+
const debit = await debitMcIdempotent(
|
|
12702
|
+
user.id,
|
|
12271
12703
|
modelMc,
|
|
12272
|
-
|
|
12273
|
-
|
|
12274
|
-
|
|
12275
|
-
|
|
12276
|
-
|
|
12277
|
-
|
|
12704
|
+
LedgerOperation.SCHEDULED_RUN,
|
|
12705
|
+
"scheduled-run:" + authorization.id + ":model",
|
|
12706
|
+
authorization.id + ":model"
|
|
12707
|
+
);
|
|
12708
|
+
balanceMc = debit.balance_mc;
|
|
12709
|
+
if (!debit.ok) {
|
|
12710
|
+
await getDb().execute({
|
|
12711
|
+
sql: "UPDATE billing_events SET status = 'cost_pending', metadata = ? WHERE id = ? AND status = 'settling'",
|
|
12712
|
+
args: [safeMetadata({ reason: "insufficient_credits_for_model_delta", billingPhase: "settlement", chargeMode: "model_delta" }), row.id]
|
|
12713
|
+
});
|
|
12714
|
+
await getDb().execute({
|
|
12715
|
+
sql: "UPDATE billing_authorizations SET status = 'cost_pending' WHERE id = ? AND status IN ('started', 'cost_pending')",
|
|
12716
|
+
args: [authorization.id]
|
|
12717
|
+
});
|
|
12718
|
+
const pending = await eventByKey(authorization.id + ":settlement");
|
|
12719
|
+
if (!pending) throw new Error("scheduled-run pending settlement lost its event");
|
|
12720
|
+
return mapSettlementEvent(pending, balanceMc, duplicate);
|
|
12721
|
+
}
|
|
12278
12722
|
}
|
|
12279
12723
|
await getDb().execute({
|
|
12280
|
-
sql: "UPDATE billing_events SET status = 'settled', settled_at = datetime('now') WHERE id = ?",
|
|
12281
|
-
args: [
|
|
12724
|
+
sql: "UPDATE billing_events SET status = 'settled', settled_at = datetime('now') WHERE id = ? AND status = 'settling'",
|
|
12725
|
+
args: [row.id]
|
|
12282
12726
|
});
|
|
12283
12727
|
await getDb().execute({
|
|
12284
|
-
sql: "UPDATE billing_authorizations SET status = 'settled', settled_at = datetime('now') WHERE id = ?",
|
|
12728
|
+
sql: "UPDATE billing_authorizations SET status = 'settled', settled_at = datetime('now') WHERE id = ? AND status IN ('started', 'cost_pending', 'settled')",
|
|
12285
12729
|
args: [authorization.id]
|
|
12286
12730
|
});
|
|
12287
|
-
|
|
12288
|
-
|
|
12289
|
-
|
|
12290
|
-
|
|
12291
|
-
|
|
12292
|
-
|
|
12293
|
-
|
|
12731
|
+
const settled = await eventByKey(authorization.id + ":settlement");
|
|
12732
|
+
if (!settled) throw new Error("scheduled-run settlement completed without an event receipt");
|
|
12733
|
+
return mapSettlementEvent(settled, balanceMc, duplicate);
|
|
12734
|
+
}
|
|
12735
|
+
async function settleScheduledRun(args) {
|
|
12736
|
+
const user = await resolveSchedulingUser(args.identity);
|
|
12737
|
+
let authorization = await authorizationById(args.authorizationId);
|
|
12738
|
+
if (!authorization || Number(authorization.user_id) !== user.id) {
|
|
12739
|
+
throw new UnifiedBillingError("authorization_not_found", "billing authorization was not found", 404);
|
|
12740
|
+
}
|
|
12741
|
+
verifyToken(authorization, args.authorizationToken);
|
|
12742
|
+
const rawCostUsd = validateModelCost(args);
|
|
12743
|
+
const startEvent = await eventByKey(authorization.id + ":start");
|
|
12744
|
+
if (!startEvent || startEvent.status !== "settled") {
|
|
12745
|
+
throw new UnifiedBillingError("authorization_closed", "scheduled-run settlement requires a captured start", 409);
|
|
12746
|
+
}
|
|
12747
|
+
if (authorization.status === "starting") {
|
|
12748
|
+
await getDb().execute({
|
|
12749
|
+
sql: "UPDATE billing_authorizations SET status = 'started' WHERE id = ? AND status = 'starting'",
|
|
12750
|
+
args: [authorization.id]
|
|
12751
|
+
});
|
|
12752
|
+
authorization = await authorizationById(authorization.id);
|
|
12753
|
+
}
|
|
12754
|
+
if (!authorization || !["started", "cost_pending", "settled"].includes(authorization.status)) {
|
|
12755
|
+
throw new UnifiedBillingError("authorization_closed", "billing authorization is not open for settlement", 409);
|
|
12756
|
+
}
|
|
12757
|
+
const settlementKey = authorization.id + ":settlement";
|
|
12758
|
+
let existing = await eventByKey(settlementKey);
|
|
12759
|
+
if (existing?.status === "settled") {
|
|
12760
|
+
return mapSettlementEvent(existing, await reconcileBalanceMc(user.id, user.balance_mc), true);
|
|
12761
|
+
}
|
|
12762
|
+
if (existing?.status === "settling") {
|
|
12763
|
+
return recoverSettlement(existing, user, authorization, true);
|
|
12764
|
+
}
|
|
12765
|
+
const hadExistingClaim = Boolean(existing);
|
|
12766
|
+
const rawCostNanos = rawCostUsd == null ? null : Math.round(rawCostUsd * USD_NANOS);
|
|
12767
|
+
const modelMc = rawCostUsd == null ? 0 : vendorCostUsdToMc(rawCostUsd, SCHEDULED_AGENT_LLM_MARKUP_BPS);
|
|
12768
|
+
const maximumModelMc = Math.max(0, Number(authorization.maximum_amount_mc) - SCHEDULED_RUN_BASE_MC);
|
|
12769
|
+
const pendingReason = args.modelCostStatus === "missing" ? "openrouter_cost_missing" : modelMc > maximumModelMc ? "max_credits_per_run_exceeded" : null;
|
|
12770
|
+
const metadata = safeMetadata({
|
|
12771
|
+
...args.metadata,
|
|
12772
|
+
billingPhase: "settlement",
|
|
12773
|
+
chargeMode: "model_delta",
|
|
12774
|
+
...pendingReason ? { reason: pendingReason } : {}
|
|
12775
|
+
});
|
|
12776
|
+
const eventId = existing?.id ?? (0, import_node_crypto4.randomUUID)();
|
|
12777
|
+
const status = pendingReason ? "cost_pending" : "settling";
|
|
12778
|
+
const eventValues = [
|
|
12294
12779
|
modelMc,
|
|
12295
|
-
|
|
12296
|
-
|
|
12297
|
-
|
|
12298
|
-
|
|
12299
|
-
|
|
12780
|
+
rawCostNanos,
|
|
12781
|
+
SCHEDULED_AGENT_LLM_MARKUP_BPS,
|
|
12782
|
+
Math.max(0, Math.round(args.inputTokens ?? 0)),
|
|
12783
|
+
Math.max(0, Math.round(args.outputTokens ?? 0)),
|
|
12784
|
+
Math.max(0, Math.round(args.cachedTokens ?? 0)),
|
|
12785
|
+
Math.max(0, Math.round(args.modelSteps ?? 0)),
|
|
12786
|
+
metadata
|
|
12787
|
+
];
|
|
12788
|
+
if (existing) {
|
|
12789
|
+
await getDb().execute({
|
|
12790
|
+
sql: "UPDATE billing_events SET status = ?, amount_mc = ?, raw_cost_usd_nanos = ?, multiplier_bps = ?, input_tokens = ?, output_tokens = ?, cached_tokens = ?, model_steps = ?, metadata = ?, settled_at = NULL WHERE id = ? AND status = 'cost_pending'",
|
|
12791
|
+
args: [status, ...eventValues, existing.id]
|
|
12792
|
+
});
|
|
12793
|
+
} else {
|
|
12794
|
+
await getDb().execute({
|
|
12795
|
+
sql: "INSERT OR IGNORE INTO billing_events (id, user_id, authorization_id, idempotency_key, billing_class, source_surface, status, amount_mc, raw_cost_usd_nanos, multiplier_bps, input_tokens, output_tokens, cached_tokens, model_steps, metadata) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
|
|
12796
|
+
args: [eventId, user.id, authorization.id, settlementKey, SCHEDULED_RUN_BILLING_CLASS, SCHEDULED_RUN_SOURCE_SURFACE, status, ...eventValues]
|
|
12797
|
+
});
|
|
12798
|
+
}
|
|
12799
|
+
existing = await eventByKey(settlementKey);
|
|
12800
|
+
if (!existing) throw new Error("scheduled-run settlement claim was not readable");
|
|
12801
|
+
const duplicateClaim = hadExistingClaim || eventId !== existing.id;
|
|
12802
|
+
if (existing.status === "settled") {
|
|
12803
|
+
return mapSettlementEvent(existing, await reconcileBalanceMc(user.id, user.balance_mc), true);
|
|
12804
|
+
}
|
|
12805
|
+
if (existing.status === "cost_pending") {
|
|
12806
|
+
await getDb().execute({
|
|
12807
|
+
sql: "UPDATE billing_authorizations SET status = 'cost_pending' WHERE id = ? AND status IN ('started', 'cost_pending')",
|
|
12808
|
+
args: [authorization.id]
|
|
12809
|
+
});
|
|
12810
|
+
return mapSettlementEvent(existing, await reconcileBalanceMc(user.id, user.balance_mc), duplicateClaim);
|
|
12811
|
+
}
|
|
12812
|
+
if (existing.status !== "settling") {
|
|
12813
|
+
throw new UnifiedBillingError("settlement_in_progress", "scheduled-run settlement could not be claimed", 409);
|
|
12814
|
+
}
|
|
12815
|
+
return recoverSettlement(existing, user, authorization, duplicateClaim);
|
|
12300
12816
|
}
|
|
12301
12817
|
async function voidScheduledRunAuthorization(args) {
|
|
12302
|
-
const
|
|
12303
|
-
const user = await getUserByEmail(identity);
|
|
12304
|
-
if (!user) throw new UnifiedBillingError("user_not_found", "billing identity was not found", 404);
|
|
12818
|
+
const user = await resolveSchedulingUser(args.identity);
|
|
12305
12819
|
const authorization = await authorizationById(args.authorizationId);
|
|
12306
12820
|
if (!authorization || Number(authorization.user_id) !== user.id) {
|
|
12307
12821
|
throw new UnifiedBillingError("authorization_not_found", "billing authorization was not found", 404);
|
|
12308
12822
|
}
|
|
12309
12823
|
verifyToken(authorization, args.authorizationToken);
|
|
12310
12824
|
if (authorization.status === "authorized") {
|
|
12311
|
-
await getDb().execute({
|
|
12825
|
+
const result = await getDb().execute({
|
|
12312
12826
|
sql: "UPDATE billing_authorizations SET status = 'void', metadata = ?, settled_at = datetime('now') WHERE id = ? AND status = 'authorized'",
|
|
12313
12827
|
args: [safeMetadata({ reason: args.reason ?? "voided_before_execution" }), authorization.id]
|
|
12314
12828
|
});
|
|
12315
|
-
return { ok: true, status: "void" };
|
|
12829
|
+
if (result.rowsAffected > 0) return { ok: true, status: "void" };
|
|
12830
|
+
const raced = await authorizationById(authorization.id);
|
|
12831
|
+
return { ok: true, status: raced?.status ?? "closed" };
|
|
12316
12832
|
}
|
|
12317
12833
|
return { ok: true, status: authorization.status };
|
|
12318
12834
|
}
|
|
@@ -12326,7 +12842,7 @@ var init_unified_billing = __esm({
|
|
|
12326
12842
|
init_scheduling_access();
|
|
12327
12843
|
SCHEDULED_RUN_BILLING_CLASS = "scheduled_agent_run";
|
|
12328
12844
|
SCHEDULED_RUN_SOURCE_SURFACE = "mastra_scheduler";
|
|
12329
|
-
AUTHORIZATION_TTL_MS =
|
|
12845
|
+
AUTHORIZATION_TTL_MS = 24 * 60 * 60 * 1e3;
|
|
12330
12846
|
UnifiedBillingError = class extends Error {
|
|
12331
12847
|
constructor(code, message, httpStatus, details = {}) {
|
|
12332
12848
|
super(message);
|
|
@@ -12551,7 +13067,7 @@ function billingError(error) {
|
|
|
12551
13067
|
status: 500
|
|
12552
13068
|
};
|
|
12553
13069
|
}
|
|
12554
|
-
var import_hono3, import_zod12, internalBillingApp, metadataSchema, schedulingAccessSchema, authorizeSchema, settleSchema, voidSchema;
|
|
13070
|
+
var import_hono3, import_zod12, internalBillingApp, metadataSchema, schedulingAccessSchema, authorizeSchema, settleSchema, startSchema, voidSchema;
|
|
12555
13071
|
var init_internal_billing_routes = __esm({
|
|
12556
13072
|
"src/api/internal-billing-routes.ts"() {
|
|
12557
13073
|
"use strict";
|
|
@@ -12585,6 +13101,12 @@ var init_internal_billing_routes = __esm({
|
|
|
12585
13101
|
modelSteps: import_zod12.z.number().finite().nonnegative().optional(),
|
|
12586
13102
|
metadata: metadataSchema
|
|
12587
13103
|
});
|
|
13104
|
+
startSchema = import_zod12.z.object({
|
|
13105
|
+
identity: import_zod12.z.string().min(1).max(320),
|
|
13106
|
+
authorizationId: import_zod12.z.string().uuid(),
|
|
13107
|
+
authorizationToken: import_zod12.z.string().min(20).max(500),
|
|
13108
|
+
metadata: metadataSchema
|
|
13109
|
+
});
|
|
12588
13110
|
voidSchema = import_zod12.z.object({
|
|
12589
13111
|
identity: import_zod12.z.string().min(1).max(320),
|
|
12590
13112
|
authorizationId: import_zod12.z.string().uuid(),
|
|
@@ -12658,6 +13180,19 @@ var init_internal_billing_routes = __esm({
|
|
|
12658
13180
|
return c.json(response.body, response.status);
|
|
12659
13181
|
}
|
|
12660
13182
|
});
|
|
13183
|
+
internalBillingApp.post("/scheduled-run/start", async (c) => {
|
|
13184
|
+
const parsed = startSchema.safeParse(await c.req.json().catch(() => null));
|
|
13185
|
+
if (!parsed.success) {
|
|
13186
|
+
return c.json({ ok: false, error_code: "invalid_request", error: "invalid scheduled-run start request" }, 400);
|
|
13187
|
+
}
|
|
13188
|
+
try {
|
|
13189
|
+
const start = await startScheduledRun(parsed.data);
|
|
13190
|
+
return c.json({ ok: true, start });
|
|
13191
|
+
} catch (error) {
|
|
13192
|
+
const response = billingError(error);
|
|
13193
|
+
return c.json(response.body, response.status);
|
|
13194
|
+
}
|
|
13195
|
+
});
|
|
12661
13196
|
internalBillingApp.post("/scheduled-run/void", async (c) => {
|
|
12662
13197
|
const parsed = voidSchema.safeParse(await c.req.json().catch(() => null));
|
|
12663
13198
|
if (!parsed.success) {
|
|
@@ -12944,6 +13479,7 @@ var init_BrowserDriver = __esm({
|
|
|
12944
13479
|
else if (config.headlessMode === "headless") useHeadless = true;
|
|
12945
13480
|
else useHeadless = costCtx?.forceHeadless === true || process.env.KERNEL_HEADLESS_DISABLED !== "true" && !!costCtx?.op && HEADLESS_OPS.has(costCtx.op);
|
|
12946
13481
|
this.kernelHeadlessSent = useHeadless ? true : null;
|
|
13482
|
+
if (costCtx?.headlessSentOut) costCtx.headlessSentOut.value = this.kernelHeadlessSent;
|
|
12947
13483
|
const kernelBrowser = await this.kernelClient.browsers.create({
|
|
12948
13484
|
stealth: true,
|
|
12949
13485
|
timeout_seconds: timeoutSeconds,
|
|
@@ -16426,6 +16962,15 @@ function buildPageIntelUrl(body, country) {
|
|
|
16426
16962
|
if (body.pageId?.trim()) return `https://www.facebook.com/ads/library/?active_status=all&ad_type=all&country=${country}&is_targeted_country=false&media_type=all&search_type=page&view_all_page_id=${body.pageId.trim()}`;
|
|
16427
16963
|
return `https://www.facebook.com/ads/library/?active_status=all&ad_type=all&country=${country}&q=${encodeURIComponent(body.query.trim())}&search_type=keyword_unordered`;
|
|
16428
16964
|
}
|
|
16965
|
+
function pageIntelRequestContext(body) {
|
|
16966
|
+
const country = body.country?.trim().toUpperCase() ?? "US";
|
|
16967
|
+
return {
|
|
16968
|
+
country,
|
|
16969
|
+
maxAds: Math.min(200, Math.max(1, body.maxAds ?? 50)),
|
|
16970
|
+
listingUrl: buildPageIntelUrl(body, country),
|
|
16971
|
+
subject: body.pageId ?? body.query ?? body.libraryId ?? ""
|
|
16972
|
+
};
|
|
16973
|
+
}
|
|
16429
16974
|
function normalizeAdvertiserName(value) {
|
|
16430
16975
|
return value.toLowerCase().normalize("NFKD").replace(/[^a-z0-9]+/g, " ").trim().replace(/\s+/g, " ");
|
|
16431
16976
|
}
|
|
@@ -16590,16 +17135,14 @@ var init_facebook_ad_routes = __esm({
|
|
|
16590
17135
|
const parsed = FacebookPageIntelBodySchema.safeParse(raw);
|
|
16591
17136
|
if (!parsed.success) return c.json(invalidRequest(parsed.error.issues[0]?.message ?? "Invalid request"), 400);
|
|
16592
17137
|
const body = parsed.data;
|
|
16593
|
-
const
|
|
16594
|
-
const country = body.country?.trim().toUpperCase() ?? "US";
|
|
16595
|
-
const listingUrl = buildPageIntelUrl(body, country);
|
|
17138
|
+
const { country, maxAds, listingUrl, subject } = pageIntelRequestContext(body);
|
|
16596
17139
|
const fbUser = c.get("user");
|
|
16597
17140
|
const gate = await acquireConcurrencyGate(fbUser, "facebook_page_intel", { reuseLockId: c.req.header("x-mcp-scraper-concurrency-lock"), metadata: { pageId: body.pageId, query: body.query, libraryId: body.libraryId, country } });
|
|
16598
17141
|
if (!gate.ok) return c.json(concurrencyLimitExceededResponse(gate), 429, { "Retry-After": String(gate.retryAfterSeconds) });
|
|
16599
17142
|
const driver = new BrowserDriver();
|
|
16600
17143
|
let refunded = false, debited = false;
|
|
16601
17144
|
try {
|
|
16602
|
-
const { ok: fbOk, balance_mc: fbBal } = await debitMc(fbUser.id, MC_COSTS.fb_ad, LedgerOperation.FB_AD,
|
|
17145
|
+
const { ok: fbOk, balance_mc: fbBal } = await debitMc(fbUser.id, MC_COSTS.fb_ad, LedgerOperation.FB_AD, subject);
|
|
16603
17146
|
if (!fbOk) return c.json(insufficientBalanceResponse(fbBal, MC_COSTS.fb_ad), 402);
|
|
16604
17147
|
debited = true;
|
|
16605
17148
|
await driver.launch(await kernelLaunchOptsResidential());
|
|
@@ -16609,17 +17152,17 @@ var init_facebook_ad_routes = __esm({
|
|
|
16609
17152
|
if (result.ads.length === 0 && await detectSoftBlock(driver)) {
|
|
16610
17153
|
await creditMc(fbUser.id, MC_COSTS.fb_ad, LedgerOperation.FB_AD_REFUND, "soft-block empty result");
|
|
16611
17154
|
refunded = true;
|
|
16612
|
-
await logRequestEvent({ userId: fbUser.id, source: "facebook_page_intel", status: "failed", query:
|
|
17155
|
+
await logRequestEvent({ userId: fbUser.id, source: "facebook_page_intel", status: "failed", query: subject, error: "soft-block: empty result refunded" });
|
|
16613
17156
|
return c.json({ error: "soft-block: no ads returned (refunded)" }, 503);
|
|
16614
17157
|
}
|
|
16615
17158
|
const match = assessPageIntelMatch(body, result);
|
|
16616
17159
|
const output = { ...result, ...match };
|
|
16617
|
-
await logRequestEvent({ userId: fbUser.id, source: "facebook_page_intel", status: "done", query:
|
|
17160
|
+
await logRequestEvent({ userId: fbUser.id, source: "facebook_page_intel", status: "done", query: subject, resultCount: result.ads.length, result: output });
|
|
16618
17161
|
return c.json(output);
|
|
16619
17162
|
} catch (err) {
|
|
16620
17163
|
const msg = err instanceof Error ? err.message : String(err);
|
|
16621
17164
|
if (debited && !refunded) await creditMc(fbUser.id, MC_COSTS.fb_ad, LedgerOperation.FB_AD_REFUND, "failed call");
|
|
16622
|
-
await logRequestEvent({ userId: fbUser.id, source: "facebook_page_intel", status: "failed", query:
|
|
17165
|
+
await logRequestEvent({ userId: fbUser.id, source: "facebook_page_intel", status: "failed", query: subject, error: msg });
|
|
16623
17166
|
if (msg.toLowerCase().includes("blocked") || msg.toLowerCase().includes("captcha")) {
|
|
16624
17167
|
return c.json({ error: msg }, 503);
|
|
16625
17168
|
}
|
|
@@ -22369,21 +22912,33 @@ function formatCaptureSerpSnapshot(raw, input) {
|
|
|
22369
22912
|
const parsed = parseData(raw);
|
|
22370
22913
|
if ("error" in parsed) return { content: [{ type: "text", text: parsed.error }], isError: true };
|
|
22371
22914
|
const d = parsed.data;
|
|
22915
|
+
const harvestResult = d.harvestResult && typeof d.harvestResult === "object" ? d.harvestResult : d;
|
|
22372
22916
|
const providerPayload = d;
|
|
22373
|
-
const organic = Array.isArray(
|
|
22374
|
-
const localPack = Array.isArray(
|
|
22375
|
-
const artifacts = Array.isArray(d.artifacts) ? d.artifacts : [];
|
|
22376
|
-
const
|
|
22377
|
-
const
|
|
22378
|
-
const
|
|
22379
|
-
const
|
|
22917
|
+
const organic = Array.isArray(harvestResult.organicResults) ? harvestResult.organicResults : Array.isArray(harvestResult.organic) ? harvestResult.organic : [];
|
|
22918
|
+
const localPack = Array.isArray(harvestResult.localPack) ? harvestResult.localPack : [];
|
|
22919
|
+
const artifacts = Array.isArray(d.pageSnapshotArtifacts) ? d.pageSnapshotArtifacts : Array.isArray(d.artifacts) ? d.artifacts : [];
|
|
22920
|
+
const attempts = Array.isArray(d.attempts) ? d.attempts : [];
|
|
22921
|
+
const diagnostics = harvestResult.diagnostics && typeof harvestResult.diagnostics === "object" ? harvestResult.diagnostics : d.diagnostics && typeof d.diagnostics === "object" ? d.diagnostics : null;
|
|
22922
|
+
const completionStatus = diagnostics && typeof diagnostics.completionStatus === "string" ? diagnostics.completionStatus : null;
|
|
22923
|
+
const status = String(d.status ?? d.captureStatus ?? completionStatus ?? "captured");
|
|
22924
|
+
const query = typeof harvestResult.seed === "string" ? harvestResult.seed : typeof harvestResult.query === "string" ? harvestResult.query : typeof input.query === "string" ? input.query : null;
|
|
22925
|
+
const location2 = typeof harvestResult.location === "string" ? harvestResult.location : typeof input.location === "string" ? input.location : null;
|
|
22926
|
+
const capturedAt = typeof harvestResult.extractedAt === "string" ? harvestResult.extractedAt : typeof harvestResult.capturedAt === "string" ? harvestResult.capturedAt : null;
|
|
22380
22927
|
const resultCount = organic.length + localPack.length;
|
|
22928
|
+
const pageSnapshotFailedCount = artifacts.filter((artifact) => artifact.status === "failed" || Boolean(artifact.error)).length;
|
|
22929
|
+
const rawBilling = d.billing && typeof d.billing === "object" ? d.billing : null;
|
|
22930
|
+
const billing = rawBilling ? {
|
|
22931
|
+
creditsUsed: typeof rawBilling.creditsUsed === "number" ? rawBilling.creditsUsed : null,
|
|
22932
|
+
requestId: typeof rawBilling.requestId === "string" ? rawBilling.requestId : null,
|
|
22933
|
+
jobId: typeof rawBilling.jobId === "string" ? rawBilling.jobId : null
|
|
22934
|
+
} : null;
|
|
22381
22935
|
const full = [
|
|
22382
22936
|
`# SERP Intelligence Snapshot: ${query ?? "query"}`,
|
|
22383
22937
|
`**Status:** ${status}`,
|
|
22384
22938
|
location2 ? `**Location:** ${location2}` : "",
|
|
22385
|
-
`**
|
|
22386
|
-
|
|
22939
|
+
`**SERP results:** ${resultCount} (${organic.length} organic, ${localPack.length} local pack)`,
|
|
22940
|
+
`**Page snapshots:** ${artifacts.length} (${pageSnapshotFailedCount} failed)`,
|
|
22941
|
+
billing?.creditsUsed !== null && billing?.creditsUsed !== void 0 ? `**Credits used:** ${billing.creditsUsed}` : "",
|
|
22387
22942
|
"",
|
|
22388
22943
|
"Use `capture_serp_page_snapshots` when you need page-level evidence for ranking URLs."
|
|
22389
22944
|
].filter(Boolean).join("\n");
|
|
@@ -22396,6 +22951,10 @@ function formatCaptureSerpSnapshot(raw, input) {
|
|
|
22396
22951
|
location: location2,
|
|
22397
22952
|
capturedAt,
|
|
22398
22953
|
resultCount,
|
|
22954
|
+
organicResultCount: organic.length,
|
|
22955
|
+
localPackResultCount: localPack.length,
|
|
22956
|
+
pageSnapshotCount: artifacts.length,
|
|
22957
|
+
pageSnapshotFailedCount,
|
|
22399
22958
|
snapshotId: typeof d.snapshotId === "string" ? d.snapshotId : typeof d.snapshot_id === "string" ? d.snapshot_id : typeof d.id === "string" ? d.id : null,
|
|
22400
22959
|
resolvedInputs: {
|
|
22401
22960
|
query: input.query ?? null,
|
|
@@ -22408,7 +22967,9 @@ function formatCaptureSerpSnapshot(raw, input) {
|
|
|
22408
22967
|
pages: input.pages ?? null
|
|
22409
22968
|
},
|
|
22410
22969
|
artifacts,
|
|
22411
|
-
|
|
22970
|
+
attempts,
|
|
22971
|
+
billing,
|
|
22972
|
+
diagnostics,
|
|
22412
22973
|
providerPayload
|
|
22413
22974
|
}
|
|
22414
22975
|
};
|
|
@@ -22417,9 +22978,13 @@ function formatCaptureSerpPageSnapshots(raw, input) {
|
|
|
22417
22978
|
const parsed = parseData(raw);
|
|
22418
22979
|
if ("error" in parsed) return { content: [{ type: "text", text: parsed.error }], isError: true };
|
|
22419
22980
|
const d = parsed.data;
|
|
22420
|
-
const captures = Array.isArray(d.captures) ? d.captures : Array.isArray(d.results) ? d.results : [];
|
|
22981
|
+
const captures = Array.isArray(d.pageSnapshotArtifacts) ? d.pageSnapshotArtifacts : Array.isArray(d.captures) ? d.captures : Array.isArray(d.results) ? d.results : [];
|
|
22421
22982
|
const failedCount = captures.filter((c) => c.status === "failed" || c.error).length;
|
|
22422
|
-
const
|
|
22983
|
+
const capturedCount = captures.filter((c) => c.status === "captured" || c.status === "ok").length;
|
|
22984
|
+
const skippedCount = captures.filter((c) => c.status === "skipped").length;
|
|
22985
|
+
const attempts = Array.isArray(d.attempts) ? d.attempts : [];
|
|
22986
|
+
const inferredStatus = captures.length === 0 ? "empty" : failedCount === captures.length ? "failed" : failedCount > 0 || skippedCount > 0 ? "partial" : "captured";
|
|
22987
|
+
const status = String(d.status ?? inferredStatus);
|
|
22423
22988
|
const rows = captures.slice(0, 25).map((capture, i) => {
|
|
22424
22989
|
const url = typeof capture.url === "string" ? capture.url : "";
|
|
22425
22990
|
const sourceKind = typeof capture.sourceKind === "string" ? capture.sourceKind : typeof capture.source_kind === "string" ? capture.source_kind : "configured_target";
|
|
@@ -22428,8 +22993,10 @@ function formatCaptureSerpPageSnapshots(raw, input) {
|
|
|
22428
22993
|
const full = [
|
|
22429
22994
|
"# SERP Intelligence Page Snapshots",
|
|
22430
22995
|
`**Status:** ${status}`,
|
|
22431
|
-
`**
|
|
22996
|
+
`**Attempted:** ${captures.length}`,
|
|
22997
|
+
`**Captured:** ${capturedCount}`,
|
|
22432
22998
|
`**Failed:** ${failedCount}`,
|
|
22999
|
+
skippedCount ? `**Skipped:** ${skippedCount}` : "",
|
|
22433
23000
|
captures.length ? `
|
|
22434
23001
|
| # | URL | Source | Status |
|
|
22435
23002
|
|---|-----|--------|--------|
|
|
@@ -22441,8 +23008,11 @@ ${rows}` : ""
|
|
|
22441
23008
|
schemaVersion: "serp-intelligence.page-snapshots.v1",
|
|
22442
23009
|
status,
|
|
22443
23010
|
count: captures.length,
|
|
23011
|
+
capturedCount,
|
|
22444
23012
|
failedCount,
|
|
23013
|
+
skippedCount,
|
|
22445
23014
|
captures,
|
|
23015
|
+
attempts,
|
|
22446
23016
|
resolvedInputs: {
|
|
22447
23017
|
urls: input.urls ?? null,
|
|
22448
23018
|
targets: input.targets ?? null,
|
|
@@ -27982,12 +28552,22 @@ var init_schemas4 = __esm({
|
|
|
27982
28552
|
sourcePosition: import_zod33.z.number().int().min(1).optional()
|
|
27983
28553
|
}).strict();
|
|
27984
28554
|
SerpIntelligencePageSnapshotsBodySchema = import_zod33.z.object({
|
|
27985
|
-
urls: import_zod33.z.array(SerpIntelligencePublicHttpUrlSchema).min(1).max(25),
|
|
28555
|
+
urls: import_zod33.z.array(SerpIntelligencePublicHttpUrlSchema).min(1).max(25).optional(),
|
|
27986
28556
|
targets: import_zod33.z.array(SerpIntelligencePageSnapshotRequestSchema).min(1).max(25).optional(),
|
|
27987
28557
|
maxConcurrency: import_zod33.z.number().int().min(1).max(5).default(2),
|
|
27988
28558
|
timeoutMs: import_zod33.z.number().int().min(1e3).max(6e4).default(15e3),
|
|
27989
28559
|
debug: import_zod33.z.boolean().default(false)
|
|
27990
|
-
}).strict()
|
|
28560
|
+
}).strict().superRefine((body, ctx) => {
|
|
28561
|
+
const hasUrls = Boolean(body.urls?.length);
|
|
28562
|
+
const hasTargets = Boolean(body.targets?.length);
|
|
28563
|
+
if (hasUrls === hasTargets) {
|
|
28564
|
+
ctx.addIssue({
|
|
28565
|
+
code: import_zod33.z.ZodIssueCode.custom,
|
|
28566
|
+
message: "provide exactly one of urls or targets",
|
|
28567
|
+
path: hasUrls ? ["targets"] : ["urls"]
|
|
28568
|
+
});
|
|
28569
|
+
}
|
|
28570
|
+
});
|
|
27991
28571
|
SerpIntelligenceAICitationSchema = import_zod33.z.object({
|
|
27992
28572
|
text: import_zod33.z.string(),
|
|
27993
28573
|
href: import_zod33.z.string()
|
|
@@ -28247,14 +28827,24 @@ async function captureSerpIntelligenceSnapshot(rawInput, runtimeOptions = {}) {
|
|
|
28247
28827
|
});
|
|
28248
28828
|
const pageSnapshotLimit = normalizePageSnapshotLimit(parsedInput);
|
|
28249
28829
|
const pageSnapshotTargets = collectPageSnapshotTargets(harvestResult, pageSnapshotLimit);
|
|
28250
|
-
const pageSnapshotArtifacts = pageSnapshotTargets.length > 0 ? (await
|
|
28251
|
-
|
|
28252
|
-
|
|
28253
|
-
|
|
28254
|
-
|
|
28255
|
-
|
|
28256
|
-
|
|
28257
|
-
|
|
28830
|
+
const pageSnapshotArtifacts = pageSnapshotTargets.length > 0 ? (await runWithCostContext(
|
|
28831
|
+
{
|
|
28832
|
+
...currentCostContext() ?? { op: "serp" },
|
|
28833
|
+
subOp: "page_snapshot",
|
|
28834
|
+
// Page snapshot launches have their own fixed product rate. They
|
|
28835
|
+
// must not overwrite the headless/headful mode used to settle the
|
|
28836
|
+
// preceding SERP harvest.
|
|
28837
|
+
headlessSentOut: void 0
|
|
28838
|
+
},
|
|
28839
|
+
() => capturePageSnapshotsFn(pageSnapshotTargets, {
|
|
28840
|
+
kernelApiKey: runtimeOptions.kernelApiKey ?? browserServiceApiKey(),
|
|
28841
|
+
timeoutMs: runtimeOptions.pageSnapshotTimeoutMs,
|
|
28842
|
+
maxConcurrency: runtimeOptions.pageSnapshotMaxConcurrency,
|
|
28843
|
+
debug,
|
|
28844
|
+
includeContent: runtimeOptions.includePageSnapshotContent,
|
|
28845
|
+
maxContentChars: runtimeOptions.maxPageSnapshotContentChars
|
|
28846
|
+
})
|
|
28847
|
+
)).pageSnapshotArtifacts.map(stripPageSnapshotArtifact) : [];
|
|
28258
28848
|
return normalizeHarvestResultForSerpIntelligence({
|
|
28259
28849
|
harvestResult,
|
|
28260
28850
|
attempts,
|
|
@@ -28271,6 +28861,7 @@ var init_serp_capture_service = __esm({
|
|
|
28271
28861
|
"use strict";
|
|
28272
28862
|
init_harvest();
|
|
28273
28863
|
init_browser_service_env();
|
|
28864
|
+
init_cost_context();
|
|
28274
28865
|
init_harvest_problems();
|
|
28275
28866
|
init_page_snapshot_extractor();
|
|
28276
28867
|
init_schemas4();
|
|
@@ -28344,17 +28935,35 @@ async function enforceSerpIntelligenceRateLimit(c, userId) {
|
|
|
28344
28935
|
);
|
|
28345
28936
|
}
|
|
28346
28937
|
function pageSnapshotTargetsFromBody(body) {
|
|
28347
|
-
return body.targets?.length ? body.targets : body.urls.map((url) => ({
|
|
28938
|
+
return body.targets?.length ? body.targets : (body.urls ?? []).map((url) => ({
|
|
28348
28939
|
url,
|
|
28349
28940
|
sourceKind: "configured_target",
|
|
28350
28941
|
sourcePosition: null
|
|
28351
28942
|
}));
|
|
28352
28943
|
}
|
|
28353
|
-
|
|
28944
|
+
async function logRequestEventBestEffort(input) {
|
|
28945
|
+
try {
|
|
28946
|
+
await logRequestEvent(input);
|
|
28947
|
+
} catch (error) {
|
|
28948
|
+
console.warn("[serp-intelligence] request audit log failed:", error instanceof Error ? error.message : String(error));
|
|
28949
|
+
}
|
|
28950
|
+
}
|
|
28951
|
+
function captureBillingKeys(userId, suppliedKey, body) {
|
|
28952
|
+
const responseKey = suppliedKey?.trim() || (0, import_node_crypto12.randomUUID)();
|
|
28953
|
+
const requestFingerprint = (0, import_node_crypto12.createHash)("sha256").update(JSON.stringify(body)).digest("hex");
|
|
28954
|
+
const keyDigest = (0, import_node_crypto12.createHash)("sha256").update(String(userId)).update("\0").update(responseKey).digest("hex");
|
|
28955
|
+
return {
|
|
28956
|
+
responseKey,
|
|
28957
|
+
debitKey: `serp-capture:${userId}:${keyDigest}`,
|
|
28958
|
+
debitDescription: `${body.query} [request:${requestFingerprint}]`
|
|
28959
|
+
};
|
|
28960
|
+
}
|
|
28961
|
+
var import_hono16, import_node_crypto12, SERP_INTELLIGENCE_RATE_LIMIT, SERP_INTELLIGENCE_RATE_WINDOW_SECONDS, POST_CAPTURE_ROUTE_LABEL, POST_PAGE_SNAPSHOTS_ROUTE_LABEL, SERP_CAPTURE_BILLING_SOURCE, serpIntelligenceApp;
|
|
28354
28962
|
var init_serp_intelligence_routes = __esm({
|
|
28355
28963
|
"src/api/serp-intelligence-routes.ts"() {
|
|
28356
28964
|
"use strict";
|
|
28357
28965
|
import_hono16 = require("hono");
|
|
28966
|
+
import_node_crypto12 = require("crypto");
|
|
28358
28967
|
init_browser_service_env();
|
|
28359
28968
|
init_page_snapshot_extractor();
|
|
28360
28969
|
init_serp_capture_service();
|
|
@@ -28362,11 +28971,13 @@ var init_serp_intelligence_routes = __esm({
|
|
|
28362
28971
|
init_api_auth();
|
|
28363
28972
|
init_db();
|
|
28364
28973
|
init_rates();
|
|
28974
|
+
init_cost_context();
|
|
28365
28975
|
init_concurrency_gates();
|
|
28366
28976
|
SERP_INTELLIGENCE_RATE_LIMIT = 60;
|
|
28367
28977
|
SERP_INTELLIGENCE_RATE_WINDOW_SECONDS = 60;
|
|
28368
28978
|
POST_CAPTURE_ROUTE_LABEL = "POST /capture";
|
|
28369
28979
|
POST_PAGE_SNAPSHOTS_ROUTE_LABEL = "POST /page-snapshots";
|
|
28980
|
+
SERP_CAPTURE_BILLING_SOURCE = "serp_intelligence_capture";
|
|
28370
28981
|
serpIntelligenceApp = new import_hono16.Hono();
|
|
28371
28982
|
serpIntelligenceApp.use("*", createApiKeyAuth());
|
|
28372
28983
|
serpIntelligenceApp.post("/capture", async (c) => {
|
|
@@ -28375,6 +28986,17 @@ var init_serp_intelligence_routes = __esm({
|
|
|
28375
28986
|
const raw = await c.req.json().catch(() => ({}));
|
|
28376
28987
|
const parsed = SerpIntelligenceCaptureBodySchema.safeParse(raw);
|
|
28377
28988
|
if (!parsed.success) return c.json(formatZodError(parsed.error), 400);
|
|
28989
|
+
const suppliedIdempotencyKey = c.req.header("idempotency-key") ?? c.req.header("x-idempotency-key");
|
|
28990
|
+
if (suppliedIdempotencyKey && suppliedIdempotencyKey.trim().length > 500) {
|
|
28991
|
+
return c.json(structuredError({
|
|
28992
|
+
error_code: "invalid_request",
|
|
28993
|
+
error_type: "validation_error",
|
|
28994
|
+
message: "Idempotency-Key must be 500 characters or fewer.",
|
|
28995
|
+
retryable: false
|
|
28996
|
+
}), 400);
|
|
28997
|
+
}
|
|
28998
|
+
const billingKeys = captureBillingKeys(user.id, suppliedIdempotencyKey, parsed.data);
|
|
28999
|
+
c.header("Idempotency-Key", billingKeys.responseKey);
|
|
28378
29000
|
const limited = await enforceSerpIntelligenceRateLimit(c, user.id);
|
|
28379
29001
|
if (limited) return limited;
|
|
28380
29002
|
const gate = await acquireConcurrencyGate(user, "serp_intelligence_capture", {
|
|
@@ -28382,51 +29004,130 @@ var init_serp_intelligence_routes = __esm({
|
|
|
28382
29004
|
metadata: { query: parsed.data.query, location: parsed.data.location }
|
|
28383
29005
|
});
|
|
28384
29006
|
if (!gate.ok) return c.json(concurrencyLimitExceededResponse(gate), 429, { "Retry-After": String(gate.retryAfterSeconds) });
|
|
28385
|
-
const
|
|
28386
|
-
|
|
29007
|
+
const pageSnapshotHold = parsed.data.includePageSnapshots ? parsed.data.pageSnapshotLimit * MC_COSTS.page_scrape : 0;
|
|
29008
|
+
const cost = MC_COSTS.serp_headful + pageSnapshotHold;
|
|
29009
|
+
const headlessSentOut = { value: null };
|
|
28387
29010
|
try {
|
|
28388
|
-
const { ok, balance_mc } = await
|
|
29011
|
+
const { ok, balance_mc } = await debitMcIdempotent(
|
|
28389
29012
|
user.id,
|
|
28390
29013
|
cost,
|
|
28391
29014
|
LedgerOperation.SERP,
|
|
28392
|
-
|
|
29015
|
+
billingKeys.debitDescription,
|
|
29016
|
+
billingKeys.debitKey
|
|
28393
29017
|
);
|
|
28394
|
-
if (!ok)
|
|
28395
|
-
|
|
28396
|
-
|
|
28397
|
-
|
|
28398
|
-
|
|
28399
|
-
|
|
28400
|
-
|
|
29018
|
+
if (!ok) {
|
|
29019
|
+
await releaseConcurrencyGate(gate.lockId);
|
|
29020
|
+
return c.json(insufficientBalanceResponse(balance_mc, cost), 402);
|
|
29021
|
+
}
|
|
29022
|
+
} catch (error) {
|
|
29023
|
+
const idempotencyConflict = error instanceof Error && error.message === "idempotent debit key conflicts with another debit";
|
|
29024
|
+
const body = structuredError({
|
|
29025
|
+
error_code: idempotencyConflict ? "idempotency_conflict" : "capture_failed",
|
|
29026
|
+
error_type: idempotencyConflict ? "validation_error" : "capture_error",
|
|
29027
|
+
message: error instanceof Error ? error.message : String(error),
|
|
29028
|
+
retryable: !idempotencyConflict
|
|
28401
29029
|
});
|
|
28402
|
-
await
|
|
29030
|
+
await logRequestEventBestEffort({
|
|
28403
29031
|
userId: user.id,
|
|
28404
|
-
source:
|
|
28405
|
-
status: "
|
|
29032
|
+
source: SERP_CAPTURE_BILLING_SOURCE,
|
|
29033
|
+
status: "failed",
|
|
28406
29034
|
query: parsed.data.query,
|
|
28407
29035
|
location: parsed.data.location,
|
|
28408
|
-
|
|
28409
|
-
result
|
|
29036
|
+
error: body.message
|
|
28410
29037
|
});
|
|
28411
|
-
|
|
29038
|
+
await releaseConcurrencyGate(gate.lockId);
|
|
29039
|
+
return c.json(body, idempotencyConflict ? 409 : 500);
|
|
29040
|
+
}
|
|
29041
|
+
let result;
|
|
29042
|
+
try {
|
|
29043
|
+
const parentCostContext = currentCostContext();
|
|
29044
|
+
result = await runWithCostContext(
|
|
29045
|
+
{
|
|
29046
|
+
...parentCostContext ?? {},
|
|
29047
|
+
op: "serp",
|
|
29048
|
+
userId: Number(user.id),
|
|
29049
|
+
headlessSentOut
|
|
29050
|
+
},
|
|
29051
|
+
() => captureSerpIntelligenceSnapshot(parsed.data, {
|
|
29052
|
+
kernelApiKey: browserServiceApiKey(),
|
|
29053
|
+
kernelProxyId: browserServiceProxyId(),
|
|
29054
|
+
signal: c.req.raw.signal
|
|
29055
|
+
})
|
|
29056
|
+
);
|
|
28412
29057
|
} catch (error) {
|
|
28413
|
-
|
|
29058
|
+
try {
|
|
29059
|
+
await settleDebitMcIdempotent(
|
|
29060
|
+
user.id,
|
|
29061
|
+
billingKeys.debitKey,
|
|
29062
|
+
0,
|
|
29063
|
+
LedgerOperation.SERP_REFUND,
|
|
29064
|
+
"failed serp intelligence capture",
|
|
29065
|
+
SERP_CAPTURE_BILLING_SOURCE
|
|
29066
|
+
);
|
|
29067
|
+
} catch (settlementError) {
|
|
29068
|
+
console.warn(
|
|
29069
|
+
"[serp-intelligence] failed capture refund remains recoverable:",
|
|
29070
|
+
settlementError instanceof Error ? settlementError.message : String(settlementError)
|
|
29071
|
+
);
|
|
29072
|
+
}
|
|
28414
29073
|
const body = error instanceof SerpIntelligenceCaptureError ? error.toJSON() : structuredError({
|
|
28415
29074
|
error_code: "capture_failed",
|
|
28416
29075
|
error_type: "capture_error",
|
|
28417
29076
|
message: error instanceof Error ? error.message : String(error),
|
|
28418
29077
|
retryable: true
|
|
28419
29078
|
});
|
|
28420
|
-
await
|
|
29079
|
+
await logRequestEventBestEffort({
|
|
28421
29080
|
userId: user.id,
|
|
28422
|
-
source:
|
|
29081
|
+
source: SERP_CAPTURE_BILLING_SOURCE,
|
|
28423
29082
|
status: "failed",
|
|
28424
29083
|
query: parsed.data.query,
|
|
28425
29084
|
location: parsed.data.location,
|
|
28426
29085
|
error: "message" in body && typeof body.message === "string" ? body.message : "SERP Intelligence capture failed"
|
|
28427
29086
|
});
|
|
28428
29087
|
const status = error instanceof SerpIntelligenceCaptureError ? error.httpStatus : 500;
|
|
29088
|
+
await releaseConcurrencyGate(gate.lockId);
|
|
28429
29089
|
return c.json(body, status);
|
|
29090
|
+
}
|
|
29091
|
+
try {
|
|
29092
|
+
const requestedActualCost = serpActualCostMc(headlessSentOut.value) + result.pageSnapshotArtifacts.length * MC_COSTS.page_scrape;
|
|
29093
|
+
const settlement = await settleDebitMcIdempotent(
|
|
29094
|
+
user.id,
|
|
29095
|
+
billingKeys.debitKey,
|
|
29096
|
+
requestedActualCost,
|
|
29097
|
+
LedgerOperation.SERP_REFUND,
|
|
29098
|
+
"SERP capture pricing settle",
|
|
29099
|
+
SERP_CAPTURE_BILLING_SOURCE
|
|
29100
|
+
);
|
|
29101
|
+
const settledResult = {
|
|
29102
|
+
...result,
|
|
29103
|
+
billing: { creditsUsed: settlement.final_amount_mc / MC_PER_CREDIT }
|
|
29104
|
+
};
|
|
29105
|
+
await logRequestEventBestEffort({
|
|
29106
|
+
userId: user.id,
|
|
29107
|
+
source: SERP_CAPTURE_BILLING_SOURCE,
|
|
29108
|
+
status: "done",
|
|
29109
|
+
query: parsed.data.query,
|
|
29110
|
+
location: parsed.data.location,
|
|
29111
|
+
resultCount: settledResult.harvestResult.organicResults.length,
|
|
29112
|
+
result: settledResult
|
|
29113
|
+
});
|
|
29114
|
+
return c.json(settledResult);
|
|
29115
|
+
} catch (error) {
|
|
29116
|
+
const body = structuredError({
|
|
29117
|
+
error_code: "capture_failed",
|
|
29118
|
+
error_type: "capture_error",
|
|
29119
|
+
message: error instanceof Error ? error.message : String(error),
|
|
29120
|
+
retryable: true
|
|
29121
|
+
});
|
|
29122
|
+
await logRequestEventBestEffort({
|
|
29123
|
+
userId: user.id,
|
|
29124
|
+
source: SERP_CAPTURE_BILLING_SOURCE,
|
|
29125
|
+
status: "failed",
|
|
29126
|
+
query: parsed.data.query,
|
|
29127
|
+
location: parsed.data.location,
|
|
29128
|
+
error: body.message
|
|
29129
|
+
});
|
|
29130
|
+
return c.json(body, 500);
|
|
28430
29131
|
} finally {
|
|
28431
29132
|
await releaseConcurrencyGate(gate.lockId);
|
|
28432
29133
|
}
|
|
@@ -28462,7 +29163,7 @@ var init_serp_intelligence_routes = __esm({
|
|
|
28462
29163
|
maxConcurrency: parsed.data.maxConcurrency,
|
|
28463
29164
|
debug: parsed.data.debug
|
|
28464
29165
|
});
|
|
28465
|
-
await
|
|
29166
|
+
await logRequestEventBestEffort({
|
|
28466
29167
|
userId: user.id,
|
|
28467
29168
|
source: "serp_intelligence_page_snapshots",
|
|
28468
29169
|
status: "done",
|
|
@@ -28479,7 +29180,7 @@ var init_serp_intelligence_routes = __esm({
|
|
|
28479
29180
|
message: error instanceof Error ? error.message : String(error),
|
|
28480
29181
|
retryable: true
|
|
28481
29182
|
});
|
|
28482
|
-
await
|
|
29183
|
+
await logRequestEventBestEffort({
|
|
28483
29184
|
userId: user.id,
|
|
28484
29185
|
source: "serp_intelligence_page_snapshots",
|
|
28485
29186
|
status: "failed",
|
|
@@ -28499,7 +29200,7 @@ var PACKAGE_VERSION;
|
|
|
28499
29200
|
var init_version = __esm({
|
|
28500
29201
|
"src/version.ts"() {
|
|
28501
29202
|
"use strict";
|
|
28502
|
-
PACKAGE_VERSION = "0.21.
|
|
29203
|
+
PACKAGE_VERSION = "0.21.4";
|
|
28503
29204
|
}
|
|
28504
29205
|
});
|
|
28505
29206
|
|
|
@@ -29785,9 +30486,19 @@ var init_mcp_tool_schemas = __esm({
|
|
|
29785
30486
|
location: NullableString,
|
|
29786
30487
|
capturedAt: NullableString,
|
|
29787
30488
|
resultCount: import_zod34.z.number().int().min(0).nullable(),
|
|
30489
|
+
organicResultCount: import_zod34.z.number().int().min(0),
|
|
30490
|
+
localPackResultCount: import_zod34.z.number().int().min(0),
|
|
30491
|
+
pageSnapshotCount: import_zod34.z.number().int().min(0),
|
|
30492
|
+
pageSnapshotFailedCount: import_zod34.z.number().int().min(0),
|
|
29788
30493
|
snapshotId: NullableString,
|
|
29789
30494
|
resolvedInputs: import_zod34.z.record(import_zod34.z.unknown()),
|
|
29790
30495
|
artifacts: import_zod34.z.array(import_zod34.z.record(import_zod34.z.unknown())),
|
|
30496
|
+
attempts: import_zod34.z.array(import_zod34.z.record(import_zod34.z.unknown())),
|
|
30497
|
+
billing: import_zod34.z.object({
|
|
30498
|
+
creditsUsed: import_zod34.z.number().min(0).nullable(),
|
|
30499
|
+
requestId: NullableString,
|
|
30500
|
+
jobId: NullableString
|
|
30501
|
+
}).nullable(),
|
|
29791
30502
|
diagnostics: import_zod34.z.record(import_zod34.z.unknown()).nullable(),
|
|
29792
30503
|
providerPayload: import_zod34.z.record(import_zod34.z.unknown())
|
|
29793
30504
|
};
|
|
@@ -29795,8 +30506,11 @@ var init_mcp_tool_schemas = __esm({
|
|
|
29795
30506
|
schemaVersion: import_zod34.z.literal("serp-intelligence.page-snapshots.v1"),
|
|
29796
30507
|
status: import_zod34.z.string(),
|
|
29797
30508
|
count: import_zod34.z.number().int().min(0),
|
|
30509
|
+
capturedCount: import_zod34.z.number().int().min(0),
|
|
29798
30510
|
failedCount: import_zod34.z.number().int().min(0),
|
|
30511
|
+
skippedCount: import_zod34.z.number().int().min(0),
|
|
29799
30512
|
captures: import_zod34.z.array(import_zod34.z.record(import_zod34.z.unknown())),
|
|
30513
|
+
attempts: import_zod34.z.array(import_zod34.z.record(import_zod34.z.unknown())),
|
|
29800
30514
|
resolvedInputs: import_zod34.z.record(import_zod34.z.unknown()),
|
|
29801
30515
|
diagnostics: import_zod34.z.record(import_zod34.z.unknown()).nullable(),
|
|
29802
30516
|
providerPayload: import_zod34.z.record(import_zod34.z.unknown())
|
|
@@ -29921,8 +30635,8 @@ var init_mcp_tool_schemas = __esm({
|
|
|
29921
30635
|
proxyZip: import_zod34.z.string().regex(/^\d{5}$/).optional().describe('US ZIP for residential geo-IP targeting. Only meaningful with proxyMode "location".'),
|
|
29922
30636
|
pages: import_zod34.z.number().int().min(1).max(2).default(1).describe("Google result pages to capture. Use 2 only for deeper ranking evidence."),
|
|
29923
30637
|
debug: import_zod34.z.boolean().default(false).describe("Include sanitized browser/proxy/location diagnostics."),
|
|
29924
|
-
includePageSnapshots: import_zod34.z.boolean().default(false).describe("Also capture ranking-page snapshots for selected SERP URLs."),
|
|
29925
|
-
pageSnapshotLimit: import_zod34.z.number().int().min(0).max(10).default(0).describe("Maximum ranking-page snapshots when includePageSnapshots is true.")
|
|
30638
|
+
includePageSnapshots: import_zod34.z.boolean().default(false).describe("Also capture ranking-page snapshots for selected SERP URLs. Each attempted snapshot adds 1 Credit."),
|
|
30639
|
+
pageSnapshotLimit: import_zod34.z.number().int().min(0).max(10).default(0).describe("Maximum ranking-page snapshots when includePageSnapshots is true. This capacity is held up front and unused capacity is refunded.")
|
|
29926
30640
|
};
|
|
29927
30641
|
ScreenshotInputSchema = {
|
|
29928
30642
|
url: import_zod34.z.string().url().describe("URL to capture as a full-page screenshot. Use http or https. Pass allowLocal: true to capture localhost or private-network URLs during development."),
|
|
@@ -29930,12 +30644,12 @@ var init_mcp_tool_schemas = __esm({
|
|
|
29930
30644
|
allowLocal: import_zod34.z.boolean().default(false).describe("Allow localhost and private-network URLs (127.x, 192.168.x, 10.x, etc.). For local development only \u2014 not for production use.")
|
|
29931
30645
|
};
|
|
29932
30646
|
CaptureSerpPageSnapshotsInputSchema = {
|
|
29933
|
-
urls: import_zod34.z.array(import_zod34.z.string().url()).min(1).max(25).describe("Public HTTP/HTTPS URLs to capture. Do not pass localhost, private IPs, file URLs, or internal admin URLs."),
|
|
30647
|
+
urls: import_zod34.z.array(import_zod34.z.string().url()).min(1).max(25).optional().describe("Public HTTP/HTTPS URLs to capture. Provide exactly one of urls or targets. Do not pass localhost, private IPs, file URLs, or internal admin URLs."),
|
|
29934
30648
|
targets: import_zod34.z.array(import_zod34.z.object({
|
|
29935
30649
|
url: import_zod34.z.string().url().describe("Public HTTP/HTTPS URL to capture."),
|
|
29936
30650
|
sourceKind: import_zod34.z.enum(["organic", "ai_citation", "local_pack_website", "configured_target", "site_subject"]).default("configured_target").describe("Why this page is being captured."),
|
|
29937
30651
|
sourcePosition: import_zod34.z.number().int().min(1).optional().describe("Ranking or citation position when the page came from SERP evidence.")
|
|
29938
|
-
}).strict()).min(1).max(25).optional().describe("Structured targets.
|
|
30652
|
+
}).strict()).min(1).max(25).optional().describe("Structured targets. Provide exactly one of targets or urls; use targets when source kind or position should be preserved."),
|
|
29939
30653
|
maxConcurrency: import_zod34.z.number().int().min(1).max(5).default(2).describe("Parallel page captures."),
|
|
29940
30654
|
timeoutMs: import_zod34.z.number().int().min(1e3).max(6e4).default(15e3).describe("Per-page capture timeout in milliseconds; timeouts return as structured capture failures."),
|
|
29941
30655
|
debug: import_zod34.z.boolean().default(false).describe("Include sanitized browser/proxy diagnostics.")
|
|
@@ -30571,7 +31285,7 @@ var init_rank_tracker_blueprint = __esm({
|
|
|
30571
31285
|
|
|
30572
31286
|
// src/mcp/paa-mcp-server.ts
|
|
30573
31287
|
function hashOwnerId(callerKey) {
|
|
30574
|
-
return (0,
|
|
31288
|
+
return (0, import_node_crypto13.createHash)("sha256").update(callerKey).digest("hex").slice(0, 24);
|
|
30575
31289
|
}
|
|
30576
31290
|
function liveWebToolAnnotations(title) {
|
|
30577
31291
|
return {
|
|
@@ -30585,14 +31299,14 @@ function liveWebToolAnnotations(title) {
|
|
|
30585
31299
|
function registerSerpIntelligenceCaptureTools(server, executor) {
|
|
30586
31300
|
server.registerTool("capture_serp_snapshot", {
|
|
30587
31301
|
title: "SERP Intelligence Snapshot",
|
|
30588
|
-
description: "Capture a structured SERP Intelligence snapshot of a Google query \u2014 the persistent evidence format used by rank-tracking and comparison pipelines. Split query from location; leave proxyMode unset.",
|
|
31302
|
+
description: "Capture a structured SERP Intelligence snapshot of a Google query \u2014 the persistent evidence format used by rank-tracking and comparison pipelines. Split query from location; leave proxyMode unset. Costs 4 Credits when headless or 14 if anti-bot escalation requires headful mode; the 14-Credit hold is settled to the mode used. Optional page snapshots add 1 Credit per attempted URL.",
|
|
30589
31303
|
inputSchema: CaptureSerpSnapshotInputSchema,
|
|
30590
31304
|
outputSchema: recordOutputSchema("capture_serp_snapshot", CaptureSerpSnapshotOutputSchema),
|
|
30591
31305
|
annotations: liveWebToolAnnotations("SERP Intelligence Snapshot")
|
|
30592
31306
|
}, async (input) => formatCaptureSerpSnapshot(await executor.captureSerpSnapshot(input), input));
|
|
30593
31307
|
server.registerTool("capture_serp_page_snapshots", {
|
|
30594
31308
|
title: "SERP Intelligence Page Snapshots",
|
|
30595
|
-
description: "Capture public ranking pages as SERP Intelligence page snapshots \u2014 persistent page evidence linked to a captured SERP. Provide urls
|
|
31309
|
+
description: "Capture public ranking pages as SERP Intelligence page snapshots \u2014 persistent page evidence linked to a captured SERP. Provide exactly one of urls or targets; use targets to preserve source metadata. Private IPs, localhost, file, and internal URLs are rejected. Costs 1 Credit per attempted URL.",
|
|
30596
31310
|
inputSchema: CaptureSerpPageSnapshotsInputSchema,
|
|
30597
31311
|
outputSchema: recordOutputSchema("capture_serp_page_snapshots", CaptureSerpPageSnapshotsOutputSchema),
|
|
30598
31312
|
annotations: liveWebToolAnnotations("SERP Intelligence Page Snapshots")
|
|
@@ -31015,14 +31729,14 @@ function registerPaaExtractorMcpTools(server, executor, options = {}) {
|
|
|
31015
31729
|
annotations: { title: "Set Scheduled Action Connections", readOnlyHint: false, destructiveHint: false, idempotentHint: true, openWorldHint: false }
|
|
31016
31730
|
}, async (input) => executor.setScheduledActionConnections(input));
|
|
31017
31731
|
}
|
|
31018
|
-
var import_mcp, import_node_fs9, import_node_path13,
|
|
31732
|
+
var import_mcp, import_node_fs9, import_node_path13, import_node_crypto13;
|
|
31019
31733
|
var init_paa_mcp_server = __esm({
|
|
31020
31734
|
"src/mcp/paa-mcp-server.ts"() {
|
|
31021
31735
|
"use strict";
|
|
31022
31736
|
import_mcp = require("@modelcontextprotocol/sdk/server/mcp.js");
|
|
31023
31737
|
import_node_fs9 = require("fs");
|
|
31024
31738
|
import_node_path13 = require("path");
|
|
31025
|
-
|
|
31739
|
+
import_node_crypto13 = require("crypto");
|
|
31026
31740
|
init_version();
|
|
31027
31741
|
init_mcp_response_formatter();
|
|
31028
31742
|
init_server_instructions();
|
|
@@ -31092,10 +31806,11 @@ function withDefaultHostedProfile(input) {
|
|
|
31092
31806
|
...typeof saveProfileChanges === "boolean" ? { saveProfileChanges } : {}
|
|
31093
31807
|
};
|
|
31094
31808
|
}
|
|
31095
|
-
var HttpMcpToolExecutor;
|
|
31809
|
+
var import_node_crypto14, HttpMcpToolExecutor;
|
|
31096
31810
|
var init_http_mcp_tool_executor = __esm({
|
|
31097
31811
|
"src/mcp/http-mcp-tool-executor.ts"() {
|
|
31098
31812
|
"use strict";
|
|
31813
|
+
import_node_crypto14 = require("crypto");
|
|
31099
31814
|
init_harvest_timeout();
|
|
31100
31815
|
init_browser_service_env();
|
|
31101
31816
|
HttpMcpToolExecutor = class {
|
|
@@ -31104,6 +31819,7 @@ var init_http_mcp_tool_executor = __esm({
|
|
|
31104
31819
|
timeoutMs;
|
|
31105
31820
|
httpTimeoutOverrideMs;
|
|
31106
31821
|
serpIntelligenceTimeoutMs;
|
|
31822
|
+
pendingSerpCaptureBillingKeys = /* @__PURE__ */ new Map();
|
|
31107
31823
|
constructor(baseUrl, apiKey) {
|
|
31108
31824
|
this.baseUrl = baseUrl.replace(/\/$/, "");
|
|
31109
31825
|
this.apiKey = apiKey;
|
|
@@ -31114,13 +31830,14 @@ var init_http_mcp_tool_executor = __esm({
|
|
|
31114
31830
|
const configuredSerpIntelligenceTimeoutMs = Number(process.env.MCP_SCRAPER_SERP_INTELLIGENCE_HTTP_TIMEOUT_MS ?? this.timeoutMs);
|
|
31115
31831
|
this.serpIntelligenceTimeoutMs = Number.isFinite(configuredSerpIntelligenceTimeoutMs) && configuredSerpIntelligenceTimeoutMs > 0 ? configuredSerpIntelligenceTimeoutMs : this.timeoutMs;
|
|
31116
31832
|
}
|
|
31117
|
-
async call(path6, body, timeoutMs = this.timeoutMs, method = "POST") {
|
|
31833
|
+
async call(path6, body, timeoutMs = this.timeoutMs, method = "POST", requestHeaders = {}) {
|
|
31118
31834
|
try {
|
|
31119
31835
|
const res = await fetch(`${this.baseUrl}${path6}`, {
|
|
31120
31836
|
method,
|
|
31121
31837
|
headers: {
|
|
31122
31838
|
"Content-Type": "application/json",
|
|
31123
|
-
"x-api-key": this.apiKey
|
|
31839
|
+
"x-api-key": this.apiKey,
|
|
31840
|
+
...requestHeaders
|
|
31124
31841
|
},
|
|
31125
31842
|
body: JSON.stringify(body),
|
|
31126
31843
|
signal: AbortSignal.timeout(timeoutMs)
|
|
@@ -31368,8 +32085,27 @@ var init_http_mcp_tool_executor = __esm({
|
|
|
31368
32085
|
connections: input.connections
|
|
31369
32086
|
}, void 0, "PUT");
|
|
31370
32087
|
}
|
|
31371
|
-
captureSerpSnapshot(input) {
|
|
31372
|
-
|
|
32088
|
+
async captureSerpSnapshot(input) {
|
|
32089
|
+
const fingerprint = (0, import_node_crypto14.createHash)("sha256").update(JSON.stringify(input)).digest("hex");
|
|
32090
|
+
const now = Date.now();
|
|
32091
|
+
for (const [pendingFingerprint, pendingEntry] of this.pendingSerpCaptureBillingKeys) {
|
|
32092
|
+
if (pendingEntry.expiresAt <= now) this.pendingSerpCaptureBillingKeys.delete(pendingFingerprint);
|
|
32093
|
+
}
|
|
32094
|
+
const pending = this.pendingSerpCaptureBillingKeys.get(fingerprint);
|
|
32095
|
+
const idempotencyKey = pending && pending.expiresAt > now ? pending.key : (0, import_node_crypto14.randomUUID)();
|
|
32096
|
+
this.pendingSerpCaptureBillingKeys.set(fingerprint, {
|
|
32097
|
+
key: idempotencyKey,
|
|
32098
|
+
expiresAt: now + 15 * 6e4
|
|
32099
|
+
});
|
|
32100
|
+
const result = await this.call(
|
|
32101
|
+
"/serp-intelligence/capture",
|
|
32102
|
+
input,
|
|
32103
|
+
this.serpIntelligenceTimeoutMs,
|
|
32104
|
+
"POST",
|
|
32105
|
+
{ "Idempotency-Key": idempotencyKey }
|
|
32106
|
+
);
|
|
32107
|
+
if (!result.isError) this.pendingSerpCaptureBillingKeys.delete(fingerprint);
|
|
32108
|
+
return result;
|
|
31373
32109
|
}
|
|
31374
32110
|
captureSerpPageSnapshots(input) {
|
|
31375
32111
|
return this.call("/serp-intelligence/page-snapshots", input, this.serpIntelligenceTimeoutMs);
|
|
@@ -36063,7 +36799,33 @@ var init_mcp_routes = __esm({
|
|
|
36063
36799
|
// src/api/browser-agent-db.ts
|
|
36064
36800
|
async function migrateBrowserAgent() {
|
|
36065
36801
|
if (_ready2) return;
|
|
36802
|
+
if (_migrationPromise2) return _migrationPromise2;
|
|
36803
|
+
_migrationPromise2 = runBrowserAgentMigration().finally(() => {
|
|
36804
|
+
_migrationPromise2 = null;
|
|
36805
|
+
});
|
|
36806
|
+
return _migrationPromise2;
|
|
36807
|
+
}
|
|
36808
|
+
async function runBrowserAgentMigration() {
|
|
36066
36809
|
const db = getDb();
|
|
36810
|
+
const existing = await db.execute(`
|
|
36811
|
+
SELECT
|
|
36812
|
+
(SELECT COUNT(*) FROM sqlite_master WHERE type = 'table' AND name IN (
|
|
36813
|
+
'browser_agent_sessions',
|
|
36814
|
+
'browser_agent_actions',
|
|
36815
|
+
'browser_agent_replays',
|
|
36816
|
+
'browser_auth_connections',
|
|
36817
|
+
'browser_agent_extensions'
|
|
36818
|
+
)) = 5
|
|
36819
|
+
AND (SELECT COUNT(*) FROM pragma_table_info('browser_agent_sessions') WHERE name = 'concurrency_lock_id') = 1
|
|
36820
|
+
AND (SELECT COUNT(*) FROM pragma_table_info('browser_auth_connections') WHERE name IN (
|
|
36821
|
+
'account_email', 'note', 'updated_at', 'status', 'browser_agent_session_id'
|
|
36822
|
+
)) = 5
|
|
36823
|
+
AS ready
|
|
36824
|
+
`);
|
|
36825
|
+
if (Number(existing.rows[0]?.ready ?? 0) === 1) {
|
|
36826
|
+
_ready2 = true;
|
|
36827
|
+
return;
|
|
36828
|
+
}
|
|
36067
36829
|
await db.execute(`
|
|
36068
36830
|
CREATE TABLE IF NOT EXISTS browser_agent_sessions (
|
|
36069
36831
|
id TEXT PRIMARY KEY,
|
|
@@ -36163,7 +36925,7 @@ async function migrateBrowserAgent() {
|
|
|
36163
36925
|
}
|
|
36164
36926
|
async function createExtensionRow(input) {
|
|
36165
36927
|
const db = getDb();
|
|
36166
|
-
const id = `bext_${(0,
|
|
36928
|
+
const id = `bext_${(0, import_node_crypto15.randomUUID)().replace(/-/g, "").slice(0, 20)}`;
|
|
36167
36929
|
await db.execute({
|
|
36168
36930
|
sql: `INSERT INTO browser_agent_extensions (id, user_id, name, backend_id, backend_name, source, source_url, size_bytes)
|
|
36169
36931
|
VALUES (?, ?, ?, ?, ?, ?, ?, ?)`,
|
|
@@ -36198,7 +36960,7 @@ async function deleteExtensionRow(userId, name) {
|
|
|
36198
36960
|
}
|
|
36199
36961
|
async function createAuthConnectionRow(input) {
|
|
36200
36962
|
const db = getDb();
|
|
36201
|
-
const connectionId = `authc_${(0,
|
|
36963
|
+
const connectionId = `authc_${(0, import_node_crypto15.randomUUID)().replace(/-/g, "").slice(0, 20)}`;
|
|
36202
36964
|
await db.execute({
|
|
36203
36965
|
sql: `INSERT INTO browser_auth_connections (connection_id, domain, profile, account_email, note, status, browser_agent_session_id)
|
|
36204
36966
|
VALUES (?, ?, ?, ?, ?, 'NEEDS_AUTH', ?)`,
|
|
@@ -36232,7 +36994,7 @@ async function listAuthConnectionsByProfile(profile) {
|
|
|
36232
36994
|
}
|
|
36233
36995
|
async function createSessionRow(input) {
|
|
36234
36996
|
const db = getDb();
|
|
36235
|
-
const id = `bas_${(0,
|
|
36997
|
+
const id = `bas_${(0, import_node_crypto15.randomUUID)().replace(/-/g, "").slice(0, 20)}`;
|
|
36236
36998
|
await db.execute({
|
|
36237
36999
|
sql: `INSERT INTO browser_agent_sessions (id, runtime_session_id, live_view_url, cdp_ws_url, status, label, user_id, concurrency_lock_id, last_action_at)
|
|
36238
37000
|
VALUES (?, ?, ?, ?, 'open', ?, ?, ?, datetime('now'))`,
|
|
@@ -36301,7 +37063,7 @@ async function recordAction(input) {
|
|
|
36301
37063
|
sql: `INSERT INTO browser_agent_actions (id, session_id, type, params_json, ok, error)
|
|
36302
37064
|
VALUES (?, ?, ?, ?, ?, ?)`,
|
|
36303
37065
|
args: [
|
|
36304
|
-
`baa_${(0,
|
|
37066
|
+
`baa_${(0, import_node_crypto15.randomUUID)().replace(/-/g, "").slice(0, 20)}`,
|
|
36305
37067
|
input.sessionId,
|
|
36306
37068
|
input.type,
|
|
36307
37069
|
input.params == null ? null : JSON.stringify(input.params),
|
|
@@ -36341,13 +37103,14 @@ async function listReplayRows(sessionId) {
|
|
|
36341
37103
|
});
|
|
36342
37104
|
return res.rows;
|
|
36343
37105
|
}
|
|
36344
|
-
var
|
|
37106
|
+
var import_node_crypto15, _ready2, _migrationPromise2;
|
|
36345
37107
|
var init_browser_agent_db = __esm({
|
|
36346
37108
|
"src/api/browser-agent-db.ts"() {
|
|
36347
37109
|
"use strict";
|
|
36348
|
-
|
|
37110
|
+
import_node_crypto15 = require("crypto");
|
|
36349
37111
|
init_db();
|
|
36350
37112
|
_ready2 = false;
|
|
37113
|
+
_migrationPromise2 = null;
|
|
36351
37114
|
}
|
|
36352
37115
|
});
|
|
36353
37116
|
|
|
@@ -38057,7 +38820,7 @@ function buildBrowserAgentRoutes() {
|
|
|
38057
38820
|
}
|
|
38058
38821
|
const existing = await getExtensionRow(user.id, name);
|
|
38059
38822
|
if (existing) return c.json({ error: `an extension named "${name}" already exists \u2014 delete it first or pick another name` }, 409);
|
|
38060
|
-
const backendName = `u${user.id}_${(0,
|
|
38823
|
+
const backendName = `u${user.id}_${(0, import_node_crypto16.randomUUID)().replace(/-/g, "")}`;
|
|
38061
38824
|
try {
|
|
38062
38825
|
const imported = await importExtensionFromStore(storeUrl, backendName);
|
|
38063
38826
|
const row = await createExtensionRow({
|
|
@@ -38116,11 +38879,11 @@ function buildBrowserAgentRoutes() {
|
|
|
38116
38879
|
});
|
|
38117
38880
|
return app2;
|
|
38118
38881
|
}
|
|
38119
|
-
var
|
|
38882
|
+
var import_node_crypto16, import_hono18, auth, DEFAULT_BROWSER_SESSION_LOCK_TTL_SECONDS, EXTENSION_NAME_RE;
|
|
38120
38883
|
var init_browser_agent_routes = __esm({
|
|
38121
38884
|
"src/api/browser-agent-routes.ts"() {
|
|
38122
38885
|
"use strict";
|
|
38123
|
-
|
|
38886
|
+
import_node_crypto16 = require("crypto");
|
|
38124
38887
|
import_hono18 = require("hono");
|
|
38125
38888
|
init_api_auth();
|
|
38126
38889
|
init_errors();
|
|
@@ -38530,7 +39293,7 @@ async function getKeys() {
|
|
|
38530
39293
|
const privateKey = await (0, import_jose2.importPKCS8)(pem, "RS256", { extractable: true });
|
|
38531
39294
|
const full = await (0, import_jose2.exportJWK)(privateKey);
|
|
38532
39295
|
const publicJwk = { kty: full.kty, n: full.n, e: full.e };
|
|
38533
|
-
const kid = (0,
|
|
39296
|
+
const kid = (0, import_node_crypto17.createHash)("sha256").update(JSON.stringify({ e: publicJwk.e, kty: publicJwk.kty, n: publicJwk.n })).digest("base64url").slice(0, 16);
|
|
38534
39297
|
publicJwk.kid = kid;
|
|
38535
39298
|
publicJwk.alg = "RS256";
|
|
38536
39299
|
publicJwk.use = "sig";
|
|
@@ -38709,23 +39472,23 @@ async function validateAuthRequest(p) {
|
|
|
38709
39472
|
}
|
|
38710
39473
|
function pkceMatches(verifier, challenge) {
|
|
38711
39474
|
if (!verifier) return false;
|
|
38712
|
-
const computed = (0,
|
|
39475
|
+
const computed = (0, import_node_crypto17.createHash)("sha256").update(verifier).digest("base64url");
|
|
38713
39476
|
return computed === challenge;
|
|
38714
39477
|
}
|
|
38715
39478
|
async function mintAccessToken(identity, scope, plan, audience) {
|
|
38716
39479
|
const { privateKey, kid } = await getKeys();
|
|
38717
|
-
return new import_jose2.SignJWT({ scope, plan }).setProtectedHeader({ alg: "RS256", kid }).setIssuer(ISSUER).setSubject(identity).setAudience(audience).setIssuedAt().setJti((0,
|
|
39480
|
+
return new import_jose2.SignJWT({ scope, plan }).setProtectedHeader({ alg: "RS256", kid }).setIssuer(ISSUER).setSubject(identity).setAudience(audience).setIssuedAt().setJti((0, import_node_crypto17.randomUUID)()).setExpirationTime(`${ACCESS_TTL_SECONDS}s`).sign(privateKey);
|
|
38718
39481
|
}
|
|
38719
39482
|
function tokenErrorResponse(c, error, description, status) {
|
|
38720
39483
|
return c.json({ error, error_description: description }, status);
|
|
38721
39484
|
}
|
|
38722
|
-
var import_hono20, import_cookie,
|
|
39485
|
+
var import_hono20, import_cookie, import_node_crypto17, import_jose2, ISSUER, RESOURCE, SCRAPER_RESOURCE, MEMORY_SCOPES, SCRAPER_SCOPES, SUPPORTED_SCOPES, ACCESS_TTL_SECONDS, REFRESH_TTL_SECONDS, CODE_TTL_SECONDS, ROTATION_GRACE_SECONDS, secureCookies, sessionCookieOptions, cachedKeys, oauthApp;
|
|
38723
39486
|
var init_oauth_routes = __esm({
|
|
38724
39487
|
"src/api/oauth-routes.ts"() {
|
|
38725
39488
|
"use strict";
|
|
38726
39489
|
import_hono20 = require("hono");
|
|
38727
39490
|
import_cookie = require("hono/cookie");
|
|
38728
|
-
|
|
39491
|
+
import_node_crypto17 = require("crypto");
|
|
38729
39492
|
import_jose2 = require("jose");
|
|
38730
39493
|
init_session();
|
|
38731
39494
|
init_db();
|
|
@@ -38811,7 +39574,7 @@ var init_oauth_routes = __esm({
|
|
|
38811
39574
|
}
|
|
38812
39575
|
}
|
|
38813
39576
|
const clientName = typeof body.client_name === "string" ? body.client_name : null;
|
|
38814
|
-
const clientId = `client_${(0,
|
|
39577
|
+
const clientId = `client_${(0, import_node_crypto17.randomBytes)(16).toString("hex")}`;
|
|
38815
39578
|
await registerClient(clientId, redirectUris, clientName);
|
|
38816
39579
|
console.log("[oauth-dcr] register OK client_id=%s redirect_uris=%s", clientId, JSON.stringify(redirectUris));
|
|
38817
39580
|
return c.json({
|
|
@@ -38864,7 +39627,7 @@ var init_oauth_routes = __esm({
|
|
|
38864
39627
|
if (action === "deny") return redirectWithError(p.redirect_uri, p.state, "access_denied");
|
|
38865
39628
|
if (action !== "approve") return c.text("unsupported action", 400);
|
|
38866
39629
|
const scope = negotiateScope(p.scope, user, p.resource);
|
|
38867
|
-
const code = `code_${(0,
|
|
39630
|
+
const code = `code_${(0, import_node_crypto17.randomBytes)(32).toString("base64url")}`;
|
|
38868
39631
|
const expiresAt = new Date(Date.now() + CODE_TTL_SECONDS * 1e3).toISOString();
|
|
38869
39632
|
await putCode({
|
|
38870
39633
|
code,
|
|
@@ -38903,7 +39666,7 @@ var init_oauth_routes = __esm({
|
|
|
38903
39666
|
const plan = user ? resolvePlan(user) : "free";
|
|
38904
39667
|
const audience = record.resource ?? RESOURCE();
|
|
38905
39668
|
const accessToken = await mintAccessToken(record.identity, record.scope, plan, audience);
|
|
38906
|
-
const refreshToken = `rt_${(0,
|
|
39669
|
+
const refreshToken = `rt_${(0, import_node_crypto17.randomBytes)(40).toString("base64url")}`;
|
|
38907
39670
|
await putRefresh({
|
|
38908
39671
|
refresh_token: refreshToken,
|
|
38909
39672
|
client_id: clientId,
|
|
@@ -38933,7 +39696,7 @@ var init_oauth_routes = __esm({
|
|
|
38933
39696
|
const plan = user ? resolvePlan(user) : "free";
|
|
38934
39697
|
const audience = record.resource ?? RESOURCE();
|
|
38935
39698
|
const accessToken = await mintAccessToken(record.identity, record.scope, plan, audience);
|
|
38936
|
-
const nextRefresh = `rt_${(0,
|
|
39699
|
+
const nextRefresh = `rt_${(0, import_node_crypto17.randomBytes)(40).toString("base64url")}`;
|
|
38937
39700
|
await rotateRefresh(refreshToken, {
|
|
38938
39701
|
refresh_token: nextRefresh,
|
|
38939
39702
|
client_id: record.client_id,
|
|
@@ -40250,7 +41013,7 @@ var init_site_audit_worker = __esm({
|
|
|
40250
41013
|
|
|
40251
41014
|
// src/api/page-diff.ts
|
|
40252
41015
|
function sha256Hex(value) {
|
|
40253
|
-
return (0,
|
|
41016
|
+
return (0, import_node_crypto18.createHash)("sha256").update(value).digest("hex");
|
|
40254
41017
|
}
|
|
40255
41018
|
function truncateForStorage(value, maxChars = MAX_SNAPSHOT_CONTENT_CHARS) {
|
|
40256
41019
|
if (value.length <= maxChars) return { value, truncated: false };
|
|
@@ -40307,11 +41070,11 @@ function diffPageContent(oldContent, newContent) {
|
|
|
40307
41070
|
totalChangedLineCount
|
|
40308
41071
|
};
|
|
40309
41072
|
}
|
|
40310
|
-
var
|
|
41073
|
+
var import_node_crypto18, import_diff, MAX_SNAPSHOT_CONTENT_CHARS, MAX_DIFF_HUNKS, MAX_DIFF_LINES_PER_RESPONSE;
|
|
40311
41074
|
var init_page_diff = __esm({
|
|
40312
41075
|
"src/api/page-diff.ts"() {
|
|
40313
41076
|
"use strict";
|
|
40314
|
-
|
|
41077
|
+
import_node_crypto18 = require("crypto");
|
|
40315
41078
|
import_diff = require("diff");
|
|
40316
41079
|
MAX_SNAPSHOT_CONTENT_CHARS = 25e4;
|
|
40317
41080
|
MAX_DIFF_HUNKS = 200;
|
|
@@ -40388,7 +41151,7 @@ function canonicalJson(value) {
|
|
|
40388
41151
|
return JSON.stringify(value);
|
|
40389
41152
|
}
|
|
40390
41153
|
function sha2562(value) {
|
|
40391
|
-
return (0,
|
|
41154
|
+
return (0, import_node_crypto19.createHash)("sha256").update(value).digest("hex");
|
|
40392
41155
|
}
|
|
40393
41156
|
function sensitiveKey(key) {
|
|
40394
41157
|
const normalized = key.toLowerCase().replace(/[^a-z0-9]/g, "");
|
|
@@ -40603,11 +41366,11 @@ async function importServiceConnectionToMemory(identity, input, dependencies) {
|
|
|
40603
41366
|
...!searchReady ? { warning: "The snapshot was stored, but no search chunks were indexed yet." } : {}
|
|
40604
41367
|
};
|
|
40605
41368
|
}
|
|
40606
|
-
var
|
|
41369
|
+
var import_node_crypto19, CONNECTION_MEMORY_IMPORT_MAX_ARGS_BYTES, CONNECTION_MEMORY_IMPORT_MAX_RESULT_BYTES, CONNECTION_MEMORY_IMPORT_MAX_STRING_CHARS, CONNECTION_MEMORY_IMPORT_MAX_DEPTH, ConnectionMemoryImportError;
|
|
40607
41370
|
var init_connection_memory_import = __esm({
|
|
40608
41371
|
"src/api/connection-memory-import.ts"() {
|
|
40609
41372
|
"use strict";
|
|
40610
|
-
|
|
41373
|
+
import_node_crypto19 = require("crypto");
|
|
40611
41374
|
init_slugify();
|
|
40612
41375
|
CONNECTION_MEMORY_IMPORT_MAX_ARGS_BYTES = 64 * 1024;
|
|
40613
41376
|
CONNECTION_MEMORY_IMPORT_MAX_RESULT_BYTES = 1e6;
|
|
@@ -40781,7 +41544,7 @@ function finitePositive(value, fallback) {
|
|
|
40781
41544
|
return Number.isFinite(value) && value > 0 ? value : fallback;
|
|
40782
41545
|
}
|
|
40783
41546
|
async function collectConnectedDataExport(args) {
|
|
40784
|
-
const exportId = (0,
|
|
41547
|
+
const exportId = (0, import_node_crypto20.randomUUID)();
|
|
40785
41548
|
const now = args.now ?? Date.now;
|
|
40786
41549
|
const startedAt = now();
|
|
40787
41550
|
const budgetMs = finitePositive(CONNECTED_DATA_EXPORT_BUDGET_MS, 24e4);
|
|
@@ -40891,11 +41654,11 @@ ${lines.length ? `${lines.join("\n")}
|
|
|
40891
41654
|
untrustedContent: true
|
|
40892
41655
|
};
|
|
40893
41656
|
}
|
|
40894
|
-
var
|
|
41657
|
+
var import_node_crypto20, CONNECTED_DATA_INLINE_BUDGET_BYTES, CONNECTED_DATA_MAX_EXPORT_BYTES, CONNECTED_DATA_EXPORT_BUDGET_MS, CONNECTED_DATA_PAGE_START_HEADROOM_MS, CONNECTED_DATA_DATASETS, ConnectedDataExportValidationError;
|
|
40895
41658
|
var init_connected_data_export = __esm({
|
|
40896
41659
|
"src/api/connected-data-export.ts"() {
|
|
40897
41660
|
"use strict";
|
|
40898
|
-
|
|
41661
|
+
import_node_crypto20 = require("crypto");
|
|
40899
41662
|
CONNECTED_DATA_INLINE_BUDGET_BYTES = Number(
|
|
40900
41663
|
process.env.MCP_SCRAPER_CONNECTED_DATA_INLINE_BUDGET_BYTES ?? 5e4
|
|
40901
41664
|
);
|
|
@@ -40933,12 +41696,6 @@ var init_connected_data_export = __esm({
|
|
|
40933
41696
|
});
|
|
40934
41697
|
|
|
40935
41698
|
// src/api/credit-operations.ts
|
|
40936
|
-
async function grantSignupCredit(userId) {
|
|
40937
|
-
if (!FREE_SIGNUP_MC || FREE_SIGNUP_MC <= 0) return;
|
|
40938
|
-
const already = await ledgerExistsForOperation(userId, LedgerOperation.SIGNUP_GRANT);
|
|
40939
|
-
if (already) return;
|
|
40940
|
-
await creditMc(userId, FREE_SIGNUP_MC, LedgerOperation.SIGNUP_GRANT, "Welcome credits");
|
|
40941
|
-
}
|
|
40942
41699
|
async function applyMonthlyFreeRefresh(user) {
|
|
40943
41700
|
return user;
|
|
40944
41701
|
}
|
|
@@ -41705,7 +42462,7 @@ function canonicalJson2(value) {
|
|
|
41705
42462
|
return JSON.stringify(value);
|
|
41706
42463
|
}
|
|
41707
42464
|
function projectedToolSchemaHash(tool) {
|
|
41708
|
-
return (0,
|
|
42465
|
+
return (0, import_node_crypto21.createHash)("sha256").update(canonicalJson2(tool)).digest("hex");
|
|
41709
42466
|
}
|
|
41710
42467
|
async function describeNangoTool(identity, connectionId, tool, fresh) {
|
|
41711
42468
|
const body = await controlRequest("/api/internal/nango/connections/actions/describe", {
|
|
@@ -41835,11 +42592,11 @@ async function callScheduleConnectionExportPage(identity, input) {
|
|
|
41835
42592
|
untrustedContent: true
|
|
41836
42593
|
};
|
|
41837
42594
|
}
|
|
41838
|
-
var
|
|
42595
|
+
var import_node_crypto21, DEFAULT_NANGO_CONTROL_URL, DISABLED_NANGO_TOOLS, CONNECTION_SYNC_REQUIRED_TOOLS, CONNECTION_SYNC_OPTIONAL_TOOLS, NangoControlError, ScheduleConnectionValidationError, SAFE_CONTROL_ERROR_CODES, CONTROL_ERROR_CODE_ALIASES;
|
|
41839
42596
|
var init_nango_control = __esm({
|
|
41840
42597
|
"src/api/nango-control.ts"() {
|
|
41841
42598
|
"use strict";
|
|
41842
|
-
|
|
42599
|
+
import_node_crypto21 = require("crypto");
|
|
41843
42600
|
init_connected_data_export();
|
|
41844
42601
|
DEFAULT_NANGO_CONTROL_URL = "https://mcp-scraper-scheduler.vercel.app";
|
|
41845
42602
|
DISABLED_NANGO_TOOLS = {
|
|
@@ -42307,8 +43064,9 @@ async function processJob(job) {
|
|
|
42307
43064
|
running++;
|
|
42308
43065
|
try {
|
|
42309
43066
|
const opts = typeof job.options === "string" ? JSON.parse(job.options) : job.options;
|
|
43067
|
+
const headlessSentOut = { value: null };
|
|
42310
43068
|
const result = await runWithCostContext(
|
|
42311
|
-
{ op: opts.serpOnly ? "serp" : "paa", userId: Number(job.user_id) },
|
|
43069
|
+
{ op: opts.serpOnly ? "serp" : "paa", userId: Number(job.user_id), headlessSentOut },
|
|
42312
43070
|
() => harvest({
|
|
42313
43071
|
...opts,
|
|
42314
43072
|
kernelApiKey: browserServiceApiKey(),
|
|
@@ -42325,6 +43083,11 @@ async function processJob(job) {
|
|
|
42325
43083
|
const diff = opts.billingHoldMc - actualCost;
|
|
42326
43084
|
if (diff > 0) await creditMc(job.user_id, diff, "paa_refund", "overestimate refund");
|
|
42327
43085
|
else if (diff < 0) await debitMc(job.user_id, -diff, "paa", opts.query ?? job.query);
|
|
43086
|
+
} else if (opts.serpOnly && typeof opts.billingHoldMc === "number") {
|
|
43087
|
+
const actualCost = serpActualCostMc(headlessSentOut.value);
|
|
43088
|
+
const diff = opts.billingHoldMc - actualCost;
|
|
43089
|
+
if (diff > 0) await creditMc(job.user_id, diff, "serp_refund", "headless-mode pricing settle");
|
|
43090
|
+
else if (diff < 0) await debitMc(job.user_id, -diff, "serp", opts.query ?? job.query);
|
|
42328
43091
|
}
|
|
42329
43092
|
if (job.callback_url) {
|
|
42330
43093
|
await deliverWebhook(job.callback_url, { job_id: job.id, status: "done", result, attempts });
|
|
@@ -42678,7 +43441,6 @@ var init_server = __esm({
|
|
|
42678
43441
|
init_browser_agent_console();
|
|
42679
43442
|
init_browser_agent_db();
|
|
42680
43443
|
init_cost_context();
|
|
42681
|
-
init_cost_telemetry();
|
|
42682
43444
|
init_stripe_routes();
|
|
42683
43445
|
init_oauth_routes();
|
|
42684
43446
|
init_chat_routes();
|
|
@@ -42784,7 +43546,6 @@ var init_server = __esm({
|
|
|
42784
43546
|
const forceHeadless = c.req.header("x-kernel-headless") === "true";
|
|
42785
43547
|
return runWithCostContext({ op, probeRunId, forceHeadless }, () => next());
|
|
42786
43548
|
});
|
|
42787
|
-
void migrateCostTelemetry();
|
|
42788
43549
|
app.post("/auth/register", requireAllowedOrigin, async (c) => {
|
|
42789
43550
|
const { email, password } = await c.req.json();
|
|
42790
43551
|
const normalizedEmail = email?.trim().toLowerCase();
|
|
@@ -42815,7 +43576,6 @@ var init_server = __esm({
|
|
|
42815
43576
|
}
|
|
42816
43577
|
}
|
|
42817
43578
|
const newKey = await rotateApiKey(user.id);
|
|
42818
|
-
await grantSignupCredit(user.id);
|
|
42819
43579
|
void (async () => {
|
|
42820
43580
|
try {
|
|
42821
43581
|
const full = await getUserById(user.id);
|
|
@@ -42923,11 +43683,7 @@ var init_server = __esm({
|
|
|
42923
43683
|
return c.json(CATALOG);
|
|
42924
43684
|
});
|
|
42925
43685
|
app.get("/rates", (c) => {
|
|
42926
|
-
|
|
42927
|
-
...cost,
|
|
42928
|
-
...cost.notes ? { notes: cost.notes.replace(/ via fal\.ai/g, "") } : {}
|
|
42929
|
-
}));
|
|
42930
|
-
return c.json({ creditsPerDollarStarter: Math.round(1 / 15e-5), costs });
|
|
43686
|
+
return c.json(getPublicRatesPayload());
|
|
42931
43687
|
});
|
|
42932
43688
|
app.get("/me", async (c) => {
|
|
42933
43689
|
const token = (0, import_cookie2.getCookie)(c, "session");
|
|
@@ -43842,7 +44598,7 @@ var init_server = __esm({
|
|
|
43842
44598
|
serpOnly: body.serpOnly ?? false,
|
|
43843
44599
|
pages: Math.min(2, Math.max(1, body.pages ?? 1))
|
|
43844
44600
|
};
|
|
43845
|
-
const harvestCost = options.serpOnly ? MC_COSTS.
|
|
44601
|
+
const harvestCost = options.serpOnly ? MC_COSTS.serp_headful : paaCostForQuestionCount2(options.maxQuestions);
|
|
43846
44602
|
const { ok: harvestOk, balance_mc: harvestBal } = await debitMc(user.id, harvestCost, options.serpOnly ? LedgerOperation.SERP : LedgerOperation.PAA, options.query);
|
|
43847
44603
|
if (!harvestOk) return c.json(insufficientBalanceResponse(harvestBal, harvestCost), 402);
|
|
43848
44604
|
const jobId = await createJob(user.id, options.query, { ...options, billingHoldMc: harvestCost }, body.callback_url);
|
|
@@ -43878,7 +44634,7 @@ var init_server = __esm({
|
|
|
43878
44634
|
serpOnly: body.serpOnly ?? false,
|
|
43879
44635
|
pages: Math.min(2, Math.max(1, body.pages ?? 1))
|
|
43880
44636
|
};
|
|
43881
|
-
const syncCost = options.serpOnly ? MC_COSTS.
|
|
44637
|
+
const syncCost = options.serpOnly ? MC_COSTS.serp_headful : paaCostForQuestionCount2(options.maxQuestions);
|
|
43882
44638
|
const { ok: syncOk, balance_mc: syncBal } = await debitMc(user.id, syncCost, options.serpOnly ? LedgerOperation.SERP : LedgerOperation.PAA, options.query);
|
|
43883
44639
|
if (!syncOk) return c.json(insufficientBalanceResponse(syncBal, syncCost), 402);
|
|
43884
44640
|
const jobId = await createRunningJob(user.id, options.query, options);
|
|
@@ -43888,10 +44644,11 @@ var init_server = __esm({
|
|
|
43888
44644
|
c.req.raw.signal,
|
|
43889
44645
|
AbortSignal.timeout(syncTimeoutMs)
|
|
43890
44646
|
]);
|
|
44647
|
+
const headlessSentOut = { value: null };
|
|
43891
44648
|
try {
|
|
43892
44649
|
const harvestCtx = currentCostContext();
|
|
43893
44650
|
const result = await runWithCostContext(
|
|
43894
|
-
{ ...harvestCtx, op: options.serpOnly ? "serp" : "paa", userId: user.id },
|
|
44651
|
+
{ ...harvestCtx, op: options.serpOnly ? "serp" : "paa", userId: user.id, headlessSentOut },
|
|
43895
44652
|
() => harvest({
|
|
43896
44653
|
...options,
|
|
43897
44654
|
kernelApiKey: browserServiceApiKey(),
|
|
@@ -43910,6 +44667,11 @@ var init_server = __esm({
|
|
|
43910
44667
|
const diff = syncCost - actualCost;
|
|
43911
44668
|
if (diff > 0) await creditMc(user.id, diff, LedgerOperation.PAA_REFUND, "overestimate refund");
|
|
43912
44669
|
else if (diff < 0) await debitMc(user.id, -diff, LedgerOperation.PAA, options.query);
|
|
44670
|
+
} else {
|
|
44671
|
+
const actualCost = serpActualCostMc(headlessSentOut.value);
|
|
44672
|
+
const diff = syncCost - actualCost;
|
|
44673
|
+
if (diff > 0) await creditMc(user.id, diff, LedgerOperation.SERP_REFUND, "headless-mode pricing settle");
|
|
44674
|
+
else if (diff < 0) await debitMc(user.id, -diff, LedgerOperation.SERP, options.query);
|
|
43913
44675
|
}
|
|
43914
44676
|
return c.json({ job_id: jobId, status: "done", result: sanitizeHarvestResult(result), attempts: sanitizeAttempts(attempts) });
|
|
43915
44677
|
} catch (err) {
|
|
@@ -43985,22 +44747,13 @@ var init_server = __esm({
|
|
|
43985
44747
|
});
|
|
43986
44748
|
app.post("/admin/backfill-signup-credits", adminAuth, async (c) => {
|
|
43987
44749
|
const users = await listUsers();
|
|
43988
|
-
|
|
43989
|
-
|
|
43990
|
-
|
|
43991
|
-
|
|
43992
|
-
|
|
43993
|
-
|
|
43994
|
-
|
|
43995
|
-
if (alreadyHasGrant) {
|
|
43996
|
-
skipped++;
|
|
43997
|
-
} else {
|
|
43998
|
-
await grantSignupCredit(user.id);
|
|
43999
|
-
credited++;
|
|
44000
|
-
users_credited.push({ id: user.id, email: user.email });
|
|
44001
|
-
}
|
|
44002
|
-
}
|
|
44003
|
-
return c.json({ processed, credited, skipped, users_credited });
|
|
44750
|
+
return c.json({
|
|
44751
|
+
processed: users.length,
|
|
44752
|
+
credited: 0,
|
|
44753
|
+
skipped: users.length,
|
|
44754
|
+
users_credited: [],
|
|
44755
|
+
retired: true
|
|
44756
|
+
});
|
|
44004
44757
|
});
|
|
44005
44758
|
app.post("/extract-url", auth2, async (c) => {
|
|
44006
44759
|
const raw = await c.req.json().catch(() => ({}));
|