mcp-scraper 0.21.3 → 0.21.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (56) hide show
  1. package/README.md +1 -1
  2. package/dist/bin/api-server.cjs +728 -181
  3. package/dist/bin/api-server.cjs.map +1 -1
  4. package/dist/bin/api-server.js +3 -3
  5. package/dist/bin/mcp-scraper-cli.cjs +1 -1
  6. package/dist/bin/mcp-scraper-cli.cjs.map +1 -1
  7. package/dist/bin/mcp-scraper-cli.js +1 -1
  8. package/dist/bin/mcp-scraper-install.cjs +1 -1
  9. package/dist/bin/mcp-scraper-install.cjs.map +1 -1
  10. package/dist/bin/mcp-scraper-install.js +1 -1
  11. package/dist/bin/mcp-stdio-server.cjs +97 -32
  12. package/dist/bin/mcp-stdio-server.cjs.map +1 -1
  13. package/dist/bin/mcp-stdio-server.js +5 -5
  14. package/dist/bin/paa-harvest.cjs +21 -0
  15. package/dist/bin/paa-harvest.cjs.map +1 -1
  16. package/dist/bin/paa-harvest.js +3 -3
  17. package/dist/{chunk-JJ5RMEGX.js → chunk-E5J4HJBO.js} +2 -2
  18. package/dist/{chunk-H7APH5UG.js → chunk-ICT7DDHL.js} +366 -50
  19. package/dist/chunk-ICT7DDHL.js.map +1 -0
  20. package/dist/{chunk-OBNKOYOE.js → chunk-JHI373VB.js} +7 -5
  21. package/dist/chunk-JHI373VB.js.map +1 -0
  22. package/dist/{chunk-5IU5ZFVO.js → chunk-MTSBI7ZH.js} +23 -6
  23. package/dist/chunk-MTSBI7ZH.js.map +1 -0
  24. package/dist/{chunk-7LO6KLKD.js → chunk-N4JHXPW6.js} +89 -27
  25. package/dist/chunk-N4JHXPW6.js.map +1 -0
  26. package/dist/{chunk-XDRHBGJR.js → chunk-PNBYYAEK.js} +3 -3
  27. package/dist/{chunk-2P5VFIJM.js → chunk-SDHPQTFM.js} +2 -2
  28. package/dist/chunk-WQI7JIIN.js +7 -0
  29. package/dist/chunk-WQI7JIIN.js.map +1 -0
  30. package/dist/{db-Z35S224E.js → db-WRILVOFY.js} +4 -2
  31. package/dist/{extract-bundle-4RO3YU4Q.js → extract-bundle-NFFVMUMT.js} +2 -2
  32. package/dist/index.cjs +21 -0
  33. package/dist/index.cjs.map +1 -1
  34. package/dist/index.js +3 -3
  35. package/dist/{server-PBBRSLBP.js → server-OLOOFN5W.js} +246 -100
  36. package/dist/server-OLOOFN5W.js.map +1 -0
  37. package/dist/{site-extract-repository-FDH6ZXKT.js → site-extract-repository-2QQSL2QC.js} +4 -4
  38. package/dist/{worker-BXJQ7J4N.js → worker-RV2TYXE3.js} +9 -8
  39. package/dist/worker-RV2TYXE3.js.map +1 -0
  40. package/docs/mcp-tool-manifest.generated.json +90 -11
  41. package/docs/specs/oauth-mcp-spec.md +2 -2
  42. package/package.json +1 -1
  43. package/dist/chunk-5IU5ZFVO.js.map +0 -1
  44. package/dist/chunk-7LO6KLKD.js.map +0 -1
  45. package/dist/chunk-H7APH5UG.js.map +0 -1
  46. package/dist/chunk-OBNKOYOE.js.map +0 -1
  47. package/dist/chunk-VYUWWIVR.js +0 -7
  48. package/dist/chunk-VYUWWIVR.js.map +0 -1
  49. package/dist/server-PBBRSLBP.js.map +0 -1
  50. package/dist/worker-BXJQ7J4N.js.map +0 -1
  51. /package/dist/{chunk-JJ5RMEGX.js.map → chunk-E5J4HJBO.js.map} +0 -0
  52. /package/dist/{chunk-XDRHBGJR.js.map → chunk-PNBYYAEK.js.map} +0 -0
  53. /package/dist/{chunk-2P5VFIJM.js.map → chunk-SDHPQTFM.js.map} +0 -0
  54. /package/dist/{db-Z35S224E.js.map → db-WRILVOFY.js.map} +0 -0
  55. /package/dist/{extract-bundle-4RO3YU4Q.js.map → extract-bundle-NFFVMUMT.js.map} +0 -0
  56. /package/dist/{site-extract-repository-FDH6ZXKT.js.map → site-extract-repository-2QQSL2QC.js.map} +0 -0
@@ -3671,12 +3671,27 @@ __export(db_exports, {
3671
3671
  setStripeCustomerId: () => setStripeCustomerId,
3672
3672
  setSubscriptionTier: () => setSubscriptionTier,
3673
3673
  setUserMemoryCreds: () => setUserMemoryCreds,
3674
+ settleDebitMcIdempotent: () => settleDebitMcIdempotent,
3674
3675
  startHarvestAttempt: () => startHarvestAttempt,
3675
3676
  stripeEventAlreadyProcessed: () => stripeEventAlreadyProcessed,
3676
3677
  updateKpoJobState: () => updateKpoJobState,
3677
3678
  upsertPageSnapshot: () => upsertPageSnapshot,
3678
3679
  verifyPassword: () => verifyPassword
3679
3680
  });
3681
+ function sqlStringList(values) {
3682
+ return values.map((value) => `'${value.replaceAll("'", "''")}'`).join(", ");
3683
+ }
3684
+ async function hasCurrentCoreSchema(db) {
3685
+ const checks = [
3686
+ `(SELECT COUNT(*) FROM sqlite_master WHERE type = 'table' AND name IN (${sqlStringList(CORE_SCHEMA_TABLES)})) = ${CORE_SCHEMA_TABLES.length}`,
3687
+ `(SELECT COUNT(*) FROM sqlite_master WHERE type = 'index' AND name IN (${sqlStringList(CORE_SCHEMA_INDEXES)})) = ${CORE_SCHEMA_INDEXES.length}`,
3688
+ ...Object.entries(CORE_SCHEMA_COLUMNS).map(
3689
+ ([table, columns]) => `(SELECT COUNT(*) FROM pragma_table_info('${table}') WHERE name IN (${sqlStringList(columns)})) = ${columns.length}`
3690
+ )
3691
+ ];
3692
+ const result = await db.execute(`SELECT ${checks.join(" AND ")} AS ready`);
3693
+ return Number(result.rows[0]?.ready ?? 0) === 1;
3694
+ }
3680
3695
  function getDb() {
3681
3696
  if (!_db) _db = (0, import_http.createClient)({ url: DB_URL, authToken: DB_TOKEN });
3682
3697
  return _db;
@@ -3686,6 +3701,31 @@ function setDbForTesting(client2) {
3686
3701
  }
3687
3702
  async function migrate() {
3688
3703
  const db = getDb();
3704
+ let applied;
3705
+ try {
3706
+ applied = await db.execute({
3707
+ sql: "SELECT 1 AS applied FROM schema_migrations WHERE version = ? LIMIT 1",
3708
+ args: [CORE_SCHEMA_VERSION]
3709
+ });
3710
+ } catch (error) {
3711
+ const message = error instanceof Error ? error.message.toLowerCase() : String(error).toLowerCase();
3712
+ if (!message.includes("no such table") || !message.includes("schema_migrations")) throw error;
3713
+ await db.execute(`
3714
+ CREATE TABLE IF NOT EXISTS schema_migrations (
3715
+ version TEXT PRIMARY KEY,
3716
+ applied_at TEXT NOT NULL DEFAULT (datetime('now'))
3717
+ )
3718
+ `);
3719
+ applied = { rows: [] };
3720
+ }
3721
+ if (applied.rows.length > 0) return;
3722
+ if (await hasCurrentCoreSchema(db)) {
3723
+ await db.execute({
3724
+ sql: "INSERT OR IGNORE INTO schema_migrations (version) VALUES (?)",
3725
+ args: [CORE_SCHEMA_VERSION]
3726
+ });
3727
+ return;
3728
+ }
3689
3729
  await db.execute(`
3690
3730
  CREATE TABLE IF NOT EXISTS users (
3691
3731
  id INTEGER PRIMARY KEY AUTOINCREMENT,
@@ -4179,6 +4219,13 @@ async function migrate() {
4179
4219
  )
4180
4220
  `);
4181
4221
  await db.execute(`CREATE INDEX IF NOT EXISTS page_snapshots_user_checked_at ON page_snapshots(user_id, checked_at DESC)`);
4222
+ if (!await hasCurrentCoreSchema(db)) {
4223
+ throw new Error("core schema migration finished without all required tables, indexes, and columns");
4224
+ }
4225
+ await db.execute({
4226
+ sql: "INSERT OR IGNORE INTO schema_migrations (version) VALUES (?)",
4227
+ args: [CORE_SCHEMA_VERSION]
4228
+ });
4182
4229
  }
4183
4230
  async function registerClient(clientId, redirectUris, name) {
4184
4231
  await getDb().execute({
@@ -4641,6 +4688,17 @@ async function upsertPageSnapshot(userId, url, snapshot) {
4641
4688
  args: [userId, url, snapshot.contentHash, snapshot.content, snapshot.title, snapshot.contentBytes, snapshot.truncated ? 1 : 0]
4642
4689
  });
4643
4690
  }
4691
+ function serializeRequestEventResult(result) {
4692
+ if (result == null) return null;
4693
+ const serialized = JSON.stringify(result);
4694
+ const bytes = Buffer.byteLength(serialized, "utf8");
4695
+ if (bytes <= REQUEST_EVENT_RESULT_MAX_BYTES) return serialized;
4696
+ return JSON.stringify({
4697
+ truncated: true,
4698
+ originalBytes: bytes,
4699
+ sha256: (0, import_node_crypto.createHash)("sha256").update(serialized).digest("hex")
4700
+ });
4701
+ }
4644
4702
  function rowToRequestEvent(row) {
4645
4703
  const rawResult = row.result != null ? String(row.result) : null;
4646
4704
  return {
@@ -4671,7 +4729,7 @@ async function logRequestEvent(input) {
4671
4729
  input.query,
4672
4730
  input.location ?? null,
4673
4731
  input.resultCount ?? null,
4674
- input.result != null ? JSON.stringify(input.result) : null,
4732
+ serializeRequestEventResult(input.result),
4675
4733
  input.error ?? null
4676
4734
  ]
4677
4735
  });
@@ -4680,7 +4738,7 @@ async function logRequestEvent(input) {
4680
4738
  async function listRequestEvents(userId, limit = 100) {
4681
4739
  const res = await getDb().execute({
4682
4740
  sql: `
4683
- SELECT id, user_id, source, status, query, location, result_count, result, error, created_at
4741
+ SELECT id, user_id, source, status, query, location, result_count, NULL AS result, error, created_at
4684
4742
  FROM request_events
4685
4743
  WHERE user_id = ?
4686
4744
  ORDER BY created_at DESC
@@ -5308,58 +5366,75 @@ async function debitMcIdempotent(userId, mc, operation, description, idempotency
5308
5366
  if (!idempotencyKey || idempotencyKey.length > 500) throw new Error("idempotent debit key is required and must be bounded");
5309
5367
  const db = getDb();
5310
5368
  await ensureMigrated(userId);
5311
- const results = await db.batch([
5312
- {
5313
- sql: "UPDATE billing_debits SET amount_mc = ?, description = ? WHERE idempotency_key = ? AND user_id = ? AND operation = ? AND status = 'claimed'",
5314
- args: [mc, description, idempotencyKey, userId, operation]
5315
- },
5316
- {
5317
- sql: "INSERT OR IGNORE INTO billing_debits (idempotency_key, user_id, amount_mc, operation, description, status) VALUES (?, ?, ?, ?, ?, 'claimed')",
5318
- args: [idempotencyKey, userId, mc, operation, description]
5319
- },
5320
- {
5321
- sql: "UPDATE users SET balance_mc = balance_mc - ? WHERE id = ? AND balance_mc >= ? AND EXISTS (SELECT 1 FROM billing_debits WHERE idempotency_key = ? AND user_id = ? AND amount_mc = ? AND operation = ? AND status = 'claimed')",
5322
- args: [mc, userId, mc, idempotencyKey, userId, mc, operation]
5323
- },
5324
- {
5325
- sql: "UPDATE billing_debits SET status = 'applying' WHERE idempotency_key = ? AND user_id = ? AND amount_mc = ? AND operation = ? AND status = 'claimed' AND changes() = 1",
5326
- args: [idempotencyKey, userId, mc, operation]
5327
- },
5328
- {
5329
- sql: `WITH ordered AS MATERIALIZED (
5330
- SELECT id,
5331
- remaining_mc,
5332
- COALESCE(SUM(remaining_mc) OVER (
5333
- ORDER BY expires_at ASC, granted_at ASC, id ASC
5334
- ROWS BETWEEN UNBOUNDED PRECEDING AND 1 PRECEDING
5335
- ), 0) AS prior_mc
5336
- FROM credit_lots
5337
- WHERE user_id = ? AND remaining_mc > 0 AND expires_at > datetime('now')
5338
- ), allocation AS (
5339
- SELECT id, MIN(remaining_mc, MAX(0, ? - prior_mc)) AS take_mc
5340
- FROM ordered
5341
- )
5342
- UPDATE credit_lots
5343
- SET remaining_mc = remaining_mc - COALESCE((SELECT take_mc FROM allocation WHERE allocation.id = credit_lots.id), 0)
5344
- WHERE id IN (SELECT id FROM allocation WHERE take_mc > 0)
5345
- AND EXISTS (SELECT 1 FROM billing_debits WHERE idempotency_key = ? AND status = 'applying')`,
5346
- args: [userId, mc, idempotencyKey]
5347
- },
5348
- {
5349
- sql: "INSERT INTO ledger (user_id, amount_mc, operation, description) SELECT user_id, -amount_mc, operation, description FROM billing_debits WHERE idempotency_key = ? AND status = 'applying'",
5350
- args: [idempotencyKey]
5351
- },
5352
- {
5353
- sql: "UPDATE billing_debits SET status = 'applied', applied_at = datetime('now') WHERE idempotency_key = ? AND status = 'applying'",
5354
- args: [idempotencyKey]
5369
+ let results;
5370
+ try {
5371
+ results = await db.batch([
5372
+ {
5373
+ sql: "UPDATE billing_debits SET amount_mc = ? WHERE idempotency_key = ? AND user_id = ? AND operation = ? AND description = ? AND status = 'claimed'",
5374
+ args: [mc, idempotencyKey, userId, operation, description]
5375
+ },
5376
+ {
5377
+ sql: "INSERT OR IGNORE INTO billing_debits (idempotency_key, user_id, amount_mc, operation, description, status) VALUES (?, ?, ?, ?, ?, 'claimed')",
5378
+ args: [idempotencyKey, userId, mc, operation, description]
5379
+ },
5380
+ {
5381
+ sql: "UPDATE users SET balance_mc = balance_mc - ? WHERE id = ? AND balance_mc >= ? AND EXISTS (SELECT 1 FROM billing_debits WHERE idempotency_key = ? AND user_id = ? AND amount_mc = ? AND operation = ? AND description = ? AND status = 'claimed')",
5382
+ args: [mc, userId, mc, idempotencyKey, userId, mc, operation, description]
5383
+ },
5384
+ {
5385
+ sql: "UPDATE billing_debits SET status = 'applying' WHERE idempotency_key = ? AND user_id = ? AND amount_mc = ? AND operation = ? AND description = ? AND status = 'claimed' AND changes() = 1",
5386
+ args: [idempotencyKey, userId, mc, operation, description]
5387
+ },
5388
+ {
5389
+ sql: `WITH ordered AS MATERIALIZED (
5390
+ SELECT id,
5391
+ remaining_mc,
5392
+ COALESCE(SUM(remaining_mc) OVER (
5393
+ ORDER BY expires_at ASC, granted_at ASC, id ASC
5394
+ ROWS BETWEEN UNBOUNDED PRECEDING AND 1 PRECEDING
5395
+ ), 0) AS prior_mc
5396
+ FROM credit_lots
5397
+ WHERE user_id = ? AND remaining_mc > 0 AND expires_at > datetime('now')
5398
+ ), allocation AS (
5399
+ SELECT id, MIN(remaining_mc, MAX(0, ? - prior_mc)) AS take_mc
5400
+ FROM ordered
5401
+ )
5402
+ UPDATE credit_lots
5403
+ SET remaining_mc = remaining_mc - COALESCE((SELECT take_mc FROM allocation WHERE allocation.id = credit_lots.id), 0)
5404
+ WHERE id IN (SELECT id FROM allocation WHERE take_mc > 0)
5405
+ AND EXISTS (SELECT 1 FROM billing_debits WHERE idempotency_key = ? AND status = 'applying')`,
5406
+ args: [userId, mc, idempotencyKey]
5407
+ },
5408
+ {
5409
+ sql: "INSERT INTO ledger (user_id, amount_mc, operation, description) SELECT user_id, -amount_mc, operation, description FROM billing_debits WHERE idempotency_key = ? AND status = 'applying'",
5410
+ args: [idempotencyKey]
5411
+ },
5412
+ {
5413
+ sql: "UPDATE billing_debits SET status = 'applied', applied_at = datetime('now') WHERE idempotency_key = ? AND status = 'applying'",
5414
+ args: [idempotencyKey]
5415
+ }
5416
+ ], "write");
5417
+ } catch (error) {
5418
+ try {
5419
+ const recovered = await db.execute({
5420
+ sql: "SELECT user_id, amount_mc, operation, description, status FROM billing_debits WHERE idempotency_key = ? AND status = 'applied' LIMIT 1",
5421
+ args: [idempotencyKey]
5422
+ });
5423
+ const row2 = recovered.rows[0];
5424
+ if (row2 && Number(row2.user_id) === userId && Number(row2.amount_mc) === mc && String(row2.operation) === operation && String(row2.description) === description) {
5425
+ const balance2 = await db.execute({ sql: "SELECT balance_mc FROM users WHERE id = ?", args: [userId] });
5426
+ return { ok: true, balance_mc: Number(balance2.rows[0]?.balance_mc ?? 0), duplicate: true };
5427
+ }
5428
+ } catch {
5355
5429
  }
5356
- ], "write");
5430
+ throw error;
5431
+ }
5357
5432
  const claim = await db.execute({
5358
- sql: "SELECT user_id, amount_mc, operation, status FROM billing_debits WHERE idempotency_key = ? LIMIT 1",
5433
+ sql: "SELECT user_id, amount_mc, operation, description, status FROM billing_debits WHERE idempotency_key = ? LIMIT 1",
5359
5434
  args: [idempotencyKey]
5360
5435
  });
5361
5436
  const row = claim.rows[0];
5362
- if (!row || Number(row.user_id) !== userId || Number(row.amount_mc) !== mc || String(row.operation) !== operation) {
5437
+ if (!row || Number(row.user_id) !== userId || Number(row.amount_mc) !== mc || String(row.operation) !== operation || String(row.description) !== description) {
5363
5438
  throw new Error("idempotent debit key conflicts with another debit");
5364
5439
  }
5365
5440
  const balance = await db.execute({ sql: "SELECT balance_mc FROM users WHERE id = ?", args: [userId] });
@@ -5369,6 +5444,158 @@ async function debitMcIdempotent(userId, mc, operation, description, idempotency
5369
5444
  duplicate: results[2]?.rowsAffected === 0 && String(row.status) === "applied"
5370
5445
  };
5371
5446
  }
5447
+ async function settleDebitMcIdempotent(userId, debitIdempotencyKey, finalAmountMc, refundOperation, description, sourceSurface) {
5448
+ if (!Number.isSafeInteger(finalAmountMc) || finalAmountMc < 0) {
5449
+ throw new Error("idempotent debit settlement amount must be a non-negative integer");
5450
+ }
5451
+ if (!debitIdempotencyKey || debitIdempotencyKey.length > 500) {
5452
+ throw new Error("idempotent debit settlement key is required and must be bounded");
5453
+ }
5454
+ if (!refundOperation || !sourceSurface) {
5455
+ throw new Error("idempotent debit settlement operation and source are required");
5456
+ }
5457
+ const db = getDb();
5458
+ await ensureMigrated(userId);
5459
+ const settlementKey = `debit-settlement:${(0, import_node_crypto.createHash)("sha256").update(debitIdempotencyKey).digest("hex")}`;
5460
+ const billingClass = "wallet_hold_settlement";
5461
+ const eventId = (0, import_node_crypto.randomUUID)();
5462
+ const metadata = JSON.stringify({ debitIdempotencyKey, refundOperation, description });
5463
+ const readReceipt = async () => {
5464
+ const receipt2 = await db.execute({
5465
+ sql: `SELECT e.user_id,
5466
+ e.status,
5467
+ e.amount_mc AS final_amount_mc,
5468
+ e.billing_class,
5469
+ e.source_surface,
5470
+ d.amount_mc AS held_amount_mc
5471
+ FROM billing_events e
5472
+ JOIN billing_debits d ON d.idempotency_key = ?
5473
+ WHERE e.idempotency_key = ?
5474
+ AND d.user_id = e.user_id
5475
+ AND d.status = 'applied'
5476
+ LIMIT 1`,
5477
+ args: [debitIdempotencyKey, settlementKey]
5478
+ });
5479
+ const row = receipt2.rows[0];
5480
+ if (!row) return null;
5481
+ if (Number(row.user_id) !== userId || String(row.billing_class) !== billingClass || String(row.source_surface) !== sourceSurface) {
5482
+ throw new Error("idempotent debit settlement key conflicts with another settlement");
5483
+ }
5484
+ if (String(row.status) !== "settled") return null;
5485
+ const heldAmountMc = Number(row.held_amount_mc);
5486
+ const durableFinalAmountMc = Number(row.final_amount_mc);
5487
+ if (!Number.isSafeInteger(heldAmountMc) || !Number.isSafeInteger(durableFinalAmountMc) || durableFinalAmountMc < 0 || durableFinalAmountMc > heldAmountMc) {
5488
+ throw new Error("idempotent debit settlement receipt is invalid");
5489
+ }
5490
+ const balance = await db.execute({ sql: "SELECT balance_mc FROM users WHERE id = ?", args: [userId] });
5491
+ return {
5492
+ balance_mc: Number(balance.rows[0]?.balance_mc ?? 0),
5493
+ final_amount_mc: durableFinalAmountMc,
5494
+ refund_mc: heldAmountMc - durableFinalAmountMc,
5495
+ duplicate: true
5496
+ };
5497
+ };
5498
+ let results;
5499
+ try {
5500
+ results = await db.batch([
5501
+ {
5502
+ sql: `INSERT OR IGNORE INTO billing_events
5503
+ (id, user_id, idempotency_key, billing_class, source_surface, status, amount_mc, metadata)
5504
+ VALUES (?, ?, ?, ?, ?, 'settling', ?, ?)`,
5505
+ args: [eventId, userId, settlementKey, billingClass, sourceSurface, finalAmountMc, metadata]
5506
+ },
5507
+ {
5508
+ sql: `UPDATE users
5509
+ SET balance_mc = balance_mc + (
5510
+ SELECT d.amount_mc - e.amount_mc
5511
+ FROM billing_events e
5512
+ JOIN billing_debits d ON d.idempotency_key = ?
5513
+ WHERE e.idempotency_key = ?
5514
+ )
5515
+ WHERE id = ?
5516
+ AND EXISTS (
5517
+ SELECT 1
5518
+ FROM billing_events e
5519
+ JOIN billing_debits d ON d.idempotency_key = ?
5520
+ WHERE e.idempotency_key = ?
5521
+ AND e.user_id = ?
5522
+ AND e.billing_class = ?
5523
+ AND e.source_surface = ?
5524
+ AND e.status = 'settling'
5525
+ AND d.user_id = e.user_id
5526
+ AND d.status = 'applied'
5527
+ AND e.amount_mc BETWEEN 0 AND d.amount_mc
5528
+ )`,
5529
+ args: [
5530
+ debitIdempotencyKey,
5531
+ settlementKey,
5532
+ userId,
5533
+ debitIdempotencyKey,
5534
+ settlementKey,
5535
+ userId,
5536
+ billingClass,
5537
+ sourceSurface
5538
+ ]
5539
+ },
5540
+ {
5541
+ sql: `UPDATE billing_events
5542
+ SET status = 'applying'
5543
+ WHERE idempotency_key = ?
5544
+ AND user_id = ?
5545
+ AND billing_class = ?
5546
+ AND source_surface = ?
5547
+ AND status = 'settling'
5548
+ AND changes() = 1`,
5549
+ args: [settlementKey, userId, billingClass, sourceSurface]
5550
+ },
5551
+ {
5552
+ sql: `INSERT INTO credit_lots
5553
+ (user_id, amount_mc, remaining_mc, source, expires_at)
5554
+ SELECT e.user_id,
5555
+ d.amount_mc - e.amount_mc,
5556
+ d.amount_mc - e.amount_mc,
5557
+ ?,
5558
+ datetime('now', ?)
5559
+ FROM billing_events e
5560
+ JOIN billing_debits d ON d.idempotency_key = ?
5561
+ WHERE e.idempotency_key = ?
5562
+ AND e.status = 'applying'
5563
+ AND d.amount_mc > e.amount_mc`,
5564
+ args: [refundOperation, CREDIT_LOT_TTL, debitIdempotencyKey, settlementKey]
5565
+ },
5566
+ {
5567
+ sql: `INSERT INTO ledger (user_id, amount_mc, operation, description)
5568
+ SELECT e.user_id,
5569
+ d.amount_mc - e.amount_mc,
5570
+ ?,
5571
+ ?
5572
+ FROM billing_events e
5573
+ JOIN billing_debits d ON d.idempotency_key = ?
5574
+ WHERE e.idempotency_key = ?
5575
+ AND e.status = 'applying'
5576
+ AND d.amount_mc > e.amount_mc`,
5577
+ args: [refundOperation, description, debitIdempotencyKey, settlementKey]
5578
+ },
5579
+ {
5580
+ sql: `UPDATE billing_events
5581
+ SET status = 'settled', settled_at = datetime('now')
5582
+ WHERE idempotency_key = ?
5583
+ AND status = 'applying'`,
5584
+ args: [settlementKey]
5585
+ }
5586
+ ], "write");
5587
+ } catch (error) {
5588
+ try {
5589
+ const recovered = await readReceipt();
5590
+ if (recovered) return recovered;
5591
+ } catch {
5592
+ }
5593
+ throw error;
5594
+ }
5595
+ const receipt = await readReceipt();
5596
+ if (!receipt) throw new Error("idempotent debit settlement did not reach a durable settled state");
5597
+ return { ...receipt, duplicate: results[0]?.rowsAffected === 0 };
5598
+ }
5372
5599
  async function getLedger(userId, limit = 50) {
5373
5600
  const res = await getDb().execute({
5374
5601
  sql: "SELECT * FROM ledger WHERE user_id = ? ORDER BY created_at DESC LIMIT ?",
@@ -5417,7 +5644,7 @@ async function reconcileBalanceMc(userId, knownBalanceMc) {
5417
5644
  }
5418
5645
  return balanceMc;
5419
5646
  }
5420
- var import_http, import_node_crypto, import_zod, DB_URL, DB_TOKEN, _db, _rateLimitSchemaReady, CREDIT_LOT_TTL, SiteAuditJobRowSchema, SiteAuditPhaseLogRowSchema;
5647
+ var import_http, import_node_crypto, import_zod, DB_URL, DB_TOKEN, _db, _rateLimitSchemaReady, CORE_SCHEMA_VERSION, CORE_SCHEMA_TABLES, CORE_SCHEMA_INDEXES, CORE_SCHEMA_COLUMNS, REQUEST_EVENT_RESULT_MAX_BYTES, CREDIT_LOT_TTL, SiteAuditJobRowSchema, SiteAuditPhaseLogRowSchema;
5421
5648
  var init_db = __esm({
5422
5649
  "src/api/db.ts"() {
5423
5650
  "use strict";
@@ -5428,6 +5655,95 @@ var init_db = __esm({
5428
5655
  DB_TOKEN = process.env.TURSO_AUTH_TOKEN;
5429
5656
  _db = null;
5430
5657
  _rateLimitSchemaReady = false;
5658
+ CORE_SCHEMA_VERSION = "2026-07-14.1";
5659
+ CORE_SCHEMA_TABLES = [
5660
+ "billing_authorizations",
5661
+ "billing_debits",
5662
+ "billing_events",
5663
+ "concurrency_locks",
5664
+ "credit_lots",
5665
+ "free_credit_refreshes",
5666
+ "harvest_attempts",
5667
+ "jobs",
5668
+ "kpo_jobs",
5669
+ "kpo_phase_log",
5670
+ "ledger",
5671
+ "oauth_clients",
5672
+ "oauth_codes",
5673
+ "oauth_tokens",
5674
+ "page_snapshots",
5675
+ "password_reset_tokens",
5676
+ "rate_limits",
5677
+ "request_events",
5678
+ "site_audit_jobs",
5679
+ "site_audit_phase_log",
5680
+ "site_extract_jobs",
5681
+ "site_extract_pages",
5682
+ "stripe_events",
5683
+ "users",
5684
+ "workflow_artifacts",
5685
+ "workflow_runs",
5686
+ "workflow_schedules",
5687
+ "workflow_webhook_deliveries"
5688
+ ];
5689
+ CORE_SCHEMA_INDEXES = [
5690
+ "billing_authorizations_user_status",
5691
+ "billing_debits_user_status",
5692
+ "billing_events_authorization",
5693
+ "billing_events_user_created_at",
5694
+ "concurrency_locks_operation",
5695
+ "concurrency_locks_user_status",
5696
+ "credit_lots_user_active",
5697
+ "harvest_attempts_job_id",
5698
+ "harvest_attempts_user_created_at",
5699
+ "jobs_status",
5700
+ "jobs_user_id",
5701
+ "kpo_jobs_status",
5702
+ "kpo_jobs_user_id",
5703
+ "kpo_phase_log_job_id",
5704
+ "ledger_stripe_pi_unique",
5705
+ "ledger_user_id",
5706
+ "oauth_tokens_identity",
5707
+ "page_snapshots_user_checked_at",
5708
+ "prt_user_id",
5709
+ "request_events_user_created_at",
5710
+ "site_audit_jobs_status",
5711
+ "site_audit_jobs_user_id",
5712
+ "site_audit_phase_log_job_id",
5713
+ "site_extract_jobs_status",
5714
+ "site_extract_jobs_user_id",
5715
+ "site_extract_pages_job_id",
5716
+ "workflow_artifacts_run",
5717
+ "workflow_runs_idempotency",
5718
+ "workflow_runs_schedule",
5719
+ "workflow_runs_user_created",
5720
+ "workflow_schedules_due",
5721
+ "workflow_schedules_user_status",
5722
+ "workflow_webhook_deliveries_run"
5723
+ ];
5724
+ CORE_SCHEMA_COLUMNS = {
5725
+ harvest_attempts: ["debug_json"],
5726
+ oauth_tokens: ["replaced_by", "rotated_at"],
5727
+ users: [
5728
+ "balance_mc",
5729
+ "concurrency_stripe_sub_id",
5730
+ "extra_concurrency_slots",
5731
+ "key_active",
5732
+ "memory_identity",
5733
+ "memory_key",
5734
+ "memory_plan",
5735
+ "memory_provisioned",
5736
+ "memory_subscription_id",
5737
+ "password_hash",
5738
+ "stripe_customer_id",
5739
+ "subscription_concurrency",
5740
+ "subscription_id",
5741
+ "subscription_tier"
5742
+ ],
5743
+ workflow_artifacts: ["content"],
5744
+ workflow_runs: ["step_index", "step_log_json", "step_state_json", "total_steps", "webhook_url"]
5745
+ };
5746
+ REQUEST_EVENT_RESULT_MAX_BYTES = 32 * 1024;
5431
5747
  CREDIT_LOT_TTL = "+3 months";
5432
5748
  SiteAuditJobRowSchema = import_zod.z.object({
5433
5749
  id: import_zod.z.string(),
@@ -5499,7 +5815,26 @@ var init_cost_rates = __esm({
5499
5815
  // src/api/cost-telemetry.ts
5500
5816
  async function migrateCostTelemetry() {
5501
5817
  if (_ready) return;
5818
+ if (_migrationPromise) return _migrationPromise;
5819
+ _migrationPromise = runCostTelemetryMigration().finally(() => {
5820
+ _migrationPromise = null;
5821
+ });
5822
+ return _migrationPromise;
5823
+ }
5824
+ async function runCostTelemetryMigration() {
5502
5825
  const db = getDb();
5826
+ const existing = await db.execute(`
5827
+ SELECT
5828
+ (SELECT COUNT(*) FROM sqlite_master WHERE type = 'table' AND name IN ('kernel_session_log', 'vendor_usage_log', 'cost_probe_runs')) = 3
5829
+ AND (SELECT COUNT(*) FROM pragma_table_info('kernel_session_log') WHERE name IN ('proxy_source', 'proxy_type', 'method')) = 3
5830
+ AND (SELECT COUNT(*) FROM pragma_table_info('vendor_usage_log') WHERE name = 'method') = 1
5831
+ AND (SELECT COUNT(*) FROM pragma_table_info('cost_probe_runs') WHERE name IN ('units', 'unit_type', 'mode')) = 3
5832
+ AS ready
5833
+ `);
5834
+ if (Number(existing.rows[0]?.ready ?? 0) === 1) {
5835
+ _ready = true;
5836
+ return;
5837
+ }
5503
5838
  await db.execute(`
5504
5839
  CREATE TABLE IF NOT EXISTS kernel_session_log (
5505
5840
  id TEXT PRIMARY KEY,
@@ -5663,7 +5998,7 @@ function boolToInt(v) {
5663
5998
  if (v === null || v === void 0) return null;
5664
5999
  return v ? 1 : 0;
5665
6000
  }
5666
- var import_node_crypto2, _ready;
6001
+ var import_node_crypto2, _ready, _migrationPromise;
5667
6002
  var init_cost_telemetry = __esm({
5668
6003
  "src/api/cost-telemetry.ts"() {
5669
6004
  "use strict";
@@ -5672,6 +6007,7 @@ var init_cost_telemetry = __esm({
5672
6007
  init_cost_context();
5673
6008
  init_cost_rates();
5674
6009
  _ready = false;
6010
+ _migrationPromise = null;
5675
6011
  }
5676
6012
  });
5677
6013
 
@@ -8563,6 +8899,9 @@ var init_schemas = __esm({
8563
8899
  });
8564
8900
 
8565
8901
  // src/api/rates.ts
8902
+ function serpActualCostMc(headlessSent) {
8903
+ return headlessSent === true ? MC_COSTS.serp_headless : MC_COSTS.serp_headful;
8904
+ }
8566
8905
  function browserActiveCostMc(activeMs) {
8567
8906
  return Math.round(activeMs * MC_PER_BROWSER_MS);
8568
8907
  }
@@ -8617,12 +8956,12 @@ function insufficientBalanceResponse(balanceMc, requiredMc) {
8617
8956
  topup_url: topupUrl
8618
8957
  };
8619
8958
  }
8620
- var MC_COSTS, MC_PER_BROWSER_MS, BROWSER_OPEN_MIN_BALANCE_MC, MC_PER_CREDIT, BILLING_MC_PER_USD_NUMERATOR, BILLING_MC_PER_USD_DENOMINATOR, BILLING_BASIS_POINTS, USD_NANOS, SCHEDULED_RUN_BASE_CREDITS, SCHEDULED_RUN_BASE_MC, SCHEDULED_AGENT_LLM_MARKUP_BPS, SCHEDULED_AGENT_LLM_MARKUP, SCHEDULED_RUN_DEFAULT_MAX_CREDITS, SCHEDULED_ACTIONS_RATE_POLICY, CREDIT_COST_CATALOG, CONCURRENCY_PRICE_ID, SUBSCRIPTION_TIERS, SUBSCRIPTION_TIER_BY_KEY, CONCURRENCY_SLOT_PRICE_USD, CONCURRENCY_SLOT_PRICE_CURRENCY, CONCURRENCY_SLOT_PRICE_INTERVAL, CONCURRENCY_SLOT_PRICE_LABEL, CONCURRENCY_SLOT_TERMINAL_COMMAND, FREE_SIGNUP_MC, MEMORY_FREE_QUOTA_BYTES, MEMORY_PLANS, MEMORY_PLAN_QUOTA, MEMORY_MARGIN_MULTIPLE, MEMORY_PRO_COST_BUDGET_USD, LedgerOperation, MEMORY_AI_MARGIN_MULTIPLE, SITE_AUDIT_LLM_MARGIN_MULTIPLE, VIDEO_ANALYSIS_LLM_MARGIN_MULTIPLE, MEDIA_TRANSCRIBE_MARGIN_MULTIPLE, MEDIA_TRANSCRIBE_HOLD_MC, MC_PER_USD;
8959
+ var MC_COSTS, MC_PER_BROWSER_MS, BROWSER_OPEN_MIN_BALANCE_MC, MC_PER_CREDIT, BILLING_MC_PER_USD_NUMERATOR, BILLING_MC_PER_USD_DENOMINATOR, BILLING_BASIS_POINTS, USD_NANOS, SCHEDULED_RUN_BASE_CREDITS, SCHEDULED_RUN_BASE_MC, SCHEDULED_AGENT_LLM_MARKUP_BPS, SCHEDULED_AGENT_LLM_MARKUP, SCHEDULED_RUN_DEFAULT_MAX_CREDITS, SCHEDULED_ACTIONS_RATE_POLICY, CREDIT_COST_CATALOG, CONCURRENCY_PRICE_ID, SUBSCRIPTION_TIERS, SUBSCRIPTION_TIER_BY_KEY, CONCURRENCY_SLOT_PRICE_USD, CONCURRENCY_SLOT_PRICE_CURRENCY, CONCURRENCY_SLOT_PRICE_INTERVAL, CONCURRENCY_SLOT_PRICE_LABEL, CONCURRENCY_SLOT_TERMINAL_COMMAND, MEMORY_FREE_QUOTA_BYTES, MEMORY_PLANS, MEMORY_PLAN_QUOTA, MEMORY_MARGIN_MULTIPLE, MEMORY_PRO_COST_BUDGET_USD, LedgerOperation, MEMORY_AI_MARGIN_MULTIPLE, SITE_AUDIT_LLM_MARGIN_MULTIPLE, VIDEO_ANALYSIS_LLM_MARGIN_MULTIPLE, MEDIA_TRANSCRIBE_MARGIN_MULTIPLE, MEDIA_TRANSCRIBE_HOLD_MC, MC_PER_USD;
8621
8960
  var init_rates = __esm({
8622
8961
  "src/api/rates.ts"() {
8623
8962
  "use strict";
8624
8963
  MC_COSTS = {
8625
- serp: 100,
8964
+ serp: 400,
8626
8965
  serp_headless: 400,
8627
8966
  serp_headful: 1400,
8628
8967
  paa: 300,
@@ -8696,7 +9035,7 @@ var init_rates = __esm({
8696
9035
  aliases: ["search_serp", "serp", "google search", "organic results"],
8697
9036
  credits: mcToCredits(MC_COSTS.serp_headless),
8698
9037
  unit: "per search (headless); escalates to headful only when needed",
8699
- notes: `Returns AI Overview, PAA snippet, videos, forums, and local pack. Billed ${mcToCredits(MC_COSTS.serp_headless)} credits when served headless (the common case), ${mcToCredits(MC_COSTS.serp_headful)} credits if an anti-bot escalation forces headful mode (overestimate refunded).`
9038
+ notes: `Returns AI Overview, PAA snippet, videos, forums, and local pack. Billed ${mcToCredits(MC_COSTS.serp_headless)} credits when served headless (the common case), ${mcToCredits(MC_COSTS.serp_headful)} credits if an anti-bot escalation forces headful mode (overestimate refunded). Optional SERP Intelligence page snapshots add ${mcToCredits(MC_COSTS.page_scrape)} credit per attempted URL.`
8700
9039
  },
8701
9040
  {
8702
9041
  key: "paa",
@@ -8919,7 +9258,6 @@ var init_rates = __esm({
8919
9258
  CONCURRENCY_SLOT_PRICE_INTERVAL = "month";
8920
9259
  CONCURRENCY_SLOT_PRICE_LABEL = "$5/month";
8921
9260
  CONCURRENCY_SLOT_TERMINAL_COMMAND = "npx -y -p mcp-scraper@latest mcp-scraper-cli billing concurrency checkout";
8922
- FREE_SIGNUP_MC = 1e5;
8923
9261
  MEMORY_FREE_QUOTA_BYTES = 1e7 * 1;
8924
9262
  MEMORY_PLANS = {
8925
9263
  "price_1TnMSTS8aAcsk3TGBgiwuvqL": { plan: "pro", label: "Pro", price_id: "price_1TnMSTS8aAcsk3TGBgiwuvqL", interval: "month", monthly_usd: 19, quota_bytes: 5e9 },
@@ -16624,6 +16962,15 @@ function buildPageIntelUrl(body, country) {
16624
16962
  if (body.pageId?.trim()) return `https://www.facebook.com/ads/library/?active_status=all&ad_type=all&country=${country}&is_targeted_country=false&media_type=all&search_type=page&view_all_page_id=${body.pageId.trim()}`;
16625
16963
  return `https://www.facebook.com/ads/library/?active_status=all&ad_type=all&country=${country}&q=${encodeURIComponent(body.query.trim())}&search_type=keyword_unordered`;
16626
16964
  }
16965
+ function pageIntelRequestContext(body) {
16966
+ const country = body.country?.trim().toUpperCase() ?? "US";
16967
+ return {
16968
+ country,
16969
+ maxAds: Math.min(200, Math.max(1, body.maxAds ?? 50)),
16970
+ listingUrl: buildPageIntelUrl(body, country),
16971
+ subject: body.pageId ?? body.query ?? body.libraryId ?? ""
16972
+ };
16973
+ }
16627
16974
  function normalizeAdvertiserName(value) {
16628
16975
  return value.toLowerCase().normalize("NFKD").replace(/[^a-z0-9]+/g, " ").trim().replace(/\s+/g, " ");
16629
16976
  }
@@ -16788,16 +17135,14 @@ var init_facebook_ad_routes = __esm({
16788
17135
  const parsed = FacebookPageIntelBodySchema.safeParse(raw);
16789
17136
  if (!parsed.success) return c.json(invalidRequest(parsed.error.issues[0]?.message ?? "Invalid request"), 400);
16790
17137
  const body = parsed.data;
16791
- const maxAds = Math.min(200, Math.max(1, body.maxAds ?? 50));
16792
- const country = body.country?.trim().toUpperCase() ?? "US";
16793
- const listingUrl = buildPageIntelUrl(body, country);
17138
+ const { country, maxAds, listingUrl, subject } = pageIntelRequestContext(body);
16794
17139
  const fbUser = c.get("user");
16795
17140
  const gate = await acquireConcurrencyGate(fbUser, "facebook_page_intel", { reuseLockId: c.req.header("x-mcp-scraper-concurrency-lock"), metadata: { pageId: body.pageId, query: body.query, libraryId: body.libraryId, country } });
16796
17141
  if (!gate.ok) return c.json(concurrencyLimitExceededResponse(gate), 429, { "Retry-After": String(gate.retryAfterSeconds) });
16797
17142
  const driver = new BrowserDriver();
16798
17143
  let refunded = false, debited = false;
16799
17144
  try {
16800
- const { ok: fbOk, balance_mc: fbBal } = await debitMc(fbUser.id, MC_COSTS.fb_ad, LedgerOperation.FB_AD, body.pageId ?? body.query ?? body.libraryId ?? "");
17145
+ const { ok: fbOk, balance_mc: fbBal } = await debitMc(fbUser.id, MC_COSTS.fb_ad, LedgerOperation.FB_AD, subject);
16801
17146
  if (!fbOk) return c.json(insufficientBalanceResponse(fbBal, MC_COSTS.fb_ad), 402);
16802
17147
  debited = true;
16803
17148
  await driver.launch(await kernelLaunchOptsResidential());
@@ -16807,17 +17152,17 @@ var init_facebook_ad_routes = __esm({
16807
17152
  if (result.ads.length === 0 && await detectSoftBlock(driver)) {
16808
17153
  await creditMc(fbUser.id, MC_COSTS.fb_ad, LedgerOperation.FB_AD_REFUND, "soft-block empty result");
16809
17154
  refunded = true;
16810
- await logRequestEvent({ userId: fbUser.id, source: "facebook_page_intel", status: "failed", query: body.pageId ?? body.query ?? body.libraryId ?? "", error: "soft-block: empty result refunded" });
17155
+ await logRequestEvent({ userId: fbUser.id, source: "facebook_page_intel", status: "failed", query: subject, error: "soft-block: empty result refunded" });
16811
17156
  return c.json({ error: "soft-block: no ads returned (refunded)" }, 503);
16812
17157
  }
16813
17158
  const match = assessPageIntelMatch(body, result);
16814
17159
  const output = { ...result, ...match };
16815
- await logRequestEvent({ userId: fbUser.id, source: "facebook_page_intel", status: "done", query: body.pageId ?? body.query ?? body.libraryId ?? "", resultCount: result.ads.length, result: output });
17160
+ await logRequestEvent({ userId: fbUser.id, source: "facebook_page_intel", status: "done", query: subject, resultCount: result.ads.length, result: output });
16816
17161
  return c.json(output);
16817
17162
  } catch (err) {
16818
17163
  const msg = err instanceof Error ? err.message : String(err);
16819
17164
  if (debited && !refunded) await creditMc(fbUser.id, MC_COSTS.fb_ad, LedgerOperation.FB_AD_REFUND, "failed call");
16820
- await logRequestEvent({ userId: fbUser.id, source: "facebook_page_intel", status: "failed", query: body.pageId ?? body.query ?? body.libraryId ?? "", error: msg });
17165
+ await logRequestEvent({ userId: fbUser.id, source: "facebook_page_intel", status: "failed", query: subject, error: msg });
16821
17166
  if (msg.toLowerCase().includes("blocked") || msg.toLowerCase().includes("captcha")) {
16822
17167
  return c.json({ error: msg }, 503);
16823
17168
  }
@@ -22567,21 +22912,33 @@ function formatCaptureSerpSnapshot(raw, input) {
22567
22912
  const parsed = parseData(raw);
22568
22913
  if ("error" in parsed) return { content: [{ type: "text", text: parsed.error }], isError: true };
22569
22914
  const d = parsed.data;
22915
+ const harvestResult = d.harvestResult && typeof d.harvestResult === "object" ? d.harvestResult : d;
22570
22916
  const providerPayload = d;
22571
- const organic = Array.isArray(d.organicResults) ? d.organicResults : Array.isArray(d.organic) ? d.organic : [];
22572
- const localPack = Array.isArray(d.localPack) ? d.localPack : [];
22573
- const artifacts = Array.isArray(d.artifacts) ? d.artifacts : [];
22574
- const status = String(d.status ?? d.captureStatus ?? "captured");
22575
- const query = typeof d.query === "string" ? d.query : typeof input.query === "string" ? input.query : null;
22576
- const location2 = typeof d.location === "string" ? d.location : typeof input.location === "string" ? input.location : null;
22577
- const capturedAt = typeof d.capturedAt === "string" ? d.capturedAt : typeof d.extractedAt === "string" ? d.extractedAt : null;
22917
+ const organic = Array.isArray(harvestResult.organicResults) ? harvestResult.organicResults : Array.isArray(harvestResult.organic) ? harvestResult.organic : [];
22918
+ const localPack = Array.isArray(harvestResult.localPack) ? harvestResult.localPack : [];
22919
+ const artifacts = Array.isArray(d.pageSnapshotArtifacts) ? d.pageSnapshotArtifacts : Array.isArray(d.artifacts) ? d.artifacts : [];
22920
+ const attempts = Array.isArray(d.attempts) ? d.attempts : [];
22921
+ const diagnostics = harvestResult.diagnostics && typeof harvestResult.diagnostics === "object" ? harvestResult.diagnostics : d.diagnostics && typeof d.diagnostics === "object" ? d.diagnostics : null;
22922
+ const completionStatus = diagnostics && typeof diagnostics.completionStatus === "string" ? diagnostics.completionStatus : null;
22923
+ const status = String(d.status ?? d.captureStatus ?? completionStatus ?? "captured");
22924
+ const query = typeof harvestResult.seed === "string" ? harvestResult.seed : typeof harvestResult.query === "string" ? harvestResult.query : typeof input.query === "string" ? input.query : null;
22925
+ const location2 = typeof harvestResult.location === "string" ? harvestResult.location : typeof input.location === "string" ? input.location : null;
22926
+ const capturedAt = typeof harvestResult.extractedAt === "string" ? harvestResult.extractedAt : typeof harvestResult.capturedAt === "string" ? harvestResult.capturedAt : null;
22578
22927
  const resultCount = organic.length + localPack.length;
22928
+ const pageSnapshotFailedCount = artifacts.filter((artifact) => artifact.status === "failed" || Boolean(artifact.error)).length;
22929
+ const rawBilling = d.billing && typeof d.billing === "object" ? d.billing : null;
22930
+ const billing = rawBilling ? {
22931
+ creditsUsed: typeof rawBilling.creditsUsed === "number" ? rawBilling.creditsUsed : null,
22932
+ requestId: typeof rawBilling.requestId === "string" ? rawBilling.requestId : null,
22933
+ jobId: typeof rawBilling.jobId === "string" ? rawBilling.jobId : null
22934
+ } : null;
22579
22935
  const full = [
22580
22936
  `# SERP Intelligence Snapshot: ${query ?? "query"}`,
22581
22937
  `**Status:** ${status}`,
22582
22938
  location2 ? `**Location:** ${location2}` : "",
22583
- `**Result count:** ${resultCount}`,
22584
- artifacts.length ? `**Artifacts:** ${artifacts.length}` : "",
22939
+ `**SERP results:** ${resultCount} (${organic.length} organic, ${localPack.length} local pack)`,
22940
+ `**Page snapshots:** ${artifacts.length} (${pageSnapshotFailedCount} failed)`,
22941
+ billing?.creditsUsed !== null && billing?.creditsUsed !== void 0 ? `**Credits used:** ${billing.creditsUsed}` : "",
22585
22942
  "",
22586
22943
  "Use `capture_serp_page_snapshots` when you need page-level evidence for ranking URLs."
22587
22944
  ].filter(Boolean).join("\n");
@@ -22594,6 +22951,10 @@ function formatCaptureSerpSnapshot(raw, input) {
22594
22951
  location: location2,
22595
22952
  capturedAt,
22596
22953
  resultCount,
22954
+ organicResultCount: organic.length,
22955
+ localPackResultCount: localPack.length,
22956
+ pageSnapshotCount: artifacts.length,
22957
+ pageSnapshotFailedCount,
22597
22958
  snapshotId: typeof d.snapshotId === "string" ? d.snapshotId : typeof d.snapshot_id === "string" ? d.snapshot_id : typeof d.id === "string" ? d.id : null,
22598
22959
  resolvedInputs: {
22599
22960
  query: input.query ?? null,
@@ -22606,7 +22967,9 @@ function formatCaptureSerpSnapshot(raw, input) {
22606
22967
  pages: input.pages ?? null
22607
22968
  },
22608
22969
  artifacts,
22609
- diagnostics: d.diagnostics && typeof d.diagnostics === "object" ? d.diagnostics : null,
22970
+ attempts,
22971
+ billing,
22972
+ diagnostics,
22610
22973
  providerPayload
22611
22974
  }
22612
22975
  };
@@ -22615,9 +22978,13 @@ function formatCaptureSerpPageSnapshots(raw, input) {
22615
22978
  const parsed = parseData(raw);
22616
22979
  if ("error" in parsed) return { content: [{ type: "text", text: parsed.error }], isError: true };
22617
22980
  const d = parsed.data;
22618
- const captures = Array.isArray(d.captures) ? d.captures : Array.isArray(d.results) ? d.results : [];
22981
+ const captures = Array.isArray(d.pageSnapshotArtifacts) ? d.pageSnapshotArtifacts : Array.isArray(d.captures) ? d.captures : Array.isArray(d.results) ? d.results : [];
22619
22982
  const failedCount = captures.filter((c) => c.status === "failed" || c.error).length;
22620
- const status = String(d.status ?? (failedCount ? "partial" : "captured"));
22983
+ const capturedCount = captures.filter((c) => c.status === "captured" || c.status === "ok").length;
22984
+ const skippedCount = captures.filter((c) => c.status === "skipped").length;
22985
+ const attempts = Array.isArray(d.attempts) ? d.attempts : [];
22986
+ const inferredStatus = captures.length === 0 ? "empty" : failedCount === captures.length ? "failed" : failedCount > 0 || skippedCount > 0 ? "partial" : "captured";
22987
+ const status = String(d.status ?? inferredStatus);
22621
22988
  const rows = captures.slice(0, 25).map((capture, i) => {
22622
22989
  const url = typeof capture.url === "string" ? capture.url : "";
22623
22990
  const sourceKind = typeof capture.sourceKind === "string" ? capture.sourceKind : typeof capture.source_kind === "string" ? capture.source_kind : "configured_target";
@@ -22626,8 +22993,10 @@ function formatCaptureSerpPageSnapshots(raw, input) {
22626
22993
  const full = [
22627
22994
  "# SERP Intelligence Page Snapshots",
22628
22995
  `**Status:** ${status}`,
22629
- `**Captured:** ${captures.length}`,
22996
+ `**Attempted:** ${captures.length}`,
22997
+ `**Captured:** ${capturedCount}`,
22630
22998
  `**Failed:** ${failedCount}`,
22999
+ skippedCount ? `**Skipped:** ${skippedCount}` : "",
22631
23000
  captures.length ? `
22632
23001
  | # | URL | Source | Status |
22633
23002
  |---|-----|--------|--------|
@@ -22639,8 +23008,11 @@ ${rows}` : ""
22639
23008
  schemaVersion: "serp-intelligence.page-snapshots.v1",
22640
23009
  status,
22641
23010
  count: captures.length,
23011
+ capturedCount,
22642
23012
  failedCount,
23013
+ skippedCount,
22643
23014
  captures,
23015
+ attempts,
22644
23016
  resolvedInputs: {
22645
23017
  urls: input.urls ?? null,
22646
23018
  targets: input.targets ?? null,
@@ -28180,12 +28552,22 @@ var init_schemas4 = __esm({
28180
28552
  sourcePosition: import_zod33.z.number().int().min(1).optional()
28181
28553
  }).strict();
28182
28554
  SerpIntelligencePageSnapshotsBodySchema = import_zod33.z.object({
28183
- urls: import_zod33.z.array(SerpIntelligencePublicHttpUrlSchema).min(1).max(25),
28555
+ urls: import_zod33.z.array(SerpIntelligencePublicHttpUrlSchema).min(1).max(25).optional(),
28184
28556
  targets: import_zod33.z.array(SerpIntelligencePageSnapshotRequestSchema).min(1).max(25).optional(),
28185
28557
  maxConcurrency: import_zod33.z.number().int().min(1).max(5).default(2),
28186
28558
  timeoutMs: import_zod33.z.number().int().min(1e3).max(6e4).default(15e3),
28187
28559
  debug: import_zod33.z.boolean().default(false)
28188
- }).strict();
28560
+ }).strict().superRefine((body, ctx) => {
28561
+ const hasUrls = Boolean(body.urls?.length);
28562
+ const hasTargets = Boolean(body.targets?.length);
28563
+ if (hasUrls === hasTargets) {
28564
+ ctx.addIssue({
28565
+ code: import_zod33.z.ZodIssueCode.custom,
28566
+ message: "provide exactly one of urls or targets",
28567
+ path: hasUrls ? ["targets"] : ["urls"]
28568
+ });
28569
+ }
28570
+ });
28189
28571
  SerpIntelligenceAICitationSchema = import_zod33.z.object({
28190
28572
  text: import_zod33.z.string(),
28191
28573
  href: import_zod33.z.string()
@@ -28445,14 +28827,24 @@ async function captureSerpIntelligenceSnapshot(rawInput, runtimeOptions = {}) {
28445
28827
  });
28446
28828
  const pageSnapshotLimit = normalizePageSnapshotLimit(parsedInput);
28447
28829
  const pageSnapshotTargets = collectPageSnapshotTargets(harvestResult, pageSnapshotLimit);
28448
- const pageSnapshotArtifacts = pageSnapshotTargets.length > 0 ? (await capturePageSnapshotsFn(pageSnapshotTargets, {
28449
- kernelApiKey: runtimeOptions.kernelApiKey ?? browserServiceApiKey(),
28450
- timeoutMs: runtimeOptions.pageSnapshotTimeoutMs,
28451
- maxConcurrency: runtimeOptions.pageSnapshotMaxConcurrency,
28452
- debug,
28453
- includeContent: runtimeOptions.includePageSnapshotContent,
28454
- maxContentChars: runtimeOptions.maxPageSnapshotContentChars
28455
- })).pageSnapshotArtifacts.map(stripPageSnapshotArtifact) : [];
28830
+ const pageSnapshotArtifacts = pageSnapshotTargets.length > 0 ? (await runWithCostContext(
28831
+ {
28832
+ ...currentCostContext() ?? { op: "serp" },
28833
+ subOp: "page_snapshot",
28834
+ // Page snapshot launches have their own fixed product rate. They
28835
+ // must not overwrite the headless/headful mode used to settle the
28836
+ // preceding SERP harvest.
28837
+ headlessSentOut: void 0
28838
+ },
28839
+ () => capturePageSnapshotsFn(pageSnapshotTargets, {
28840
+ kernelApiKey: runtimeOptions.kernelApiKey ?? browserServiceApiKey(),
28841
+ timeoutMs: runtimeOptions.pageSnapshotTimeoutMs,
28842
+ maxConcurrency: runtimeOptions.pageSnapshotMaxConcurrency,
28843
+ debug,
28844
+ includeContent: runtimeOptions.includePageSnapshotContent,
28845
+ maxContentChars: runtimeOptions.maxPageSnapshotContentChars
28846
+ })
28847
+ )).pageSnapshotArtifacts.map(stripPageSnapshotArtifact) : [];
28456
28848
  return normalizeHarvestResultForSerpIntelligence({
28457
28849
  harvestResult,
28458
28850
  attempts,
@@ -28469,6 +28861,7 @@ var init_serp_capture_service = __esm({
28469
28861
  "use strict";
28470
28862
  init_harvest();
28471
28863
  init_browser_service_env();
28864
+ init_cost_context();
28472
28865
  init_harvest_problems();
28473
28866
  init_page_snapshot_extractor();
28474
28867
  init_schemas4();
@@ -28542,17 +28935,35 @@ async function enforceSerpIntelligenceRateLimit(c, userId) {
28542
28935
  );
28543
28936
  }
28544
28937
  function pageSnapshotTargetsFromBody(body) {
28545
- return body.targets?.length ? body.targets : body.urls.map((url) => ({
28938
+ return body.targets?.length ? body.targets : (body.urls ?? []).map((url) => ({
28546
28939
  url,
28547
28940
  sourceKind: "configured_target",
28548
28941
  sourcePosition: null
28549
28942
  }));
28550
28943
  }
28551
- var import_hono16, SERP_INTELLIGENCE_RATE_LIMIT, SERP_INTELLIGENCE_RATE_WINDOW_SECONDS, POST_CAPTURE_ROUTE_LABEL, POST_PAGE_SNAPSHOTS_ROUTE_LABEL, serpIntelligenceApp;
28944
+ async function logRequestEventBestEffort(input) {
28945
+ try {
28946
+ await logRequestEvent(input);
28947
+ } catch (error) {
28948
+ console.warn("[serp-intelligence] request audit log failed:", error instanceof Error ? error.message : String(error));
28949
+ }
28950
+ }
28951
+ function captureBillingKeys(userId, suppliedKey, body) {
28952
+ const responseKey = suppliedKey?.trim() || (0, import_node_crypto12.randomUUID)();
28953
+ const requestFingerprint = (0, import_node_crypto12.createHash)("sha256").update(JSON.stringify(body)).digest("hex");
28954
+ const keyDigest = (0, import_node_crypto12.createHash)("sha256").update(String(userId)).update("\0").update(responseKey).digest("hex");
28955
+ return {
28956
+ responseKey,
28957
+ debitKey: `serp-capture:${userId}:${keyDigest}`,
28958
+ debitDescription: `${body.query} [request:${requestFingerprint}]`
28959
+ };
28960
+ }
28961
+ var import_hono16, import_node_crypto12, SERP_INTELLIGENCE_RATE_LIMIT, SERP_INTELLIGENCE_RATE_WINDOW_SECONDS, POST_CAPTURE_ROUTE_LABEL, POST_PAGE_SNAPSHOTS_ROUTE_LABEL, SERP_CAPTURE_BILLING_SOURCE, serpIntelligenceApp;
28552
28962
  var init_serp_intelligence_routes = __esm({
28553
28963
  "src/api/serp-intelligence-routes.ts"() {
28554
28964
  "use strict";
28555
28965
  import_hono16 = require("hono");
28966
+ import_node_crypto12 = require("crypto");
28556
28967
  init_browser_service_env();
28557
28968
  init_page_snapshot_extractor();
28558
28969
  init_serp_capture_service();
@@ -28560,11 +28971,13 @@ var init_serp_intelligence_routes = __esm({
28560
28971
  init_api_auth();
28561
28972
  init_db();
28562
28973
  init_rates();
28974
+ init_cost_context();
28563
28975
  init_concurrency_gates();
28564
28976
  SERP_INTELLIGENCE_RATE_LIMIT = 60;
28565
28977
  SERP_INTELLIGENCE_RATE_WINDOW_SECONDS = 60;
28566
28978
  POST_CAPTURE_ROUTE_LABEL = "POST /capture";
28567
28979
  POST_PAGE_SNAPSHOTS_ROUTE_LABEL = "POST /page-snapshots";
28980
+ SERP_CAPTURE_BILLING_SOURCE = "serp_intelligence_capture";
28568
28981
  serpIntelligenceApp = new import_hono16.Hono();
28569
28982
  serpIntelligenceApp.use("*", createApiKeyAuth());
28570
28983
  serpIntelligenceApp.post("/capture", async (c) => {
@@ -28573,6 +28986,17 @@ var init_serp_intelligence_routes = __esm({
28573
28986
  const raw = await c.req.json().catch(() => ({}));
28574
28987
  const parsed = SerpIntelligenceCaptureBodySchema.safeParse(raw);
28575
28988
  if (!parsed.success) return c.json(formatZodError(parsed.error), 400);
28989
+ const suppliedIdempotencyKey = c.req.header("idempotency-key") ?? c.req.header("x-idempotency-key");
28990
+ if (suppliedIdempotencyKey && suppliedIdempotencyKey.trim().length > 500) {
28991
+ return c.json(structuredError({
28992
+ error_code: "invalid_request",
28993
+ error_type: "validation_error",
28994
+ message: "Idempotency-Key must be 500 characters or fewer.",
28995
+ retryable: false
28996
+ }), 400);
28997
+ }
28998
+ const billingKeys = captureBillingKeys(user.id, suppliedIdempotencyKey, parsed.data);
28999
+ c.header("Idempotency-Key", billingKeys.responseKey);
28576
29000
  const limited = await enforceSerpIntelligenceRateLimit(c, user.id);
28577
29001
  if (limited) return limited;
28578
29002
  const gate = await acquireConcurrencyGate(user, "serp_intelligence_capture", {
@@ -28580,51 +29004,130 @@ var init_serp_intelligence_routes = __esm({
28580
29004
  metadata: { query: parsed.data.query, location: parsed.data.location }
28581
29005
  });
28582
29006
  if (!gate.ok) return c.json(concurrencyLimitExceededResponse(gate), 429, { "Retry-After": String(gate.retryAfterSeconds) });
28583
- const cost = MC_COSTS.serp;
28584
- let debited = false;
29007
+ const pageSnapshotHold = parsed.data.includePageSnapshots ? parsed.data.pageSnapshotLimit * MC_COSTS.page_scrape : 0;
29008
+ const cost = MC_COSTS.serp_headful + pageSnapshotHold;
29009
+ const headlessSentOut = { value: null };
28585
29010
  try {
28586
- const { ok, balance_mc } = await debitMc(
29011
+ const { ok, balance_mc } = await debitMcIdempotent(
28587
29012
  user.id,
28588
29013
  cost,
28589
29014
  LedgerOperation.SERP,
28590
- parsed.data.query
29015
+ billingKeys.debitDescription,
29016
+ billingKeys.debitKey
28591
29017
  );
28592
- if (!ok) return c.json(insufficientBalanceResponse(balance_mc, cost), 402);
28593
- debited = true;
28594
- const result = await captureSerpIntelligenceSnapshot(parsed.data, {
28595
- kernelApiKey: browserServiceApiKey(),
28596
- kernelProxyId: browserServiceProxyId(),
28597
- signal: c.req.raw.signal,
28598
- billing: { creditsUsed: cost / MC_PER_CREDIT }
29018
+ if (!ok) {
29019
+ await releaseConcurrencyGate(gate.lockId);
29020
+ return c.json(insufficientBalanceResponse(balance_mc, cost), 402);
29021
+ }
29022
+ } catch (error) {
29023
+ const idempotencyConflict = error instanceof Error && error.message === "idempotent debit key conflicts with another debit";
29024
+ const body = structuredError({
29025
+ error_code: idempotencyConflict ? "idempotency_conflict" : "capture_failed",
29026
+ error_type: idempotencyConflict ? "validation_error" : "capture_error",
29027
+ message: error instanceof Error ? error.message : String(error),
29028
+ retryable: !idempotencyConflict
28599
29029
  });
28600
- await logRequestEvent({
29030
+ await logRequestEventBestEffort({
28601
29031
  userId: user.id,
28602
- source: "serp_intelligence_capture",
28603
- status: "done",
29032
+ source: SERP_CAPTURE_BILLING_SOURCE,
29033
+ status: "failed",
28604
29034
  query: parsed.data.query,
28605
29035
  location: parsed.data.location,
28606
- resultCount: result.harvestResult.organicResults.length,
28607
- result
29036
+ error: body.message
28608
29037
  });
28609
- return c.json(result);
29038
+ await releaseConcurrencyGate(gate.lockId);
29039
+ return c.json(body, idempotencyConflict ? 409 : 500);
29040
+ }
29041
+ let result;
29042
+ try {
29043
+ const parentCostContext = currentCostContext();
29044
+ result = await runWithCostContext(
29045
+ {
29046
+ ...parentCostContext ?? {},
29047
+ op: "serp",
29048
+ userId: Number(user.id),
29049
+ headlessSentOut
29050
+ },
29051
+ () => captureSerpIntelligenceSnapshot(parsed.data, {
29052
+ kernelApiKey: browserServiceApiKey(),
29053
+ kernelProxyId: browserServiceProxyId(),
29054
+ signal: c.req.raw.signal
29055
+ })
29056
+ );
28610
29057
  } catch (error) {
28611
- if (debited) await creditMc(user.id, cost, LedgerOperation.REFUND, "failed serp intelligence capture");
29058
+ try {
29059
+ await settleDebitMcIdempotent(
29060
+ user.id,
29061
+ billingKeys.debitKey,
29062
+ 0,
29063
+ LedgerOperation.SERP_REFUND,
29064
+ "failed serp intelligence capture",
29065
+ SERP_CAPTURE_BILLING_SOURCE
29066
+ );
29067
+ } catch (settlementError) {
29068
+ console.warn(
29069
+ "[serp-intelligence] failed capture refund remains recoverable:",
29070
+ settlementError instanceof Error ? settlementError.message : String(settlementError)
29071
+ );
29072
+ }
28612
29073
  const body = error instanceof SerpIntelligenceCaptureError ? error.toJSON() : structuredError({
28613
29074
  error_code: "capture_failed",
28614
29075
  error_type: "capture_error",
28615
29076
  message: error instanceof Error ? error.message : String(error),
28616
29077
  retryable: true
28617
29078
  });
28618
- await logRequestEvent({
29079
+ await logRequestEventBestEffort({
28619
29080
  userId: user.id,
28620
- source: "serp_intelligence_capture",
29081
+ source: SERP_CAPTURE_BILLING_SOURCE,
28621
29082
  status: "failed",
28622
29083
  query: parsed.data.query,
28623
29084
  location: parsed.data.location,
28624
29085
  error: "message" in body && typeof body.message === "string" ? body.message : "SERP Intelligence capture failed"
28625
29086
  });
28626
29087
  const status = error instanceof SerpIntelligenceCaptureError ? error.httpStatus : 500;
29088
+ await releaseConcurrencyGate(gate.lockId);
28627
29089
  return c.json(body, status);
29090
+ }
29091
+ try {
29092
+ const requestedActualCost = serpActualCostMc(headlessSentOut.value) + result.pageSnapshotArtifacts.length * MC_COSTS.page_scrape;
29093
+ const settlement = await settleDebitMcIdempotent(
29094
+ user.id,
29095
+ billingKeys.debitKey,
29096
+ requestedActualCost,
29097
+ LedgerOperation.SERP_REFUND,
29098
+ "SERP capture pricing settle",
29099
+ SERP_CAPTURE_BILLING_SOURCE
29100
+ );
29101
+ const settledResult = {
29102
+ ...result,
29103
+ billing: { creditsUsed: settlement.final_amount_mc / MC_PER_CREDIT }
29104
+ };
29105
+ await logRequestEventBestEffort({
29106
+ userId: user.id,
29107
+ source: SERP_CAPTURE_BILLING_SOURCE,
29108
+ status: "done",
29109
+ query: parsed.data.query,
29110
+ location: parsed.data.location,
29111
+ resultCount: settledResult.harvestResult.organicResults.length,
29112
+ result: settledResult
29113
+ });
29114
+ return c.json(settledResult);
29115
+ } catch (error) {
29116
+ const body = structuredError({
29117
+ error_code: "capture_failed",
29118
+ error_type: "capture_error",
29119
+ message: error instanceof Error ? error.message : String(error),
29120
+ retryable: true
29121
+ });
29122
+ await logRequestEventBestEffort({
29123
+ userId: user.id,
29124
+ source: SERP_CAPTURE_BILLING_SOURCE,
29125
+ status: "failed",
29126
+ query: parsed.data.query,
29127
+ location: parsed.data.location,
29128
+ error: body.message
29129
+ });
29130
+ return c.json(body, 500);
28628
29131
  } finally {
28629
29132
  await releaseConcurrencyGate(gate.lockId);
28630
29133
  }
@@ -28660,7 +29163,7 @@ var init_serp_intelligence_routes = __esm({
28660
29163
  maxConcurrency: parsed.data.maxConcurrency,
28661
29164
  debug: parsed.data.debug
28662
29165
  });
28663
- await logRequestEvent({
29166
+ await logRequestEventBestEffort({
28664
29167
  userId: user.id,
28665
29168
  source: "serp_intelligence_page_snapshots",
28666
29169
  status: "done",
@@ -28677,7 +29180,7 @@ var init_serp_intelligence_routes = __esm({
28677
29180
  message: error instanceof Error ? error.message : String(error),
28678
29181
  retryable: true
28679
29182
  });
28680
- await logRequestEvent({
29183
+ await logRequestEventBestEffort({
28681
29184
  userId: user.id,
28682
29185
  source: "serp_intelligence_page_snapshots",
28683
29186
  status: "failed",
@@ -28697,7 +29200,7 @@ var PACKAGE_VERSION;
28697
29200
  var init_version = __esm({
28698
29201
  "src/version.ts"() {
28699
29202
  "use strict";
28700
- PACKAGE_VERSION = "0.21.3";
29203
+ PACKAGE_VERSION = "0.21.4";
28701
29204
  }
28702
29205
  });
28703
29206
 
@@ -29983,9 +30486,19 @@ var init_mcp_tool_schemas = __esm({
29983
30486
  location: NullableString,
29984
30487
  capturedAt: NullableString,
29985
30488
  resultCount: import_zod34.z.number().int().min(0).nullable(),
30489
+ organicResultCount: import_zod34.z.number().int().min(0),
30490
+ localPackResultCount: import_zod34.z.number().int().min(0),
30491
+ pageSnapshotCount: import_zod34.z.number().int().min(0),
30492
+ pageSnapshotFailedCount: import_zod34.z.number().int().min(0),
29986
30493
  snapshotId: NullableString,
29987
30494
  resolvedInputs: import_zod34.z.record(import_zod34.z.unknown()),
29988
30495
  artifacts: import_zod34.z.array(import_zod34.z.record(import_zod34.z.unknown())),
30496
+ attempts: import_zod34.z.array(import_zod34.z.record(import_zod34.z.unknown())),
30497
+ billing: import_zod34.z.object({
30498
+ creditsUsed: import_zod34.z.number().min(0).nullable(),
30499
+ requestId: NullableString,
30500
+ jobId: NullableString
30501
+ }).nullable(),
29989
30502
  diagnostics: import_zod34.z.record(import_zod34.z.unknown()).nullable(),
29990
30503
  providerPayload: import_zod34.z.record(import_zod34.z.unknown())
29991
30504
  };
@@ -29993,8 +30506,11 @@ var init_mcp_tool_schemas = __esm({
29993
30506
  schemaVersion: import_zod34.z.literal("serp-intelligence.page-snapshots.v1"),
29994
30507
  status: import_zod34.z.string(),
29995
30508
  count: import_zod34.z.number().int().min(0),
30509
+ capturedCount: import_zod34.z.number().int().min(0),
29996
30510
  failedCount: import_zod34.z.number().int().min(0),
30511
+ skippedCount: import_zod34.z.number().int().min(0),
29997
30512
  captures: import_zod34.z.array(import_zod34.z.record(import_zod34.z.unknown())),
30513
+ attempts: import_zod34.z.array(import_zod34.z.record(import_zod34.z.unknown())),
29998
30514
  resolvedInputs: import_zod34.z.record(import_zod34.z.unknown()),
29999
30515
  diagnostics: import_zod34.z.record(import_zod34.z.unknown()).nullable(),
30000
30516
  providerPayload: import_zod34.z.record(import_zod34.z.unknown())
@@ -30119,8 +30635,8 @@ var init_mcp_tool_schemas = __esm({
30119
30635
  proxyZip: import_zod34.z.string().regex(/^\d{5}$/).optional().describe('US ZIP for residential geo-IP targeting. Only meaningful with proxyMode "location".'),
30120
30636
  pages: import_zod34.z.number().int().min(1).max(2).default(1).describe("Google result pages to capture. Use 2 only for deeper ranking evidence."),
30121
30637
  debug: import_zod34.z.boolean().default(false).describe("Include sanitized browser/proxy/location diagnostics."),
30122
- includePageSnapshots: import_zod34.z.boolean().default(false).describe("Also capture ranking-page snapshots for selected SERP URLs."),
30123
- pageSnapshotLimit: import_zod34.z.number().int().min(0).max(10).default(0).describe("Maximum ranking-page snapshots when includePageSnapshots is true.")
30638
+ includePageSnapshots: import_zod34.z.boolean().default(false).describe("Also capture ranking-page snapshots for selected SERP URLs. Each attempted snapshot adds 1 Credit."),
30639
+ pageSnapshotLimit: import_zod34.z.number().int().min(0).max(10).default(0).describe("Maximum ranking-page snapshots when includePageSnapshots is true. This capacity is held up front and unused capacity is refunded.")
30124
30640
  };
30125
30641
  ScreenshotInputSchema = {
30126
30642
  url: import_zod34.z.string().url().describe("URL to capture as a full-page screenshot. Use http or https. Pass allowLocal: true to capture localhost or private-network URLs during development."),
@@ -30128,12 +30644,12 @@ var init_mcp_tool_schemas = __esm({
30128
30644
  allowLocal: import_zod34.z.boolean().default(false).describe("Allow localhost and private-network URLs (127.x, 192.168.x, 10.x, etc.). For local development only \u2014 not for production use.")
30129
30645
  };
30130
30646
  CaptureSerpPageSnapshotsInputSchema = {
30131
- urls: import_zod34.z.array(import_zod34.z.string().url()).min(1).max(25).describe("Public HTTP/HTTPS URLs to capture. Do not pass localhost, private IPs, file URLs, or internal admin URLs."),
30647
+ urls: import_zod34.z.array(import_zod34.z.string().url()).min(1).max(25).optional().describe("Public HTTP/HTTPS URLs to capture. Provide exactly one of urls or targets. Do not pass localhost, private IPs, file URLs, or internal admin URLs."),
30132
30648
  targets: import_zod34.z.array(import_zod34.z.object({
30133
30649
  url: import_zod34.z.string().url().describe("Public HTTP/HTTPS URL to capture."),
30134
30650
  sourceKind: import_zod34.z.enum(["organic", "ai_citation", "local_pack_website", "configured_target", "site_subject"]).default("configured_target").describe("Why this page is being captured."),
30135
30651
  sourcePosition: import_zod34.z.number().int().min(1).optional().describe("Ranking or citation position when the page came from SERP evidence.")
30136
- }).strict()).min(1).max(25).optional().describe("Structured targets. Use instead of urls when source kind or position should be preserved."),
30652
+ }).strict()).min(1).max(25).optional().describe("Structured targets. Provide exactly one of targets or urls; use targets when source kind or position should be preserved."),
30137
30653
  maxConcurrency: import_zod34.z.number().int().min(1).max(5).default(2).describe("Parallel page captures."),
30138
30654
  timeoutMs: import_zod34.z.number().int().min(1e3).max(6e4).default(15e3).describe("Per-page capture timeout in milliseconds; timeouts return as structured capture failures."),
30139
30655
  debug: import_zod34.z.boolean().default(false).describe("Include sanitized browser/proxy diagnostics.")
@@ -30769,7 +31285,7 @@ var init_rank_tracker_blueprint = __esm({
30769
31285
 
30770
31286
  // src/mcp/paa-mcp-server.ts
30771
31287
  function hashOwnerId(callerKey) {
30772
- return (0, import_node_crypto12.createHash)("sha256").update(callerKey).digest("hex").slice(0, 24);
31288
+ return (0, import_node_crypto13.createHash)("sha256").update(callerKey).digest("hex").slice(0, 24);
30773
31289
  }
30774
31290
  function liveWebToolAnnotations(title) {
30775
31291
  return {
@@ -30783,14 +31299,14 @@ function liveWebToolAnnotations(title) {
30783
31299
  function registerSerpIntelligenceCaptureTools(server, executor) {
30784
31300
  server.registerTool("capture_serp_snapshot", {
30785
31301
  title: "SERP Intelligence Snapshot",
30786
- description: "Capture a structured SERP Intelligence snapshot of a Google query \u2014 the persistent evidence format used by rank-tracking and comparison pipelines. Split query from location; leave proxyMode unset.",
31302
+ description: "Capture a structured SERP Intelligence snapshot of a Google query \u2014 the persistent evidence format used by rank-tracking and comparison pipelines. Split query from location; leave proxyMode unset. Costs 4 Credits when headless or 14 if anti-bot escalation requires headful mode; the 14-Credit hold is settled to the mode used. Optional page snapshots add 1 Credit per attempted URL.",
30787
31303
  inputSchema: CaptureSerpSnapshotInputSchema,
30788
31304
  outputSchema: recordOutputSchema("capture_serp_snapshot", CaptureSerpSnapshotOutputSchema),
30789
31305
  annotations: liveWebToolAnnotations("SERP Intelligence Snapshot")
30790
31306
  }, async (input) => formatCaptureSerpSnapshot(await executor.captureSerpSnapshot(input), input));
30791
31307
  server.registerTool("capture_serp_page_snapshots", {
30792
31308
  title: "SERP Intelligence Page Snapshots",
30793
- description: "Capture public ranking pages as SERP Intelligence page snapshots \u2014 persistent page evidence linked to a captured SERP. Provide urls, or targets to preserve source metadata. Private IPs, localhost, file, and internal URLs are rejected.",
31309
+ description: "Capture public ranking pages as SERP Intelligence page snapshots \u2014 persistent page evidence linked to a captured SERP. Provide exactly one of urls or targets; use targets to preserve source metadata. Private IPs, localhost, file, and internal URLs are rejected. Costs 1 Credit per attempted URL.",
30794
31310
  inputSchema: CaptureSerpPageSnapshotsInputSchema,
30795
31311
  outputSchema: recordOutputSchema("capture_serp_page_snapshots", CaptureSerpPageSnapshotsOutputSchema),
30796
31312
  annotations: liveWebToolAnnotations("SERP Intelligence Page Snapshots")
@@ -31213,14 +31729,14 @@ function registerPaaExtractorMcpTools(server, executor, options = {}) {
31213
31729
  annotations: { title: "Set Scheduled Action Connections", readOnlyHint: false, destructiveHint: false, idempotentHint: true, openWorldHint: false }
31214
31730
  }, async (input) => executor.setScheduledActionConnections(input));
31215
31731
  }
31216
- var import_mcp, import_node_fs9, import_node_path13, import_node_crypto12;
31732
+ var import_mcp, import_node_fs9, import_node_path13, import_node_crypto13;
31217
31733
  var init_paa_mcp_server = __esm({
31218
31734
  "src/mcp/paa-mcp-server.ts"() {
31219
31735
  "use strict";
31220
31736
  import_mcp = require("@modelcontextprotocol/sdk/server/mcp.js");
31221
31737
  import_node_fs9 = require("fs");
31222
31738
  import_node_path13 = require("path");
31223
- import_node_crypto12 = require("crypto");
31739
+ import_node_crypto13 = require("crypto");
31224
31740
  init_version();
31225
31741
  init_mcp_response_formatter();
31226
31742
  init_server_instructions();
@@ -31290,10 +31806,11 @@ function withDefaultHostedProfile(input) {
31290
31806
  ...typeof saveProfileChanges === "boolean" ? { saveProfileChanges } : {}
31291
31807
  };
31292
31808
  }
31293
- var HttpMcpToolExecutor;
31809
+ var import_node_crypto14, HttpMcpToolExecutor;
31294
31810
  var init_http_mcp_tool_executor = __esm({
31295
31811
  "src/mcp/http-mcp-tool-executor.ts"() {
31296
31812
  "use strict";
31813
+ import_node_crypto14 = require("crypto");
31297
31814
  init_harvest_timeout();
31298
31815
  init_browser_service_env();
31299
31816
  HttpMcpToolExecutor = class {
@@ -31302,6 +31819,7 @@ var init_http_mcp_tool_executor = __esm({
31302
31819
  timeoutMs;
31303
31820
  httpTimeoutOverrideMs;
31304
31821
  serpIntelligenceTimeoutMs;
31822
+ pendingSerpCaptureBillingKeys = /* @__PURE__ */ new Map();
31305
31823
  constructor(baseUrl, apiKey) {
31306
31824
  this.baseUrl = baseUrl.replace(/\/$/, "");
31307
31825
  this.apiKey = apiKey;
@@ -31312,13 +31830,14 @@ var init_http_mcp_tool_executor = __esm({
31312
31830
  const configuredSerpIntelligenceTimeoutMs = Number(process.env.MCP_SCRAPER_SERP_INTELLIGENCE_HTTP_TIMEOUT_MS ?? this.timeoutMs);
31313
31831
  this.serpIntelligenceTimeoutMs = Number.isFinite(configuredSerpIntelligenceTimeoutMs) && configuredSerpIntelligenceTimeoutMs > 0 ? configuredSerpIntelligenceTimeoutMs : this.timeoutMs;
31314
31832
  }
31315
- async call(path6, body, timeoutMs = this.timeoutMs, method = "POST") {
31833
+ async call(path6, body, timeoutMs = this.timeoutMs, method = "POST", requestHeaders = {}) {
31316
31834
  try {
31317
31835
  const res = await fetch(`${this.baseUrl}${path6}`, {
31318
31836
  method,
31319
31837
  headers: {
31320
31838
  "Content-Type": "application/json",
31321
- "x-api-key": this.apiKey
31839
+ "x-api-key": this.apiKey,
31840
+ ...requestHeaders
31322
31841
  },
31323
31842
  body: JSON.stringify(body),
31324
31843
  signal: AbortSignal.timeout(timeoutMs)
@@ -31566,8 +32085,27 @@ var init_http_mcp_tool_executor = __esm({
31566
32085
  connections: input.connections
31567
32086
  }, void 0, "PUT");
31568
32087
  }
31569
- captureSerpSnapshot(input) {
31570
- return this.call("/serp-intelligence/capture", input, this.serpIntelligenceTimeoutMs);
32088
+ async captureSerpSnapshot(input) {
32089
+ const fingerprint = (0, import_node_crypto14.createHash)("sha256").update(JSON.stringify(input)).digest("hex");
32090
+ const now = Date.now();
32091
+ for (const [pendingFingerprint, pendingEntry] of this.pendingSerpCaptureBillingKeys) {
32092
+ if (pendingEntry.expiresAt <= now) this.pendingSerpCaptureBillingKeys.delete(pendingFingerprint);
32093
+ }
32094
+ const pending = this.pendingSerpCaptureBillingKeys.get(fingerprint);
32095
+ const idempotencyKey = pending && pending.expiresAt > now ? pending.key : (0, import_node_crypto14.randomUUID)();
32096
+ this.pendingSerpCaptureBillingKeys.set(fingerprint, {
32097
+ key: idempotencyKey,
32098
+ expiresAt: now + 15 * 6e4
32099
+ });
32100
+ const result = await this.call(
32101
+ "/serp-intelligence/capture",
32102
+ input,
32103
+ this.serpIntelligenceTimeoutMs,
32104
+ "POST",
32105
+ { "Idempotency-Key": idempotencyKey }
32106
+ );
32107
+ if (!result.isError) this.pendingSerpCaptureBillingKeys.delete(fingerprint);
32108
+ return result;
31571
32109
  }
31572
32110
  captureSerpPageSnapshots(input) {
31573
32111
  return this.call("/serp-intelligence/page-snapshots", input, this.serpIntelligenceTimeoutMs);
@@ -36261,7 +36799,33 @@ var init_mcp_routes = __esm({
36261
36799
  // src/api/browser-agent-db.ts
36262
36800
  async function migrateBrowserAgent() {
36263
36801
  if (_ready2) return;
36802
+ if (_migrationPromise2) return _migrationPromise2;
36803
+ _migrationPromise2 = runBrowserAgentMigration().finally(() => {
36804
+ _migrationPromise2 = null;
36805
+ });
36806
+ return _migrationPromise2;
36807
+ }
36808
+ async function runBrowserAgentMigration() {
36264
36809
  const db = getDb();
36810
+ const existing = await db.execute(`
36811
+ SELECT
36812
+ (SELECT COUNT(*) FROM sqlite_master WHERE type = 'table' AND name IN (
36813
+ 'browser_agent_sessions',
36814
+ 'browser_agent_actions',
36815
+ 'browser_agent_replays',
36816
+ 'browser_auth_connections',
36817
+ 'browser_agent_extensions'
36818
+ )) = 5
36819
+ AND (SELECT COUNT(*) FROM pragma_table_info('browser_agent_sessions') WHERE name = 'concurrency_lock_id') = 1
36820
+ AND (SELECT COUNT(*) FROM pragma_table_info('browser_auth_connections') WHERE name IN (
36821
+ 'account_email', 'note', 'updated_at', 'status', 'browser_agent_session_id'
36822
+ )) = 5
36823
+ AS ready
36824
+ `);
36825
+ if (Number(existing.rows[0]?.ready ?? 0) === 1) {
36826
+ _ready2 = true;
36827
+ return;
36828
+ }
36265
36829
  await db.execute(`
36266
36830
  CREATE TABLE IF NOT EXISTS browser_agent_sessions (
36267
36831
  id TEXT PRIMARY KEY,
@@ -36361,7 +36925,7 @@ async function migrateBrowserAgent() {
36361
36925
  }
36362
36926
  async function createExtensionRow(input) {
36363
36927
  const db = getDb();
36364
- const id = `bext_${(0, import_node_crypto13.randomUUID)().replace(/-/g, "").slice(0, 20)}`;
36928
+ const id = `bext_${(0, import_node_crypto15.randomUUID)().replace(/-/g, "").slice(0, 20)}`;
36365
36929
  await db.execute({
36366
36930
  sql: `INSERT INTO browser_agent_extensions (id, user_id, name, backend_id, backend_name, source, source_url, size_bytes)
36367
36931
  VALUES (?, ?, ?, ?, ?, ?, ?, ?)`,
@@ -36396,7 +36960,7 @@ async function deleteExtensionRow(userId, name) {
36396
36960
  }
36397
36961
  async function createAuthConnectionRow(input) {
36398
36962
  const db = getDb();
36399
- const connectionId = `authc_${(0, import_node_crypto13.randomUUID)().replace(/-/g, "").slice(0, 20)}`;
36963
+ const connectionId = `authc_${(0, import_node_crypto15.randomUUID)().replace(/-/g, "").slice(0, 20)}`;
36400
36964
  await db.execute({
36401
36965
  sql: `INSERT INTO browser_auth_connections (connection_id, domain, profile, account_email, note, status, browser_agent_session_id)
36402
36966
  VALUES (?, ?, ?, ?, ?, 'NEEDS_AUTH', ?)`,
@@ -36430,7 +36994,7 @@ async function listAuthConnectionsByProfile(profile) {
36430
36994
  }
36431
36995
  async function createSessionRow(input) {
36432
36996
  const db = getDb();
36433
- const id = `bas_${(0, import_node_crypto13.randomUUID)().replace(/-/g, "").slice(0, 20)}`;
36997
+ const id = `bas_${(0, import_node_crypto15.randomUUID)().replace(/-/g, "").slice(0, 20)}`;
36434
36998
  await db.execute({
36435
36999
  sql: `INSERT INTO browser_agent_sessions (id, runtime_session_id, live_view_url, cdp_ws_url, status, label, user_id, concurrency_lock_id, last_action_at)
36436
37000
  VALUES (?, ?, ?, ?, 'open', ?, ?, ?, datetime('now'))`,
@@ -36499,7 +37063,7 @@ async function recordAction(input) {
36499
37063
  sql: `INSERT INTO browser_agent_actions (id, session_id, type, params_json, ok, error)
36500
37064
  VALUES (?, ?, ?, ?, ?, ?)`,
36501
37065
  args: [
36502
- `baa_${(0, import_node_crypto13.randomUUID)().replace(/-/g, "").slice(0, 20)}`,
37066
+ `baa_${(0, import_node_crypto15.randomUUID)().replace(/-/g, "").slice(0, 20)}`,
36503
37067
  input.sessionId,
36504
37068
  input.type,
36505
37069
  input.params == null ? null : JSON.stringify(input.params),
@@ -36539,13 +37103,14 @@ async function listReplayRows(sessionId) {
36539
37103
  });
36540
37104
  return res.rows;
36541
37105
  }
36542
- var import_node_crypto13, _ready2;
37106
+ var import_node_crypto15, _ready2, _migrationPromise2;
36543
37107
  var init_browser_agent_db = __esm({
36544
37108
  "src/api/browser-agent-db.ts"() {
36545
37109
  "use strict";
36546
- import_node_crypto13 = require("crypto");
37110
+ import_node_crypto15 = require("crypto");
36547
37111
  init_db();
36548
37112
  _ready2 = false;
37113
+ _migrationPromise2 = null;
36549
37114
  }
36550
37115
  });
36551
37116
 
@@ -38255,7 +38820,7 @@ function buildBrowserAgentRoutes() {
38255
38820
  }
38256
38821
  const existing = await getExtensionRow(user.id, name);
38257
38822
  if (existing) return c.json({ error: `an extension named "${name}" already exists \u2014 delete it first or pick another name` }, 409);
38258
- const backendName = `u${user.id}_${(0, import_node_crypto14.randomUUID)().replace(/-/g, "")}`;
38823
+ const backendName = `u${user.id}_${(0, import_node_crypto16.randomUUID)().replace(/-/g, "")}`;
38259
38824
  try {
38260
38825
  const imported = await importExtensionFromStore(storeUrl, backendName);
38261
38826
  const row = await createExtensionRow({
@@ -38314,11 +38879,11 @@ function buildBrowserAgentRoutes() {
38314
38879
  });
38315
38880
  return app2;
38316
38881
  }
38317
- var import_node_crypto14, import_hono18, auth, DEFAULT_BROWSER_SESSION_LOCK_TTL_SECONDS, EXTENSION_NAME_RE;
38882
+ var import_node_crypto16, import_hono18, auth, DEFAULT_BROWSER_SESSION_LOCK_TTL_SECONDS, EXTENSION_NAME_RE;
38318
38883
  var init_browser_agent_routes = __esm({
38319
38884
  "src/api/browser-agent-routes.ts"() {
38320
38885
  "use strict";
38321
- import_node_crypto14 = require("crypto");
38886
+ import_node_crypto16 = require("crypto");
38322
38887
  import_hono18 = require("hono");
38323
38888
  init_api_auth();
38324
38889
  init_errors();
@@ -38728,7 +39293,7 @@ async function getKeys() {
38728
39293
  const privateKey = await (0, import_jose2.importPKCS8)(pem, "RS256", { extractable: true });
38729
39294
  const full = await (0, import_jose2.exportJWK)(privateKey);
38730
39295
  const publicJwk = { kty: full.kty, n: full.n, e: full.e };
38731
- const kid = (0, import_node_crypto15.createHash)("sha256").update(JSON.stringify({ e: publicJwk.e, kty: publicJwk.kty, n: publicJwk.n })).digest("base64url").slice(0, 16);
39296
+ const kid = (0, import_node_crypto17.createHash)("sha256").update(JSON.stringify({ e: publicJwk.e, kty: publicJwk.kty, n: publicJwk.n })).digest("base64url").slice(0, 16);
38732
39297
  publicJwk.kid = kid;
38733
39298
  publicJwk.alg = "RS256";
38734
39299
  publicJwk.use = "sig";
@@ -38907,23 +39472,23 @@ async function validateAuthRequest(p) {
38907
39472
  }
38908
39473
  function pkceMatches(verifier, challenge) {
38909
39474
  if (!verifier) return false;
38910
- const computed = (0, import_node_crypto15.createHash)("sha256").update(verifier).digest("base64url");
39475
+ const computed = (0, import_node_crypto17.createHash)("sha256").update(verifier).digest("base64url");
38911
39476
  return computed === challenge;
38912
39477
  }
38913
39478
  async function mintAccessToken(identity, scope, plan, audience) {
38914
39479
  const { privateKey, kid } = await getKeys();
38915
- return new import_jose2.SignJWT({ scope, plan }).setProtectedHeader({ alg: "RS256", kid }).setIssuer(ISSUER).setSubject(identity).setAudience(audience).setIssuedAt().setJti((0, import_node_crypto15.randomUUID)()).setExpirationTime(`${ACCESS_TTL_SECONDS}s`).sign(privateKey);
39480
+ return new import_jose2.SignJWT({ scope, plan }).setProtectedHeader({ alg: "RS256", kid }).setIssuer(ISSUER).setSubject(identity).setAudience(audience).setIssuedAt().setJti((0, import_node_crypto17.randomUUID)()).setExpirationTime(`${ACCESS_TTL_SECONDS}s`).sign(privateKey);
38916
39481
  }
38917
39482
  function tokenErrorResponse(c, error, description, status) {
38918
39483
  return c.json({ error, error_description: description }, status);
38919
39484
  }
38920
- var import_hono20, import_cookie, import_node_crypto15, import_jose2, ISSUER, RESOURCE, SCRAPER_RESOURCE, MEMORY_SCOPES, SCRAPER_SCOPES, SUPPORTED_SCOPES, ACCESS_TTL_SECONDS, REFRESH_TTL_SECONDS, CODE_TTL_SECONDS, ROTATION_GRACE_SECONDS, secureCookies, sessionCookieOptions, cachedKeys, oauthApp;
39485
+ var import_hono20, import_cookie, import_node_crypto17, import_jose2, ISSUER, RESOURCE, SCRAPER_RESOURCE, MEMORY_SCOPES, SCRAPER_SCOPES, SUPPORTED_SCOPES, ACCESS_TTL_SECONDS, REFRESH_TTL_SECONDS, CODE_TTL_SECONDS, ROTATION_GRACE_SECONDS, secureCookies, sessionCookieOptions, cachedKeys, oauthApp;
38921
39486
  var init_oauth_routes = __esm({
38922
39487
  "src/api/oauth-routes.ts"() {
38923
39488
  "use strict";
38924
39489
  import_hono20 = require("hono");
38925
39490
  import_cookie = require("hono/cookie");
38926
- import_node_crypto15 = require("crypto");
39491
+ import_node_crypto17 = require("crypto");
38927
39492
  import_jose2 = require("jose");
38928
39493
  init_session();
38929
39494
  init_db();
@@ -39009,7 +39574,7 @@ var init_oauth_routes = __esm({
39009
39574
  }
39010
39575
  }
39011
39576
  const clientName = typeof body.client_name === "string" ? body.client_name : null;
39012
- const clientId = `client_${(0, import_node_crypto15.randomBytes)(16).toString("hex")}`;
39577
+ const clientId = `client_${(0, import_node_crypto17.randomBytes)(16).toString("hex")}`;
39013
39578
  await registerClient(clientId, redirectUris, clientName);
39014
39579
  console.log("[oauth-dcr] register OK client_id=%s redirect_uris=%s", clientId, JSON.stringify(redirectUris));
39015
39580
  return c.json({
@@ -39062,7 +39627,7 @@ var init_oauth_routes = __esm({
39062
39627
  if (action === "deny") return redirectWithError(p.redirect_uri, p.state, "access_denied");
39063
39628
  if (action !== "approve") return c.text("unsupported action", 400);
39064
39629
  const scope = negotiateScope(p.scope, user, p.resource);
39065
- const code = `code_${(0, import_node_crypto15.randomBytes)(32).toString("base64url")}`;
39630
+ const code = `code_${(0, import_node_crypto17.randomBytes)(32).toString("base64url")}`;
39066
39631
  const expiresAt = new Date(Date.now() + CODE_TTL_SECONDS * 1e3).toISOString();
39067
39632
  await putCode({
39068
39633
  code,
@@ -39101,7 +39666,7 @@ var init_oauth_routes = __esm({
39101
39666
  const plan = user ? resolvePlan(user) : "free";
39102
39667
  const audience = record.resource ?? RESOURCE();
39103
39668
  const accessToken = await mintAccessToken(record.identity, record.scope, plan, audience);
39104
- const refreshToken = `rt_${(0, import_node_crypto15.randomBytes)(40).toString("base64url")}`;
39669
+ const refreshToken = `rt_${(0, import_node_crypto17.randomBytes)(40).toString("base64url")}`;
39105
39670
  await putRefresh({
39106
39671
  refresh_token: refreshToken,
39107
39672
  client_id: clientId,
@@ -39131,7 +39696,7 @@ var init_oauth_routes = __esm({
39131
39696
  const plan = user ? resolvePlan(user) : "free";
39132
39697
  const audience = record.resource ?? RESOURCE();
39133
39698
  const accessToken = await mintAccessToken(record.identity, record.scope, plan, audience);
39134
- const nextRefresh = `rt_${(0, import_node_crypto15.randomBytes)(40).toString("base64url")}`;
39699
+ const nextRefresh = `rt_${(0, import_node_crypto17.randomBytes)(40).toString("base64url")}`;
39135
39700
  await rotateRefresh(refreshToken, {
39136
39701
  refresh_token: nextRefresh,
39137
39702
  client_id: record.client_id,
@@ -40448,7 +41013,7 @@ var init_site_audit_worker = __esm({
40448
41013
 
40449
41014
  // src/api/page-diff.ts
40450
41015
  function sha256Hex(value) {
40451
- return (0, import_node_crypto16.createHash)("sha256").update(value).digest("hex");
41016
+ return (0, import_node_crypto18.createHash)("sha256").update(value).digest("hex");
40452
41017
  }
40453
41018
  function truncateForStorage(value, maxChars = MAX_SNAPSHOT_CONTENT_CHARS) {
40454
41019
  if (value.length <= maxChars) return { value, truncated: false };
@@ -40505,11 +41070,11 @@ function diffPageContent(oldContent, newContent) {
40505
41070
  totalChangedLineCount
40506
41071
  };
40507
41072
  }
40508
- var import_node_crypto16, import_diff, MAX_SNAPSHOT_CONTENT_CHARS, MAX_DIFF_HUNKS, MAX_DIFF_LINES_PER_RESPONSE;
41073
+ var import_node_crypto18, import_diff, MAX_SNAPSHOT_CONTENT_CHARS, MAX_DIFF_HUNKS, MAX_DIFF_LINES_PER_RESPONSE;
40509
41074
  var init_page_diff = __esm({
40510
41075
  "src/api/page-diff.ts"() {
40511
41076
  "use strict";
40512
- import_node_crypto16 = require("crypto");
41077
+ import_node_crypto18 = require("crypto");
40513
41078
  import_diff = require("diff");
40514
41079
  MAX_SNAPSHOT_CONTENT_CHARS = 25e4;
40515
41080
  MAX_DIFF_HUNKS = 200;
@@ -40586,7 +41151,7 @@ function canonicalJson(value) {
40586
41151
  return JSON.stringify(value);
40587
41152
  }
40588
41153
  function sha2562(value) {
40589
- return (0, import_node_crypto17.createHash)("sha256").update(value).digest("hex");
41154
+ return (0, import_node_crypto19.createHash)("sha256").update(value).digest("hex");
40590
41155
  }
40591
41156
  function sensitiveKey(key) {
40592
41157
  const normalized = key.toLowerCase().replace(/[^a-z0-9]/g, "");
@@ -40801,11 +41366,11 @@ async function importServiceConnectionToMemory(identity, input, dependencies) {
40801
41366
  ...!searchReady ? { warning: "The snapshot was stored, but no search chunks were indexed yet." } : {}
40802
41367
  };
40803
41368
  }
40804
- var import_node_crypto17, CONNECTION_MEMORY_IMPORT_MAX_ARGS_BYTES, CONNECTION_MEMORY_IMPORT_MAX_RESULT_BYTES, CONNECTION_MEMORY_IMPORT_MAX_STRING_CHARS, CONNECTION_MEMORY_IMPORT_MAX_DEPTH, ConnectionMemoryImportError;
41369
+ var import_node_crypto19, CONNECTION_MEMORY_IMPORT_MAX_ARGS_BYTES, CONNECTION_MEMORY_IMPORT_MAX_RESULT_BYTES, CONNECTION_MEMORY_IMPORT_MAX_STRING_CHARS, CONNECTION_MEMORY_IMPORT_MAX_DEPTH, ConnectionMemoryImportError;
40805
41370
  var init_connection_memory_import = __esm({
40806
41371
  "src/api/connection-memory-import.ts"() {
40807
41372
  "use strict";
40808
- import_node_crypto17 = require("crypto");
41373
+ import_node_crypto19 = require("crypto");
40809
41374
  init_slugify();
40810
41375
  CONNECTION_MEMORY_IMPORT_MAX_ARGS_BYTES = 64 * 1024;
40811
41376
  CONNECTION_MEMORY_IMPORT_MAX_RESULT_BYTES = 1e6;
@@ -40979,7 +41544,7 @@ function finitePositive(value, fallback) {
40979
41544
  return Number.isFinite(value) && value > 0 ? value : fallback;
40980
41545
  }
40981
41546
  async function collectConnectedDataExport(args) {
40982
- const exportId = (0, import_node_crypto18.randomUUID)();
41547
+ const exportId = (0, import_node_crypto20.randomUUID)();
40983
41548
  const now = args.now ?? Date.now;
40984
41549
  const startedAt = now();
40985
41550
  const budgetMs = finitePositive(CONNECTED_DATA_EXPORT_BUDGET_MS, 24e4);
@@ -41089,11 +41654,11 @@ ${lines.length ? `${lines.join("\n")}
41089
41654
  untrustedContent: true
41090
41655
  };
41091
41656
  }
41092
- var import_node_crypto18, CONNECTED_DATA_INLINE_BUDGET_BYTES, CONNECTED_DATA_MAX_EXPORT_BYTES, CONNECTED_DATA_EXPORT_BUDGET_MS, CONNECTED_DATA_PAGE_START_HEADROOM_MS, CONNECTED_DATA_DATASETS, ConnectedDataExportValidationError;
41657
+ var import_node_crypto20, CONNECTED_DATA_INLINE_BUDGET_BYTES, CONNECTED_DATA_MAX_EXPORT_BYTES, CONNECTED_DATA_EXPORT_BUDGET_MS, CONNECTED_DATA_PAGE_START_HEADROOM_MS, CONNECTED_DATA_DATASETS, ConnectedDataExportValidationError;
41093
41658
  var init_connected_data_export = __esm({
41094
41659
  "src/api/connected-data-export.ts"() {
41095
41660
  "use strict";
41096
- import_node_crypto18 = require("crypto");
41661
+ import_node_crypto20 = require("crypto");
41097
41662
  CONNECTED_DATA_INLINE_BUDGET_BYTES = Number(
41098
41663
  process.env.MCP_SCRAPER_CONNECTED_DATA_INLINE_BUDGET_BYTES ?? 5e4
41099
41664
  );
@@ -41131,12 +41696,6 @@ var init_connected_data_export = __esm({
41131
41696
  });
41132
41697
 
41133
41698
  // src/api/credit-operations.ts
41134
- async function grantSignupCredit(userId) {
41135
- if (!FREE_SIGNUP_MC || FREE_SIGNUP_MC <= 0) return;
41136
- const already = await ledgerExistsForOperation(userId, LedgerOperation.SIGNUP_GRANT);
41137
- if (already) return;
41138
- await creditMc(userId, FREE_SIGNUP_MC, LedgerOperation.SIGNUP_GRANT, "Welcome credits");
41139
- }
41140
41699
  async function applyMonthlyFreeRefresh(user) {
41141
41700
  return user;
41142
41701
  }
@@ -41903,7 +42462,7 @@ function canonicalJson2(value) {
41903
42462
  return JSON.stringify(value);
41904
42463
  }
41905
42464
  function projectedToolSchemaHash(tool) {
41906
- return (0, import_node_crypto19.createHash)("sha256").update(canonicalJson2(tool)).digest("hex");
42465
+ return (0, import_node_crypto21.createHash)("sha256").update(canonicalJson2(tool)).digest("hex");
41907
42466
  }
41908
42467
  async function describeNangoTool(identity, connectionId, tool, fresh) {
41909
42468
  const body = await controlRequest("/api/internal/nango/connections/actions/describe", {
@@ -42033,11 +42592,11 @@ async function callScheduleConnectionExportPage(identity, input) {
42033
42592
  untrustedContent: true
42034
42593
  };
42035
42594
  }
42036
- var import_node_crypto19, DEFAULT_NANGO_CONTROL_URL, DISABLED_NANGO_TOOLS, CONNECTION_SYNC_REQUIRED_TOOLS, CONNECTION_SYNC_OPTIONAL_TOOLS, NangoControlError, ScheduleConnectionValidationError, SAFE_CONTROL_ERROR_CODES, CONTROL_ERROR_CODE_ALIASES;
42595
+ var import_node_crypto21, DEFAULT_NANGO_CONTROL_URL, DISABLED_NANGO_TOOLS, CONNECTION_SYNC_REQUIRED_TOOLS, CONNECTION_SYNC_OPTIONAL_TOOLS, NangoControlError, ScheduleConnectionValidationError, SAFE_CONTROL_ERROR_CODES, CONTROL_ERROR_CODE_ALIASES;
42037
42596
  var init_nango_control = __esm({
42038
42597
  "src/api/nango-control.ts"() {
42039
42598
  "use strict";
42040
- import_node_crypto19 = require("crypto");
42599
+ import_node_crypto21 = require("crypto");
42041
42600
  init_connected_data_export();
42042
42601
  DEFAULT_NANGO_CONTROL_URL = "https://mcp-scraper-scheduler.vercel.app";
42043
42602
  DISABLED_NANGO_TOOLS = {
@@ -42525,7 +43084,7 @@ async function processJob(job) {
42525
43084
  if (diff > 0) await creditMc(job.user_id, diff, "paa_refund", "overestimate refund");
42526
43085
  else if (diff < 0) await debitMc(job.user_id, -diff, "paa", opts.query ?? job.query);
42527
43086
  } else if (opts.serpOnly && typeof opts.billingHoldMc === "number") {
42528
- const actualCost = headlessSentOut.value === true ? MC_COSTS.serp_headless : MC_COSTS.serp_headful;
43087
+ const actualCost = serpActualCostMc(headlessSentOut.value);
42529
43088
  const diff = opts.billingHoldMc - actualCost;
42530
43089
  if (diff > 0) await creditMc(job.user_id, diff, "serp_refund", "headless-mode pricing settle");
42531
43090
  else if (diff < 0) await debitMc(job.user_id, -diff, "serp", opts.query ?? job.query);
@@ -42882,7 +43441,6 @@ var init_server = __esm({
42882
43441
  init_browser_agent_console();
42883
43442
  init_browser_agent_db();
42884
43443
  init_cost_context();
42885
- init_cost_telemetry();
42886
43444
  init_stripe_routes();
42887
43445
  init_oauth_routes();
42888
43446
  init_chat_routes();
@@ -42988,7 +43546,6 @@ var init_server = __esm({
42988
43546
  const forceHeadless = c.req.header("x-kernel-headless") === "true";
42989
43547
  return runWithCostContext({ op, probeRunId, forceHeadless }, () => next());
42990
43548
  });
42991
- void migrateCostTelemetry();
42992
43549
  app.post("/auth/register", requireAllowedOrigin, async (c) => {
42993
43550
  const { email, password } = await c.req.json();
42994
43551
  const normalizedEmail = email?.trim().toLowerCase();
@@ -43019,7 +43576,6 @@ var init_server = __esm({
43019
43576
  }
43020
43577
  }
43021
43578
  const newKey = await rotateApiKey(user.id);
43022
- await grantSignupCredit(user.id);
43023
43579
  void (async () => {
43024
43580
  try {
43025
43581
  const full = await getUserById(user.id);
@@ -44112,7 +44668,7 @@ var init_server = __esm({
44112
44668
  if (diff > 0) await creditMc(user.id, diff, LedgerOperation.PAA_REFUND, "overestimate refund");
44113
44669
  else if (diff < 0) await debitMc(user.id, -diff, LedgerOperation.PAA, options.query);
44114
44670
  } else {
44115
- const actualCost = headlessSentOut.value === true ? MC_COSTS.serp_headless : MC_COSTS.serp_headful;
44671
+ const actualCost = serpActualCostMc(headlessSentOut.value);
44116
44672
  const diff = syncCost - actualCost;
44117
44673
  if (diff > 0) await creditMc(user.id, diff, LedgerOperation.SERP_REFUND, "headless-mode pricing settle");
44118
44674
  else if (diff < 0) await debitMc(user.id, -diff, LedgerOperation.SERP, options.query);
@@ -44191,22 +44747,13 @@ var init_server = __esm({
44191
44747
  });
44192
44748
  app.post("/admin/backfill-signup-credits", adminAuth, async (c) => {
44193
44749
  const users = await listUsers();
44194
- let processed = 0;
44195
- let credited = 0;
44196
- let skipped = 0;
44197
- const users_credited = [];
44198
- for (const user of users) {
44199
- processed++;
44200
- const alreadyHasGrant = await ledgerExistsForOperation(user.id, LedgerOperation.SIGNUP_GRANT);
44201
- if (alreadyHasGrant) {
44202
- skipped++;
44203
- } else {
44204
- await grantSignupCredit(user.id);
44205
- credited++;
44206
- users_credited.push({ id: user.id, email: user.email });
44207
- }
44208
- }
44209
- return c.json({ processed, credited, skipped, users_credited });
44750
+ return c.json({
44751
+ processed: users.length,
44752
+ credited: 0,
44753
+ skipped: users.length,
44754
+ users_credited: [],
44755
+ retired: true
44756
+ });
44210
44757
  });
44211
44758
  app.post("/extract-url", auth2, async (c) => {
44212
44759
  const raw = await c.req.json().catch(() => ({}));