mcp-scraper 0.3.24 → 0.3.25

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (65) hide show
  1. package/dist/bin/api-server.cjs +8140 -5941
  2. package/dist/bin/api-server.cjs.map +1 -1
  3. package/dist/bin/api-server.js +3 -3
  4. package/dist/bin/mcp-scraper-cli.cjs +320 -58
  5. package/dist/bin/mcp-scraper-cli.cjs.map +1 -1
  6. package/dist/bin/mcp-scraper-cli.js +4 -3
  7. package/dist/bin/mcp-scraper-cli.js.map +1 -1
  8. package/dist/bin/mcp-scraper-install.cjs +3 -3
  9. package/dist/bin/mcp-scraper-install.cjs.map +1 -1
  10. package/dist/bin/mcp-scraper-install.js +2 -2
  11. package/dist/bin/mcp-stdio-server.cjs +121 -24
  12. package/dist/bin/mcp-stdio-server.cjs.map +1 -1
  13. package/dist/bin/mcp-stdio-server.js +5 -5
  14. package/dist/bin/paa-harvest.cjs +284 -10
  15. package/dist/bin/paa-harvest.cjs.map +1 -1
  16. package/dist/bin/paa-harvest.js +5 -3
  17. package/dist/bin/paa-harvest.js.map +1 -1
  18. package/dist/{chunk-ZAP7MYVO.js → chunk-3PRO376E.js} +2 -3
  19. package/dist/chunk-3PRO376E.js.map +1 -0
  20. package/dist/{chunk-LFATOGDF.js → chunk-BP27CZ5Q.js} +4 -3
  21. package/dist/chunk-BP27CZ5Q.js.map +1 -0
  22. package/dist/{chunk-DE7WP4UU.js → chunk-DOBQN3EY.js} +3 -3
  23. package/dist/chunk-DOBQN3EY.js.map +1 -0
  24. package/dist/{chunk-GZ4W72WF.js → chunk-FE2WC4JR.js} +119 -23
  25. package/dist/chunk-FE2WC4JR.js.map +1 -0
  26. package/dist/{chunk-IGOEAKC7.js → chunk-G7PQ64LM.js} +274 -9
  27. package/dist/chunk-G7PQ64LM.js.map +1 -0
  28. package/dist/chunk-K443GQY5.js +24 -0
  29. package/dist/chunk-K443GQY5.js.map +1 -0
  30. package/dist/{chunk-H2R232HK.js → chunk-NNEIXK5L.js} +307 -62
  31. package/dist/chunk-NNEIXK5L.js.map +1 -0
  32. package/dist/{chunk-3KYRG7O7.js → chunk-TRNSXR46.js} +185 -2
  33. package/dist/chunk-TRNSXR46.js.map +1 -0
  34. package/dist/chunk-WPHAMNT7.js +7 -0
  35. package/dist/chunk-WPHAMNT7.js.map +1 -0
  36. package/dist/{chunk-JTWQGPKF.js → chunk-WYCER2HW.js} +33 -10
  37. package/dist/chunk-WYCER2HW.js.map +1 -0
  38. package/dist/{db-EG5ETPTY.js → db-FP2ABUU4.js} +32 -2
  39. package/dist/index.cjs +284 -10
  40. package/dist/index.cjs.map +1 -1
  41. package/dist/index.d.cts +7 -3
  42. package/dist/index.d.ts +7 -3
  43. package/dist/index.js +5 -3
  44. package/dist/index.js.map +1 -1
  45. package/dist/{server-2SRARAO5.js → server-RRCKVBHG.js} +1563 -265
  46. package/dist/server-RRCKVBHG.js.map +1 -0
  47. package/dist/{worker-UM5UM6O5.js → worker-UC6D2756.js} +7 -6
  48. package/dist/{worker-UM5UM6O5.js.map → worker-UC6D2756.js.map} +1 -1
  49. package/docs/kernel-proxy-future-enhancements.md +80 -0
  50. package/docs/spec-browser-routing.md +108 -0
  51. package/docs/spec-mcp-tools.md +146 -0
  52. package/docs/specs/audit-visual-demo.md +309 -0
  53. package/package.json +5 -2
  54. package/dist/chunk-3KYRG7O7.js.map +0 -1
  55. package/dist/chunk-DE7WP4UU.js.map +0 -1
  56. package/dist/chunk-GZ4W72WF.js.map +0 -1
  57. package/dist/chunk-H2R232HK.js.map +0 -1
  58. package/dist/chunk-H7R4PPMG.js +0 -7
  59. package/dist/chunk-H7R4PPMG.js.map +0 -1
  60. package/dist/chunk-IGOEAKC7.js.map +0 -1
  61. package/dist/chunk-JTWQGPKF.js.map +0 -1
  62. package/dist/chunk-LFATOGDF.js.map +0 -1
  63. package/dist/chunk-ZAP7MYVO.js.map +0 -1
  64. package/dist/server-2SRARAO5.js.map +0 -1
  65. /package/dist/{db-EG5ETPTY.js.map → db-FP2ABUU4.js.map} +0 -0
@@ -21,26 +21,30 @@ import {
21
21
  renderLinkReport,
22
22
  sanitizeAttempts,
23
23
  sanitizeHarvestResult
24
- } from "./chunk-GZ4W72WF.js";
24
+ } from "./chunk-FE2WC4JR.js";
25
25
  import {
26
26
  csvRecords,
27
27
  listWorkflowDefinitions,
28
28
  rowsToCsv,
29
29
  runWorkflow,
30
30
  runWorkflowStep,
31
+ slugify,
31
32
  workflowDefinition,
32
33
  workflowOutputBaseDir,
33
34
  workflowStepCount,
34
35
  workflowSupportsSteps
35
- } from "./chunk-H2R232HK.js";
36
- import "./chunk-H7R4PPMG.js";
36
+ } from "./chunk-NNEIXK5L.js";
37
+ import "./chunk-WPHAMNT7.js";
37
38
  import {
38
39
  BROWSER_OPEN_MIN_BALANCE_MC,
39
40
  CONCURRENCY_PRICE_ID,
40
41
  CREDIT_COST_CATALOG,
42
+ FREE_SIGNUP_MC,
41
43
  LedgerOperation,
42
44
  MC_COSTS,
43
45
  MC_PER_CREDIT,
46
+ MEMORY_PLANS,
47
+ MEMORY_PLAN_QUOTA,
44
48
  SUBSCRIPTION_TIERS,
45
49
  SUBSCRIPTION_TIER_BY_KEY,
46
50
  browserActiveCostMc,
@@ -50,15 +54,20 @@ import {
50
54
  harvestProblemResponse,
51
55
  insufficientBalanceResponse,
52
56
  serializeHarvestProblem
53
- } from "./chunk-JTWQGPKF.js";
57
+ } from "./chunk-WYCER2HW.js";
54
58
  import {
55
59
  BrowserDriver,
56
60
  MapsSelectors,
57
61
  buildYouTubeChannelVideosUrl,
62
+ currentCostContext,
58
63
  deleteKernelProxyId,
59
64
  harvest,
60
- resolveKernelProxyId
61
- } from "./chunk-IGOEAKC7.js";
65
+ migrateCostTelemetry,
66
+ recordKernelSession,
67
+ recordVendorUsage,
68
+ resolveKernelProxyId,
69
+ runWithCostContext
70
+ } from "./chunk-G7PQ64LM.js";
62
71
  import {
63
72
  CaptchaError,
64
73
  RECAPTCHA_INSTRUCTIONS,
@@ -67,7 +76,8 @@ import {
67
76
  browserServiceProfileSaveChanges,
68
77
  browserServiceProxyId,
69
78
  sanitizeVendorName
70
- } from "./chunk-ZAP7MYVO.js";
79
+ } from "./chunk-3PRO376E.js";
80
+ import "./chunk-K443GQY5.js";
71
81
  import {
72
82
  DEFAULT_MAPS_PROXY_MODE,
73
83
  DEFAULT_PROXY_MODE,
@@ -77,7 +87,7 @@ import {
77
87
  RawMapsHoursRowSchema,
78
88
  RawMapsOverviewSchema,
79
89
  RawMapsReviewStatsSchema
80
- } from "./chunk-LFATOGDF.js";
90
+ } from "./chunk-BP27CZ5Q.js";
81
91
  import {
82
92
  SiteAuditJobRowSchema,
83
93
  advanceWorkflowRunStep,
@@ -86,6 +96,7 @@ import {
86
96
  checkRateLimit,
87
97
  completeJob,
88
98
  completeWorkflowRunRecord,
99
+ consumeCode,
89
100
  consumePasswordResetToken,
90
101
  countActiveUsers,
91
102
  createJob,
@@ -101,13 +112,18 @@ import {
101
112
  expireOldLots,
102
113
  failJob,
103
114
  failWorkflowRunRecord,
115
+ getClient,
104
116
  getDb,
105
117
  getJob,
106
118
  getLedger,
119
+ getMemoryPlan,
120
+ getMemoryProvisioned,
121
+ getRefresh,
107
122
  getUserByApiKey,
108
123
  getUserByEmail,
109
124
  getUserById,
110
125
  getUserByStripeCustomerId,
126
+ getUserMemoryCreds,
111
127
  getUserStats,
112
128
  getWorkflowArtifact,
113
129
  getWorkflowRun,
@@ -127,19 +143,27 @@ import {
127
143
  markWorkflowRunRunning,
128
144
  markWorkflowScheduleRan,
129
145
  patchWorkflowSchedule,
146
+ putCode,
147
+ putRefresh,
130
148
  reconcileBalanceMc,
131
149
  recordStripeEvent,
132
150
  recordWorkflowWebhookDelivery,
151
+ registerClient,
133
152
  replaceWorkflowArtifacts,
153
+ resolveUserByMemorySubscription,
134
154
  revokeApiKey,
135
155
  rotateApiKey,
156
+ rotateRefresh,
136
157
  setConcurrencySubId,
137
158
  setExtraConcurrencySlots,
159
+ setMemoryPlan,
160
+ setMemoryProvisioned,
138
161
  setPassword,
139
162
  setStripeCustomerId,
140
163
  setSubscriptionTier,
164
+ setUserMemoryCreds,
141
165
  verifyPassword
142
- } from "./chunk-3KYRG7O7.js";
166
+ } from "./chunk-TRNSXR46.js";
143
167
 
144
168
  // src/blog/registry.ts
145
169
  var posts = [
@@ -3511,7 +3535,9 @@ async function fetchWithKernel(url) {
3511
3535
  const apiKey = browserServiceApiKey();
3512
3536
  if (!apiKey) throw new Error("Browser backend API key not set");
3513
3537
  const client2 = new Kernel({ apiKey });
3514
- const kb = await client2.browsers.create({ stealth: true, timeout_seconds: 60 });
3538
+ const openedAtMs = Date.now();
3539
+ const useHeadless = currentCostContext()?.forceHeadful !== true;
3540
+ const kb = await client2.browsers.create({ stealth: true, ...useHeadless ? { headless: true } : {}, timeout_seconds: 60 });
3515
3541
  const browser = await chromium.connectOverCDP(kb.cdp_ws_url);
3516
3542
  try {
3517
3543
  const context = browser.contexts()[0] ?? await browser.newContext({
@@ -3526,6 +3552,7 @@ async function fetchWithKernel(url) {
3526
3552
  });
3527
3553
  await client2.browsers.deleteByID(kb.session_id).catch(() => {
3528
3554
  });
3555
+ void recordKernelSession({ kernelSessionId: kb.session_id, source: "kernel_fetch", stealth: typeof kb.stealth === "boolean" ? kb.stealth : null, headlessSent: useHeadless ? true : null, proxyUsed: typeof kb.proxy_id === "string" ? Boolean(kb.proxy_id) : null, fallback: true, openedAtMs, closedAtMs: Date.now() });
3529
3556
  }
3530
3557
  }
3531
3558
 
@@ -4127,6 +4154,7 @@ async function captureLocal(url, device, wantScreenshot, wantBranding) {
4127
4154
  }
4128
4155
  async function captureKernel(url, kernelApiKey, device, wantScreenshot, wantBranding) {
4129
4156
  const kernel = new Kernel2({ apiKey: kernelApiKey });
4157
+ const openedAtMs = Date.now();
4130
4158
  const session = await kernel.browsers.create({ stealth: true, timeout_seconds: 240 });
4131
4159
  const browser = await chromium2.connectOverCDP(session.cdp_ws_url);
4132
4160
  const vp = device === "mobile" ? MOBILE_VIEWPORT : DESKTOP_VIEWPORT;
@@ -4146,6 +4174,7 @@ async function captureKernel(url, kernelApiKey, device, wantScreenshot, wantBran
4146
4174
  return result;
4147
4175
  } finally {
4148
4176
  await browser.close();
4177
+ void recordKernelSession({ kernelSessionId: session.session_id, source: "screenshot", stealth: typeof session.stealth === "boolean" ? session.stealth : null, headlessSent: null, proxyUsed: typeof session.proxy_id === "string" ? Boolean(session.proxy_id) : null, openedAtMs, closedAtMs: Date.now() });
4149
4178
  }
4150
4179
  }
4151
4180
  async function captureScreenshot(url, kernelApiKey, device = "desktop") {
@@ -4800,14 +4829,19 @@ async function crawlWithRotation(urls, opts) {
4800
4829
  let kb = null;
4801
4830
  let browser = null;
4802
4831
  let inBrowser = 0;
4832
+ let kbOpenedAtMs = 0;
4803
4833
  const retire = () => {
4804
- const b = browser, k = kb;
4834
+ const b = browser, k = kb, openedAtMs = kbOpenedAtMs;
4805
4835
  browser = null;
4806
4836
  kb = null;
4837
+ kbOpenedAtMs = 0;
4807
4838
  if (b) void b.close().catch(() => {
4808
4839
  });
4809
- if (k) void client2.browsers.deleteByID(k.session_id).catch(() => {
4810
- });
4840
+ if (k) {
4841
+ void client2.browsers.deleteByID(k.session_id).catch(() => {
4842
+ });
4843
+ void recordKernelSession({ kernelSessionId: k.session_id, source: "rotating_crawl", stealth: typeof k.stealth === "boolean" ? k.stealth : null, headlessSent: headless, proxyUsed: typeof k.proxy_id === "string" ? Boolean(k.proxy_id) : null, openedAtMs, closedAtMs: Date.now() });
4844
+ }
4811
4845
  };
4812
4846
  try {
4813
4847
  while (true) {
@@ -4819,6 +4853,7 @@ async function crawlWithRotation(urls, opts) {
4819
4853
  if (!browser || inBrowser >= urlsPerBrowser) {
4820
4854
  retire();
4821
4855
  kb = await createBrowserWithRetry(client2, headless);
4856
+ kbOpenedAtMs = Date.now();
4822
4857
  browser = await withTimeout(chromium3.connectOverCDP(kb.cdp_ws_url), 3e4);
4823
4858
  inBrowser = 0;
4824
4859
  }
@@ -5226,7 +5261,7 @@ async function extractSite(opts) {
5226
5261
  }
5227
5262
 
5228
5263
  // src/api/server.ts
5229
- import { Hono as Hono13 } from "hono";
5264
+ import { Hono as Hono15 } from "hono";
5230
5265
  import { serve as serveInngest } from "inngest/hono";
5231
5266
 
5232
5267
  // src/inngest/client.ts
@@ -5301,6 +5336,7 @@ var DeepInfraLlmClient = class {
5301
5336
  });
5302
5337
  if (!resp.ok) throw new Error(`DeepInfra error ${resp.status}: ${await resp.text()}`);
5303
5338
  const data = await resp.json();
5339
+ void recordVendorUsage({ vendor: "deepinfra_qwen", model: this.model, units: data.usage?.completion_tokens ?? 0, unitType: "tokens" });
5304
5340
  return data.choices[0].message.content;
5305
5341
  }
5306
5342
  async completeJson(prompt) {
@@ -8924,6 +8960,8 @@ async function attemptKernelWhisper(videoId, kernelApiKey, kernelProxyId, falKey
8924
8960
  timestamp: c.timestamp,
8925
8961
  text: c.text.trim()
8926
8962
  }));
8963
+ const audioSec = chunks.length ? chunks[chunks.length - 1]?.timestamp?.[1] ?? 0 : 0;
8964
+ void recordVendorUsage({ vendor: "fal_wizper", model: "fal-ai/wizper", units: audioSec / 60, unitType: "audio_min" });
8927
8965
  const text = data.text ?? chunks.map((c) => c.text).join(" ");
8928
8966
  if (!text) return null;
8929
8967
  return { videoId, text, chunks, durationMs: Date.now() - start, method: "browser-whisper" };
@@ -9007,7 +9045,9 @@ var ExtractUrlBodySchema = z11.object({
9007
9045
  extractBranding: z11.boolean().optional(),
9008
9046
  downloadMedia: z11.boolean().optional(),
9009
9047
  mediaTypes: z11.array(z11.enum(["image", "video", "audio"])).optional(),
9010
- allowLocal: z11.boolean().optional()
9048
+ allowLocal: z11.boolean().optional(),
9049
+ depositToVault: z11.boolean().optional(),
9050
+ vaultName: z11.string().trim().min(1).max(120).optional()
9011
9051
  });
9012
9052
  var MapUrlsBodySchema = z11.object({
9013
9053
  url: z11.string().min(1, "url is required"),
@@ -10351,6 +10391,8 @@ async function transcribeMediaUrl(mediaUrl, markdownTitle = "# Media Transcript"
10351
10391
  const text = data.text ?? "";
10352
10392
  const chunks = data.chunks ?? [];
10353
10393
  const durationMs = Date.now() - startMs;
10394
+ const audioSec = chunks.length ? chunks[chunks.length - 1]?.timestamp?.[1] ?? 0 : 0;
10395
+ void recordVendorUsage({ vendor: "fal_wizper", model: "fal-ai/wizper", units: audioSec / 60, unitType: "audio_min" });
10354
10396
  return {
10355
10397
  text,
10356
10398
  chunks,
@@ -10410,21 +10452,24 @@ function buildPageIntelUrl(body, country) {
10410
10452
  if (body.pageId?.trim()) return `https://www.facebook.com/ads/library/?active_status=all&ad_type=all&country=${country}&is_targeted_country=false&media_type=all&search_type=page&view_all_page_id=${body.pageId.trim()}`;
10411
10453
  return `https://www.facebook.com/ads/library/?active_status=all&ad_type=all&country=${country}&q=${encodeURIComponent(body.query.trim())}&search_type=keyword_unordered`;
10412
10454
  }
10455
+ function configuredKernelProxy() {
10456
+ return (process.env.BROWSER_SERVICE_PROXY_ID ?? process.env.KERNEL_PROXY_ID)?.trim() || void 0;
10457
+ }
10413
10458
  function kernelLaunchOpts() {
10414
- return { headless: true, kernelApiKey: browserServiceApiKey(), kernelProxyId: browserServiceProxyId(), viewport: { width: 1280, height: 900 }, locale: "en-US" };
10459
+ return { headless: true, kernelApiKey: browserServiceApiKey(), kernelProxyId: configuredKernelProxy(), viewport: { width: 1280, height: 900 }, locale: "en-US" };
10415
10460
  }
10416
10461
  async function kernelLaunchOptsResidential() {
10417
- let proxyId = browserServiceProxyId();
10462
+ let proxyId = configuredKernelProxy();
10418
10463
  try {
10419
10464
  const resolution = await resolveKernelProxyId({
10420
10465
  kernelApiKey: browserServiceApiKey(),
10421
10466
  proxyMode: DEFAULT_PROXY_MODE,
10422
- configuredKernelProxyId: browserServiceProxyId(),
10467
+ configuredKernelProxyId: configuredKernelProxy(),
10423
10468
  gl: "us"
10424
10469
  });
10425
10470
  if (resolution.kernelProxyId) proxyId = resolution.kernelProxyId;
10426
10471
  } catch {
10427
- proxyId = browserServiceProxyId();
10472
+ proxyId = configuredKernelProxy();
10428
10473
  }
10429
10474
  return { headless: true, kernelApiKey: browserServiceApiKey(), kernelProxyId: proxyId, viewport: { width: 1280, height: 900 }, locale: "en-US" };
10430
10475
  }
@@ -11162,10 +11207,14 @@ function resolveProfileName(body) {
11162
11207
  const explicit = body.profile?.trim();
11163
11208
  return explicit || browserServiceProfileName();
11164
11209
  }
11210
+ function configuredKernelProxy2() {
11211
+ return (process.env.BROWSER_SERVICE_PROXY_ID ?? process.env.KERNEL_PROXY_ID)?.trim() || void 0;
11212
+ }
11165
11213
  async function kernelLaunchOptsDirect(profileName, saveProfileChanges) {
11166
11214
  return {
11167
11215
  headless: true,
11168
11216
  kernelApiKey: browserServiceApiKey(),
11217
+ kernelProxyId: configuredKernelProxy2(),
11169
11218
  ...profileName ? { kernelProfileName: profileName } : {},
11170
11219
  ...typeof saveProfileChanges === "boolean" ? { kernelProfileSaveChanges: saveProfileChanges } : {},
11171
11220
  viewport: { width: 1280, height: 900 },
@@ -11420,8 +11469,136 @@ instagramApp.post("/media-download", createApiKeyAuth(), async (c) => {
11420
11469
  }
11421
11470
  });
11422
11471
 
11423
- // src/api/maps-routes.ts
11472
+ // src/api/reddit-routes.ts
11424
11473
  import { Hono as Hono6 } from "hono";
11474
+ import { z as z15 } from "zod";
11475
+ var RedditThreadBodySchema = z15.object({
11476
+ url: z15.string().trim().min(1, "url is required"),
11477
+ maxComments: z15.number().int().min(1).max(2e3).optional()
11478
+ });
11479
+ function invalidRequest3(message) {
11480
+ return { error_code: "invalid_request", message };
11481
+ }
11482
+ function toOldReddit(url) {
11483
+ try {
11484
+ const u = new URL(url.trim());
11485
+ if (!/(^|\.)reddit\.com$/.test(u.hostname.toLowerCase()) && u.hostname.toLowerCase() !== "redd.it") return null;
11486
+ u.hostname = "old.reddit.com";
11487
+ u.protocol = "https:";
11488
+ return u.toString();
11489
+ } catch {
11490
+ return null;
11491
+ }
11492
+ }
11493
+ async function residentialProxyId(attemptIndex) {
11494
+ try {
11495
+ const r = await resolveKernelProxyId({
11496
+ kernelApiKey: browserServiceApiKey(),
11497
+ proxyMode: "location",
11498
+ location: "Austin, TX",
11499
+ gl: "us",
11500
+ attemptIndex,
11501
+ fresh: true
11502
+ });
11503
+ return r.kernelProxyId;
11504
+ } catch {
11505
+ return void 0;
11506
+ }
11507
+ }
11508
+ var PARSE_REDDIT = `(() => {
11509
+ const txt = el => ((el && el.innerText) || '').trim();
11510
+ const attr = (el, a) => ((el && el.getAttribute(a)) || '').trim();
11511
+ const bodyText = (document.body && document.body.innerText) || '';
11512
+ const blocked = /whoa there|blocked by network|you've been blocked|network (policy|security)|log in to your reddit account/i.test(bodyText);
11513
+ const post = document.querySelector('#siteTable .thing.link') || document.querySelector('.thing.link');
11514
+ const title = txt(document.querySelector('a.title')) || (document.title || '');
11515
+ const author = attr(post, 'data-author');
11516
+ const scoreEl = post && post.querySelector('.score.unvoted');
11517
+ const score = scoreEl ? ((scoreEl.getAttribute('title') || scoreEl.textContent || '').trim()) : '';
11518
+ const postBody = txt(post && post.querySelector('.usertext-body .md'));
11519
+ const comments = [];
11520
+ document.querySelectorAll('.commentarea .comment').forEach(c => {
11521
+ const entry = c.querySelector(':scope > .entry');
11522
+ if (!entry) return;
11523
+ const a = attr(c, 'data-author');
11524
+ const b = txt(entry.querySelector('.usertext-body .md'));
11525
+ if (!b) return;
11526
+ const s = ((entry.querySelector('.score.unvoted') || {}).textContent || '').trim();
11527
+ let depth = 0, p = c.parentElement;
11528
+ while (p) { if (p.classList && p.classList.contains('comment')) depth++; p = p.parentElement; }
11529
+ comments.push({ author: a, score: s, depth: depth, body: b });
11530
+ });
11531
+ return { title: title, author: author, score: score, postBody: postBody, blocked: blocked, numComments: comments.length, comments: comments };
11532
+ })()`;
11533
+ var redditApp = new Hono6();
11534
+ redditApp.post("/thread", createApiKeyAuth(), async (c) => {
11535
+ const raw = await c.req.json().catch(() => ({}));
11536
+ const parsed = RedditThreadBodySchema.safeParse(raw);
11537
+ if (!parsed.success) return c.json(invalidRequest3(parsed.error.issues[0]?.message ?? "Invalid request"), 400);
11538
+ const body = parsed.data;
11539
+ const oldUrl = toOldReddit(body.url);
11540
+ if (!oldUrl) return c.json(invalidRequest3("url must be a reddit.com link"), 400);
11541
+ const user = c.get("user");
11542
+ const gate = await acquireConcurrencyGate(user, "reddit_thread", {
11543
+ reuseLockId: c.req.header("x-mcp-scraper-concurrency-lock"),
11544
+ metadata: { url: body.url }
11545
+ });
11546
+ if (!gate.ok) return c.json(concurrencyLimitExceededResponse(gate), 429, { "Retry-After": String(gate.retryAfterSeconds) });
11547
+ let debited = false;
11548
+ let refunded = false;
11549
+ try {
11550
+ const { ok, balance_mc } = await debitMc(user.id, MC_COSTS.reddit_thread, LedgerOperation.REDDIT_THREAD, body.url);
11551
+ if (!ok) return c.json(insufficientBalanceResponse(balance_mc, MC_COSTS.reddit_thread), 402);
11552
+ debited = true;
11553
+ let result = null;
11554
+ for (let attempt = 0; attempt < 3 && !result; attempt++) {
11555
+ const proxyId = await residentialProxyId(attempt);
11556
+ const driver = new BrowserDriver();
11557
+ try {
11558
+ await driver.launch({
11559
+ headless: true,
11560
+ headlessMode: "headful",
11561
+ kernelApiKey: browserServiceApiKey(),
11562
+ kernelProxyId: proxyId,
11563
+ viewport: { width: 1280, height: 900 },
11564
+ locale: "en-US"
11565
+ });
11566
+ const page = driver.getPage();
11567
+ await page.goto(oldUrl, { waitUntil: "domcontentloaded", timeout: 45e3 });
11568
+ await page.waitForTimeout(2500);
11569
+ const data = await page.evaluate(PARSE_REDDIT);
11570
+ if (!data.blocked && (data.postBody || data.comments.length > 0)) {
11571
+ result = { ...data, sourceUrl: body.url, oldRedditUrl: oldUrl, attempts: attempt + 1 };
11572
+ }
11573
+ } catch {
11574
+ } finally {
11575
+ await driver.close();
11576
+ }
11577
+ }
11578
+ if (!result) {
11579
+ await creditMc(user.id, MC_COSTS.reddit_thread, LedgerOperation.REDDIT_THREAD_REFUND, "reddit blocked/empty after retries");
11580
+ refunded = true;
11581
+ await logRequestEvent({ userId: user.id, source: "reddit_thread", status: "failed", query: body.url, error: "reddit blocked/empty after retries" });
11582
+ return c.json({ error: "Reddit blocked the request after retries (refunded)" }, 503);
11583
+ }
11584
+ if (body.maxComments && Array.isArray(result.comments)) {
11585
+ result.comments = result.comments.slice(0, body.maxComments);
11586
+ result.numComments = result.comments.length;
11587
+ }
11588
+ await logRequestEvent({ userId: user.id, source: "reddit_thread", status: "done", query: body.url, resultCount: Number(result.numComments ?? 0), result });
11589
+ return c.json(result);
11590
+ } catch (err) {
11591
+ if (debited && !refunded) await creditMc(user.id, MC_COSTS.reddit_thread, LedgerOperation.REDDIT_THREAD_REFUND, "failed call");
11592
+ const msg = err instanceof Error ? err.message : String(err);
11593
+ await logRequestEvent({ userId: user.id, source: "reddit_thread", status: "failed", query: body.url, error: msg });
11594
+ return c.json({ error: msg }, 500);
11595
+ } finally {
11596
+ await releaseConcurrencyGate(gate.lockId);
11597
+ }
11598
+ });
11599
+
11600
+ // src/api/maps-routes.ts
11601
+ import { Hono as Hono7 } from "hono";
11425
11602
 
11426
11603
  // src/extractor/MapsNavigator.ts
11427
11604
  var MapsNavigator = class {
@@ -11596,7 +11773,7 @@ var MapsReviewCollector = class {
11596
11773
  };
11597
11774
 
11598
11775
  // src/extractor/MapsExtractor.ts
11599
- import { z as z15 } from "zod";
11776
+ import { z as z16 } from "zod";
11600
11777
  var MapsExtractor = class {
11601
11778
  constructor(driver) {
11602
11779
  this.driver = driver;
@@ -11782,7 +11959,7 @@ var MapsExtractor = class {
11782
11959
  });
11783
11960
  return rows;
11784
11961
  }, { hoursTable: MapsSelectors.hoursTable, hoursTableAlt: MapsSelectors.hoursTableAlt });
11785
- const result = z15.array(RawMapsHoursRowSchema).safeParse(raw);
11962
+ const result = z16.array(RawMapsHoursRowSchema).safeParse(raw);
11786
11963
  if (!result.success) {
11787
11964
  console.warn("[MapsExtractor] hours parse failed", result.error.flatten());
11788
11965
  return [];
@@ -11846,7 +12023,7 @@ var MapsExtractor = class {
11846
12023
  });
11847
12024
  return results;
11848
12025
  });
11849
- const result = z15.array(RawMapsAboutAttributeSchema).safeParse(raw);
12026
+ const result = z16.array(RawMapsAboutAttributeSchema).safeParse(raw);
11850
12027
  if (!result.success) {
11851
12028
  console.warn("[MapsExtractor] about parse failed", result.error.flatten());
11852
12029
  return [];
@@ -12107,7 +12284,7 @@ async function cleanupDisposableProxy(kernelApiKey, proxyId, eventName) {
12107
12284
  }
12108
12285
  async function runMapsSearchWithRotation(options) {
12109
12286
  const attempts = [];
12110
- const maxAttempts = mapsSearchMaxAttemptsForProxyMode(options.proxyMode);
12287
+ const maxAttempts = mapsSearchMaxAttemptsForProxyMode(options.proxyMode) + 1;
12111
12288
  const started = Date.now();
12112
12289
  let lastError = null;
12113
12290
  for (let attemptIndex = 0; attemptIndex < maxAttempts; attemptIndex += 1) {
@@ -12129,12 +12306,15 @@ async function runMapsSearchWithRotation(options) {
12129
12306
  fresh: options.proxyMode === "location"
12130
12307
  });
12131
12308
  disposableProxyId = resolution.disposableProxyId;
12132
- const result = await extractor.extract({
12309
+ const extractArgs = {
12133
12310
  ...options,
12134
12311
  proxyZip,
12135
12312
  kernelProxyId: options.proxyMode === "none" ? void 0 : resolution.kernelProxyId,
12136
12313
  kernelProxyResolution: resolution.resolution
12137
- });
12314
+ };
12315
+ const isHeadfulEscalation = attemptIndex === maxAttempts - 1;
12316
+ const baseCtx = currentCostContext();
12317
+ const result = isHeadfulEscalation && baseCtx ? await runWithCostContext({ ...baseCtx, forceHeadful: true }, () => extractor.extract(extractArgs)) : await extractor.extract(extractArgs);
12138
12318
  const attempt = attemptFromDebug({
12139
12319
  attemptIndex,
12140
12320
  maxAttempts,
@@ -12219,7 +12399,7 @@ function mapsErrorResponse(c, err, errorCode) {
12219
12399
  attempts: rotationError?.attempts ?? void 0
12220
12400
  }, retryable ? 503 : 500);
12221
12401
  }
12222
- var mapsApp = new Hono6();
12402
+ var mapsApp = new Hono7();
12223
12403
  mapsApp.post("/search", createApiKeyAuth(), async (c) => {
12224
12404
  const user = c.get("user");
12225
12405
  const body = await c.req.json().catch(() => ({}));
@@ -12357,12 +12537,12 @@ mapsApp.post("/place", createApiKeyAuth(), async (c) => {
12357
12537
  });
12358
12538
 
12359
12539
  // src/api/directory-routes.ts
12360
- import { Hono as Hono7 } from "hono";
12540
+ import { Hono as Hono8 } from "hono";
12361
12541
 
12362
12542
  // src/directory/directory-workflow.ts
12363
12543
  import { mkdir as mkdir2, writeFile } from "fs/promises";
12364
12544
  import { join as join6 } from "path";
12365
- import { z as z16 } from "zod";
12545
+ import { z as z17 } from "zod";
12366
12546
 
12367
12547
  // src/directory/location-db.ts
12368
12548
  import { access, readFile } from "fs/promises";
@@ -12523,24 +12703,24 @@ async function resolveDirectoryMarkets(options) {
12523
12703
  }
12524
12704
 
12525
12705
  // src/directory/directory-workflow.ts
12526
- var DirectoryWorkflowOptionsSchema = z16.object({
12527
- query: z16.string().min(1),
12528
- state: z16.string().min(2).default("TN"),
12529
- minPopulation: z16.number().int().min(0).default(1e5),
12530
- populationYear: z16.union(POPULATION_YEARS.map((year) => z16.literal(year))).default(2025),
12531
- maxCities: z16.number().int().min(1).max(100).default(25),
12532
- maxResultsPerCity: z16.number().int().min(1).max(50).default(50),
12533
- concurrency: z16.number().int().min(1).max(5).default(5),
12534
- includeZipGroups: z16.boolean().default(true),
12535
- usZipsCsvPath: z16.string().optional(),
12536
- saveCsv: z16.boolean().default(true),
12537
- gl: z16.string().length(2).default("us"),
12538
- hl: z16.string().length(2).default("en"),
12539
- proxyMode: z16.enum(["location", "configured", "none"]).default(DEFAULT_MAPS_PROXY_MODE),
12540
- proxyZip: z16.string().regex(/^\d{5}$/).optional(),
12541
- debug: z16.boolean().default(false),
12542
- headless: z16.boolean().default(true),
12543
- kernelApiKey: z16.string().optional()
12706
+ var DirectoryWorkflowOptionsSchema = z17.object({
12707
+ query: z17.string().min(1),
12708
+ state: z17.string().min(2).default("TN"),
12709
+ minPopulation: z17.number().int().min(0).default(1e5),
12710
+ populationYear: z17.union(POPULATION_YEARS.map((year) => z17.literal(year))).default(2025),
12711
+ maxCities: z17.number().int().min(1).max(100).default(25),
12712
+ maxResultsPerCity: z17.number().int().min(1).max(50).default(50),
12713
+ concurrency: z17.number().int().min(1).max(5).default(5),
12714
+ includeZipGroups: z17.boolean().default(true),
12715
+ usZipsCsvPath: z17.string().optional(),
12716
+ saveCsv: z17.boolean().default(true),
12717
+ gl: z17.string().length(2).default("us"),
12718
+ hl: z17.string().length(2).default("en"),
12719
+ proxyMode: z17.enum(["location", "configured", "none"]).default(DEFAULT_MAPS_PROXY_MODE),
12720
+ proxyZip: z17.string().regex(/^\d{5}$/).optional(),
12721
+ debug: z17.boolean().default(false),
12722
+ headless: z17.boolean().default(true),
12723
+ kernelApiKey: z17.string().optional()
12544
12724
  });
12545
12725
  function errorMessage(err) {
12546
12726
  return err instanceof Error ? err.message : String(err);
@@ -12761,7 +12941,7 @@ async function runDirectoryWorkflowFromPlan(options, plan) {
12761
12941
  }
12762
12942
 
12763
12943
  // src/api/directory-routes.ts
12764
- var directoryApp = new Hono7();
12944
+ var directoryApp = new Hono8();
12765
12945
  directoryApp.post("/run", createApiKeyAuth(), async (c) => {
12766
12946
  const user = c.get("user");
12767
12947
  const body = await c.req.json().catch(() => ({}));
@@ -12834,35 +13014,35 @@ directoryApp.post("/run", createApiKeyAuth(), async (c) => {
12834
13014
  // src/api/workflow-routes.ts
12835
13015
  import { createHmac } from "crypto";
12836
13016
  import { readFile as readFile2 } from "fs/promises";
12837
- import { Hono as Hono8 } from "hono";
12838
- import { z as z17 } from "zod";
12839
- var workflowApp = new Hono8();
12840
- var WorkflowInputSchema = z17.record(z17.unknown()).default({});
12841
- var WorkflowIdSchema = z17.string().min(1);
12842
- var CadenceSchema = z17.enum(["daily", "weekly", "monthly"]);
12843
- var ScheduleStatusSchema = z17.enum(["active", "paused"]);
12844
- var RunBodySchema = z17.object({
13017
+ import { Hono as Hono9 } from "hono";
13018
+ import { z as z18 } from "zod";
13019
+ var workflowApp = new Hono9();
13020
+ var WorkflowInputSchema = z18.record(z18.unknown()).default({});
13021
+ var WorkflowIdSchema = z18.string().min(1);
13022
+ var CadenceSchema = z18.enum(["daily", "weekly", "monthly"]);
13023
+ var ScheduleStatusSchema = z18.enum(["active", "paused"]);
13024
+ var RunBodySchema = z18.object({
12845
13025
  workflowId: WorkflowIdSchema,
12846
13026
  input: WorkflowInputSchema,
12847
- webhookUrl: z17.string().url().optional()
13027
+ webhookUrl: z18.string().url().optional()
12848
13028
  });
12849
- var ScheduleCreateSchema = z17.object({
13029
+ var ScheduleCreateSchema = z18.object({
12850
13030
  workflowId: WorkflowIdSchema,
12851
- name: z17.string().min(1).max(120).optional(),
13031
+ name: z18.string().min(1).max(120).optional(),
12852
13032
  input: WorkflowInputSchema,
12853
13033
  cadence: CadenceSchema.default("weekly"),
12854
- timezone: z17.string().min(1).max(64).default("UTC"),
12855
- webhookUrl: z17.string().url().optional(),
12856
- nextRunAt: z17.string().datetime().optional()
13034
+ timezone: z18.string().min(1).max(64).default("UTC"),
13035
+ webhookUrl: z18.string().url().optional(),
13036
+ nextRunAt: z18.string().datetime().optional()
12857
13037
  });
12858
- var SchedulePatchSchema = z17.object({
12859
- name: z17.string().min(1).max(120).optional(),
13038
+ var SchedulePatchSchema = z18.object({
13039
+ name: z18.string().min(1).max(120).optional(),
12860
13040
  status: ScheduleStatusSchema.optional(),
12861
13041
  input: WorkflowInputSchema.optional(),
12862
13042
  cadence: CadenceSchema.optional(),
12863
- timezone: z17.string().min(1).max(64).optional(),
12864
- webhookUrl: z17.string().url().nullable().optional(),
12865
- nextRunAt: z17.string().datetime().nullable().optional()
13043
+ timezone: z18.string().min(1).max(64).optional(),
13044
+ webhookUrl: z18.string().url().nullable().optional(),
13045
+ nextRunAt: z18.string().datetime().nullable().optional()
12866
13046
  });
12867
13047
  function hostedWorkflowOutputDir() {
12868
13048
  return workflowOutputBaseDir(process.env.MCP_SCRAPER_WORKFLOW_OUTPUT_DIR?.trim() || "/tmp/mcp-scraper-workflows");
@@ -13366,7 +13546,7 @@ workflowApp.post("/cron/dispatch", async (c) => {
13366
13546
  });
13367
13547
 
13368
13548
  // src/api/serp-intelligence-routes.ts
13369
- import { Hono as Hono9 } from "hono";
13549
+ import { Hono as Hono10 } from "hono";
13370
13550
 
13371
13551
  // src/serp-intelligence/page-snapshot-extractor.ts
13372
13552
  import { createHash as createHash2 } from "crypto";
@@ -13678,7 +13858,7 @@ async function capturePageSnapshots(targets, options = {}) {
13678
13858
  }
13679
13859
 
13680
13860
  // src/serp-intelligence/schemas.ts
13681
- import { z as z18 } from "zod";
13861
+ import { z as z19 } from "zod";
13682
13862
  var SerpIntelligenceDeviceValues = ["desktop", "mobile"];
13683
13863
  var SerpIntelligenceProxyModeValues = ["location", "configured", "none"];
13684
13864
  var SerpIntelligenceAttemptOutcomeValues = [
@@ -13740,171 +13920,171 @@ function isPublicHttpUrl(value) {
13740
13920
  return false;
13741
13921
  }
13742
13922
  }
13743
- var SerpIntelligencePublicHttpUrlSchema = z18.string().url().refine(isPublicHttpUrl, "url must be a public HTTP or HTTPS URL");
13744
- var SerpIntelligenceCaptureBodySchema = z18.object({
13745
- query: z18.string().trim().min(1, "query is required"),
13746
- location: z18.string().trim().min(1).optional(),
13747
- gl: z18.string().trim().length(2).default("us"),
13748
- hl: z18.string().trim().length(2).default("en"),
13749
- device: z18.enum(SerpIntelligenceDeviceValues).default("desktop"),
13750
- proxyMode: z18.enum(SerpIntelligenceProxyModeValues).default(DEFAULT_PROXY_MODE),
13751
- proxyZip: z18.string().regex(/^\d{5}$/).optional(),
13752
- pages: z18.number().int().min(1).max(2).default(1),
13753
- debug: z18.boolean().default(false),
13754
- includePageSnapshots: z18.boolean().default(false),
13755
- pageSnapshotLimit: z18.number().int().min(0).max(10).default(0)
13923
+ var SerpIntelligencePublicHttpUrlSchema = z19.string().url().refine(isPublicHttpUrl, "url must be a public HTTP or HTTPS URL");
13924
+ var SerpIntelligenceCaptureBodySchema = z19.object({
13925
+ query: z19.string().trim().min(1, "query is required"),
13926
+ location: z19.string().trim().min(1).optional(),
13927
+ gl: z19.string().trim().length(2).default("us"),
13928
+ hl: z19.string().trim().length(2).default("en"),
13929
+ device: z19.enum(SerpIntelligenceDeviceValues).default("desktop"),
13930
+ proxyMode: z19.enum(SerpIntelligenceProxyModeValues).default(DEFAULT_PROXY_MODE),
13931
+ proxyZip: z19.string().regex(/^\d{5}$/).optional(),
13932
+ pages: z19.number().int().min(1).max(2).default(1),
13933
+ debug: z19.boolean().default(false),
13934
+ includePageSnapshots: z19.boolean().default(false),
13935
+ pageSnapshotLimit: z19.number().int().min(0).max(10).default(0)
13756
13936
  }).strict();
13757
- var SerpIntelligencePageSnapshotRequestSchema = z18.object({
13937
+ var SerpIntelligencePageSnapshotRequestSchema = z19.object({
13758
13938
  url: SerpIntelligencePublicHttpUrlSchema,
13759
- sourceKind: z18.enum(SerpPageSnapshotSourceKindValues).default("configured_target"),
13760
- sourcePosition: z18.number().int().min(1).optional()
13939
+ sourceKind: z19.enum(SerpPageSnapshotSourceKindValues).default("configured_target"),
13940
+ sourcePosition: z19.number().int().min(1).optional()
13761
13941
  }).strict();
13762
- var SerpIntelligencePageSnapshotsBodySchema = z18.object({
13763
- urls: z18.array(SerpIntelligencePublicHttpUrlSchema).min(1).max(25),
13764
- targets: z18.array(SerpIntelligencePageSnapshotRequestSchema).min(1).max(25).optional(),
13765
- maxConcurrency: z18.number().int().min(1).max(5).default(2),
13766
- timeoutMs: z18.number().int().min(1e3).max(6e4).default(15e3),
13767
- debug: z18.boolean().default(false)
13942
+ var SerpIntelligencePageSnapshotsBodySchema = z19.object({
13943
+ urls: z19.array(SerpIntelligencePublicHttpUrlSchema).min(1).max(25),
13944
+ targets: z19.array(SerpIntelligencePageSnapshotRequestSchema).min(1).max(25).optional(),
13945
+ maxConcurrency: z19.number().int().min(1).max(5).default(2),
13946
+ timeoutMs: z19.number().int().min(1e3).max(6e4).default(15e3),
13947
+ debug: z19.boolean().default(false)
13768
13948
  }).strict();
13769
- var SerpIntelligenceAICitationSchema = z18.object({
13770
- text: z18.string(),
13771
- href: z18.string()
13949
+ var SerpIntelligenceAICitationSchema = z19.object({
13950
+ text: z19.string(),
13951
+ href: z19.string()
13772
13952
  }).strict();
13773
- var SerpIntelligenceOrganicResultSchema = z18.object({
13774
- position: z18.number().int().min(1),
13775
- title: z18.string(),
13776
- url: z18.string(),
13777
- domain: z18.string(),
13778
- cite: z18.string().nullable(),
13779
- snippet: z18.string().nullable(),
13780
- isRedditStyle: z18.boolean(),
13781
- inlineRating: z18.object({
13782
- value: z18.string(),
13783
- count: z18.string()
13953
+ var SerpIntelligenceOrganicResultSchema = z19.object({
13954
+ position: z19.number().int().min(1),
13955
+ title: z19.string(),
13956
+ url: z19.string(),
13957
+ domain: z19.string(),
13958
+ cite: z19.string().nullable(),
13959
+ snippet: z19.string().nullable(),
13960
+ isRedditStyle: z19.boolean(),
13961
+ inlineRating: z19.object({
13962
+ value: z19.string(),
13963
+ count: z19.string()
13784
13964
  }).strict().nullable()
13785
13965
  }).strict();
13786
- var SerpIntelligenceLocationEvidenceSchema = z18.object({
13787
- status: z18.enum(SerpIntelligenceLocalizationStatusValues),
13788
- expected: z18.object({
13789
- city: z18.string(),
13790
- regionCode: z18.string().nullable(),
13791
- canonicalLocation: z18.string()
13966
+ var SerpIntelligenceLocationEvidenceSchema = z19.object({
13967
+ status: z19.enum(SerpIntelligenceLocalizationStatusValues),
13968
+ expected: z19.object({
13969
+ city: z19.string(),
13970
+ regionCode: z19.string().nullable(),
13971
+ canonicalLocation: z19.string()
13792
13972
  }).strict().nullable(),
13793
- candidates: z18.array(z18.object({
13794
- city: z18.string(),
13795
- regionCode: z18.string(),
13796
- count: z18.number().int().min(0),
13797
- examples: z18.array(z18.string())
13973
+ candidates: z19.array(z19.object({
13974
+ city: z19.string(),
13975
+ regionCode: z19.string(),
13976
+ count: z19.number().int().min(0),
13977
+ examples: z19.array(z19.string())
13798
13978
  }).strict())
13799
13979
  }).strict();
13800
- var SerpIntelligenceHarvestResultSchema = z18.object({
13801
- seed: z18.string(),
13802
- location: z18.string().nullable(),
13803
- extractedAt: z18.string(),
13804
- totalQuestions: z18.number().int().min(0),
13805
- surface: z18.enum(["web", "aim", "unknown"]),
13806
- aiOverview: z18.object({
13807
- detected: z18.boolean(),
13808
- text: z18.string().nullable(),
13809
- citations: z18.array(SerpIntelligenceAICitationSchema),
13810
- expanded: z18.boolean().optional(),
13811
- fullyExpanded: z18.boolean().optional(),
13812
- sections: z18.array(z18.string()).optional()
13980
+ var SerpIntelligenceHarvestResultSchema = z19.object({
13981
+ seed: z19.string(),
13982
+ location: z19.string().nullable(),
13983
+ extractedAt: z19.string(),
13984
+ totalQuestions: z19.number().int().min(0),
13985
+ surface: z19.enum(["web", "aim", "unknown"]),
13986
+ aiOverview: z19.object({
13987
+ detected: z19.boolean(),
13988
+ text: z19.string().nullable(),
13989
+ citations: z19.array(SerpIntelligenceAICitationSchema),
13990
+ expanded: z19.boolean().optional(),
13991
+ fullyExpanded: z19.boolean().optional(),
13992
+ sections: z19.array(z19.string()).optional()
13813
13993
  }).strict(),
13814
- aiMode: z18.object({
13815
- detected: z18.boolean(),
13816
- text: z18.string().nullable(),
13817
- citations: z18.array(SerpIntelligenceAICitationSchema)
13994
+ aiMode: z19.object({
13995
+ detected: z19.boolean(),
13996
+ text: z19.string().nullable(),
13997
+ citations: z19.array(SerpIntelligenceAICitationSchema)
13818
13998
  }).strict(),
13819
- tree: z18.array(z18.unknown()),
13820
- flat: z18.array(z18.unknown()),
13821
- videos: z18.array(z18.unknown()),
13822
- forums: z18.array(z18.unknown()),
13823
- organicResults: z18.array(SerpIntelligenceOrganicResultSchema),
13824
- localPack: z18.array(z18.unknown()),
13825
- entityIds: z18.object({
13826
- entities: z18.array(z18.object({
13827
- name: z18.string(),
13828
- kgId: z18.string().nullable(),
13829
- cid: z18.string().nullable(),
13830
- gcid: z18.string().nullable()
13999
+ tree: z19.array(z19.unknown()),
14000
+ flat: z19.array(z19.unknown()),
14001
+ videos: z19.array(z19.unknown()),
14002
+ forums: z19.array(z19.unknown()),
14003
+ organicResults: z19.array(SerpIntelligenceOrganicResultSchema),
14004
+ localPack: z19.array(z19.unknown()),
14005
+ entityIds: z19.object({
14006
+ entities: z19.array(z19.object({
14007
+ name: z19.string(),
14008
+ kgId: z19.string().nullable(),
14009
+ cid: z19.string().nullable(),
14010
+ gcid: z19.string().nullable()
13831
14011
  }).strict()),
13832
- kgIds: z18.array(z18.string()),
13833
- cids: z18.array(z18.string()),
13834
- gcids: z18.array(z18.string())
14012
+ kgIds: z19.array(z19.string()),
14013
+ cids: z19.array(z19.string()),
14014
+ gcids: z19.array(z19.string())
13835
14015
  }).strict(),
13836
- stats: z18.object({
13837
- seed: z18.string(),
13838
- totalQuestions: z18.number().int().min(0),
13839
- maxDepthReached: z18.number().int().min(0),
13840
- durationMs: z18.number().min(0),
13841
- errorCount: z18.number().int().min(0)
14016
+ stats: z19.object({
14017
+ seed: z19.string(),
14018
+ totalQuestions: z19.number().int().min(0),
14019
+ maxDepthReached: z19.number().int().min(0),
14020
+ durationMs: z19.number().min(0),
14021
+ errorCount: z19.number().int().min(0)
13842
14022
  }).strict(),
13843
- diagnostics: z18.object({
13844
- completionStatus: z18.enum(["paa_found", "no_paa", "serp_only"]),
13845
- problem: z18.null(),
13846
- warnings: z18.array(z18.unknown()).optional(),
13847
- debug: z18.object({
14023
+ diagnostics: z19.object({
14024
+ completionStatus: z19.enum(["paa_found", "no_paa", "serp_only"]),
14025
+ problem: z19.null(),
14026
+ warnings: z19.array(z19.unknown()).optional(),
14027
+ debug: z19.object({
13848
14028
  locationEvidence: SerpIntelligenceLocationEvidenceSchema.optional()
13849
14029
  }).passthrough().optional()
13850
14030
  }).passthrough(),
13851
- whatPeopleSaying: z18.array(z18.unknown())
14031
+ whatPeopleSaying: z19.array(z19.unknown())
13852
14032
  }).strict();
13853
- var SerpIntelligenceCaptureAttemptSchema = z18.object({
13854
- attemptNumber: z18.number().int().min(1),
13855
- outcome: z18.enum(SerpIntelligenceAttemptOutcomeValues),
13856
- startedAt: z18.string().optional(),
13857
- completedAt: z18.string().optional(),
13858
- durationMs: z18.number().min(0).optional(),
13859
- problemCode: z18.string().optional(),
13860
- message: z18.string().optional(),
13861
- kernelSessionId: z18.string().nullable().optional(),
13862
- cleanupSucceeded: z18.boolean().nullable().optional()
14033
+ var SerpIntelligenceCaptureAttemptSchema = z19.object({
14034
+ attemptNumber: z19.number().int().min(1),
14035
+ outcome: z19.enum(SerpIntelligenceAttemptOutcomeValues),
14036
+ startedAt: z19.string().optional(),
14037
+ completedAt: z19.string().optional(),
14038
+ durationMs: z19.number().min(0).optional(),
14039
+ problemCode: z19.string().optional(),
14040
+ message: z19.string().optional(),
14041
+ kernelSessionId: z19.string().nullable().optional(),
14042
+ cleanupSucceeded: z19.boolean().nullable().optional()
13863
14043
  }).strict();
13864
- var SerpPageSnapshotCaptureSchema = z18.object({
14044
+ var SerpPageSnapshotCaptureSchema = z19.object({
13865
14045
  url: SerpIntelligencePublicHttpUrlSchema,
13866
14046
  requestedUrl: SerpIntelligencePublicHttpUrlSchema,
13867
14047
  finalUrl: SerpIntelligencePublicHttpUrlSchema.nullable(),
13868
- sourceKind: z18.enum(SerpPageSnapshotSourceKindValues),
13869
- sourcePosition: z18.number().int().min(1).nullable(),
13870
- status: z18.enum(SerpPageFetchStatusValues),
13871
- fetchedVia: z18.enum(SerpPageFetchedViaValues).nullable(),
13872
- httpStatus: z18.number().int().min(100).max(599).nullable(),
13873
- contentType: z18.string().nullable(),
13874
- title: z18.string().nullable(),
14048
+ sourceKind: z19.enum(SerpPageSnapshotSourceKindValues),
14049
+ sourcePosition: z19.number().int().min(1).nullable(),
14050
+ status: z19.enum(SerpPageFetchStatusValues),
14051
+ fetchedVia: z19.enum(SerpPageFetchedViaValues).nullable(),
14052
+ httpStatus: z19.number().int().min(100).max(599).nullable(),
14053
+ contentType: z19.string().nullable(),
14054
+ title: z19.string().nullable(),
13875
14055
  canonicalUrl: SerpIntelligencePublicHttpUrlSchema.nullable(),
13876
- metaDescription: z18.string().nullable(),
13877
- headings: z18.array(z18.object({
13878
- level: z18.number().int().min(1).max(6),
13879
- text: z18.string()
14056
+ metaDescription: z19.string().nullable(),
14057
+ headings: z19.array(z19.object({
14058
+ level: z19.number().int().min(1).max(6),
14059
+ text: z19.string()
13880
14060
  }).strict()).default([]),
13881
- artifact: z18.object({
13882
- htmlBlobUrl: z18.string().url().nullable(),
13883
- textBlobUrl: z18.string().url().nullable(),
13884
- markdownBlobUrl: z18.string().url().nullable(),
13885
- screenshotBlobUrl: z18.string().url().nullable(),
13886
- contentSha256: z18.string().nullable(),
13887
- capturedAt: z18.string().nullable()
14061
+ artifact: z19.object({
14062
+ htmlBlobUrl: z19.string().url().nullable(),
14063
+ textBlobUrl: z19.string().url().nullable(),
14064
+ markdownBlobUrl: z19.string().url().nullable(),
14065
+ screenshotBlobUrl: z19.string().url().nullable(),
14066
+ contentSha256: z19.string().nullable(),
14067
+ capturedAt: z19.string().nullable()
13888
14068
  }).strict(),
13889
- error: z18.object({
13890
- code: z18.string(),
13891
- message: z18.string()
14069
+ error: z19.object({
14070
+ code: z19.string(),
14071
+ message: z19.string()
13892
14072
  }).strict().nullable()
13893
14073
  }).strict();
13894
- var SerpIntelligenceCaptureResponseSchema = z18.object({
14074
+ var SerpIntelligenceCaptureResponseSchema = z19.object({
13895
14075
  harvestResult: SerpIntelligenceHarvestResultSchema,
13896
- attempts: z18.array(SerpIntelligenceCaptureAttemptSchema),
14076
+ attempts: z19.array(SerpIntelligenceCaptureAttemptSchema),
13897
14077
  locationEvidence: SerpIntelligenceLocationEvidenceSchema.nullable(),
13898
- pageSnapshotArtifacts: z18.array(SerpPageSnapshotCaptureSchema),
13899
- billing: z18.object({
13900
- creditsUsed: z18.number().min(0).optional(),
13901
- requestId: z18.string().optional(),
13902
- jobId: z18.string().optional()
14078
+ pageSnapshotArtifacts: z19.array(SerpPageSnapshotCaptureSchema),
14079
+ billing: z19.object({
14080
+ creditsUsed: z19.number().min(0).optional(),
14081
+ requestId: z19.string().optional(),
14082
+ jobId: z19.string().optional()
13903
14083
  }).strict().optional()
13904
14084
  }).strict();
13905
- var SerpIntelligencePageSnapshotsResponseSchema = z18.object({
13906
- pageSnapshotArtifacts: z18.array(SerpPageSnapshotCaptureSchema),
13907
- attempts: z18.array(SerpIntelligenceCaptureAttemptSchema).default([])
14085
+ var SerpIntelligencePageSnapshotsResponseSchema = z19.object({
14086
+ pageSnapshotArtifacts: z19.array(SerpPageSnapshotCaptureSchema),
14087
+ attempts: z19.array(SerpIntelligenceCaptureAttemptSchema).default([])
13908
14088
  }).strict();
13909
14089
 
13910
14090
  // src/serp-intelligence/serp-capture-service.ts
@@ -14076,7 +14256,7 @@ var SERP_INTELLIGENCE_RATE_LIMIT = 60;
14076
14256
  var SERP_INTELLIGENCE_RATE_WINDOW_SECONDS = 60;
14077
14257
  var POST_CAPTURE_ROUTE_LABEL = "POST /capture";
14078
14258
  var POST_PAGE_SNAPSHOTS_ROUTE_LABEL = "POST /page-snapshots";
14079
- var serpIntelligenceApp = new Hono9();
14259
+ var serpIntelligenceApp = new Hono10();
14080
14260
  serpIntelligenceApp.use("*", createApiKeyAuth());
14081
14261
  function structuredError(input) {
14082
14262
  return {
@@ -14245,41 +14425,95 @@ serpIntelligenceApp.post("/page-snapshots", async (c) => {
14245
14425
  });
14246
14426
 
14247
14427
  // src/mcp/mcp-routes.ts
14248
- import { Hono as Hono10 } from "hono";
14428
+ import { Hono as Hono11 } from "hono";
14249
14429
  import { WebStandardStreamableHTTPServerTransport } from "@modelcontextprotocol/sdk/server/webStandardStreamableHttp.js";
14430
+
14431
+ // src/mcp/mcp-oauth.ts
14432
+ import { createRemoteJWKSet, jwtVerify } from "jose";
14433
+ function oauthIssuer() {
14434
+ return (process.env.OAUTH_ISSUER ?? "https://mcpscraper.dev").replace(/\/$/, "");
14435
+ }
14436
+ function oauthResourceUrl() {
14437
+ return (process.env.MCP_SCRAPER_RESOURCE_URL ?? "https://mcpscraper.dev/mcp").replace(/\/$/, "");
14438
+ }
14439
+ function oauthResourceBaseUrl() {
14440
+ return (process.env.OAUTH_RS_BASE_URL ?? "https://mcpscraper.dev").replace(/\/$/, "");
14441
+ }
14442
+ function protectedResourceMetadata() {
14443
+ return {
14444
+ resource: oauthResourceUrl(),
14445
+ authorization_servers: [oauthIssuer()],
14446
+ scopes_supported: ["scraper.use"],
14447
+ bearer_methods_supported: ["header"]
14448
+ };
14449
+ }
14450
+ function wwwAuthenticateValue() {
14451
+ return `Bearer resource_metadata="${oauthResourceBaseUrl()}/.well-known/oauth-protected-resource"`;
14452
+ }
14453
+ function isBearerJwt(token) {
14454
+ return !token.startsWith("sk_") && token.split(".").length === 3;
14455
+ }
14456
+ var jwksSingleton = null;
14457
+ function getJwks() {
14458
+ if (jwksSingleton) return jwksSingleton;
14459
+ jwksSingleton = createRemoteJWKSet(new URL(`${oauthIssuer()}/.well-known/jwks.json`));
14460
+ return jwksSingleton;
14461
+ }
14462
+ async function verifyScraperOAuthJwt(token) {
14463
+ const { payload } = await jwtVerify(token, getJwks(), {
14464
+ issuer: oauthIssuer(),
14465
+ audience: oauthResourceUrl()
14466
+ });
14467
+ return payload;
14468
+ }
14469
+
14470
+ // src/mcp/mcp-routes.ts
14250
14471
  configureReportSaving(false);
14251
- function mcpAuthError() {
14472
+ function mcpAuthError(message = "authentication required") {
14252
14473
  const body = JSON.stringify({
14253
14474
  jsonrpc: "2.0",
14254
14475
  id: null,
14255
14476
  error: {
14256
14477
  code: -32001,
14257
- message: "authentication required"
14478
+ message
14258
14479
  }
14259
14480
  });
14260
14481
  return new Response(body, {
14261
14482
  status: 401,
14262
14483
  headers: {
14263
14484
  "Content-Type": "application/json",
14264
- "WWW-Authenticate": 'Bearer realm="mcp-scraper", error="invalid_token", error_description="Pass an MCP Scraper API key as x-api-key or Bearer token"'
14485
+ "WWW-Authenticate": wwwAuthenticateValue()
14265
14486
  }
14266
14487
  });
14267
14488
  }
14268
14489
  async function requireMcpCallerKey(c) {
14269
14490
  const xApiKey = c.req.header("x-api-key")?.trim();
14270
14491
  const authHeader = c.req.header("Authorization")?.trim();
14271
- const bearerKey = authHeader?.startsWith("Bearer ") ? authHeader.slice(7).trim() : void 0;
14272
- const callerKey = xApiKey ?? bearerKey;
14492
+ const bearerToken = authHeader?.startsWith("Bearer ") ? authHeader.slice(7).trim() : void 0;
14493
+ if (!xApiKey && bearerToken && isBearerJwt(bearerToken)) {
14494
+ let identity;
14495
+ try {
14496
+ const payload = await verifyScraperOAuthJwt(bearerToken);
14497
+ identity = typeof payload.sub === "string" ? payload.sub : "";
14498
+ } catch {
14499
+ return mcpAuthError("Bearer token is invalid or expired");
14500
+ }
14501
+ if (!identity) return mcpAuthError("Bearer token has no subject");
14502
+ const user2 = await getUserByEmail(identity);
14503
+ if (!user2 || !user2.api_key || user2.key_active !== 1) return mcpAuthError("No active MCP Scraper account for this identity");
14504
+ return user2.api_key;
14505
+ }
14506
+ const callerKey = xApiKey ?? bearerToken;
14273
14507
  if (!callerKey) return mcpAuthError();
14274
14508
  const user = await getUserByApiKey(callerKey);
14275
14509
  if (!user) return mcpAuthError();
14276
14510
  return callerKey;
14277
14511
  }
14278
- var mcpApp = new Hono10();
14512
+ var mcpApp = new Hono11();
14279
14513
  function registerSerpIntelligenceCaptureTools(server, executor) {
14280
14514
  server.registerTool("capture_serp_snapshot", {
14281
14515
  title: "SERP Intelligence Snapshot",
14282
- description: "Capture a structured SERP Intelligence Google snapshot through POST /serp-intelligence/capture, the same product capture path used by Phoenix. Split query from location and infer gl/hl. Omit proxyMode for normal use; the service defaults to the configured browser-service proxy without city/ZIP targeting for the highest general success rate. Use proxyMode location only when the user explicitly needs city/ZIP-targeted residential proxy evidence. Use none only for direct-network debugging. Set debug true when investigating location evidence, proxy behavior, CAPTCHA, or capture reliability.",
14516
+ description: "Capture a structured SERP Intelligence Google snapshot through POST /serp-intelligence/capture, the same product capture path used by Phoenix. Split query from location and infer gl/hl. Routing is automatic \u2014 leave proxyMode unset. Set debug true when investigating location evidence, proxy behavior, CAPTCHA, or capture reliability.",
14283
14517
  inputSchema: CaptureSerpSnapshotInputSchema,
14284
14518
  outputSchema: CaptureSerpSnapshotOutputSchema,
14285
14519
  annotations: liveWebToolAnnotations("SERP Intelligence Snapshot")
@@ -14313,7 +14547,7 @@ mcpApp.all("/", async (c) => {
14313
14547
  });
14314
14548
 
14315
14549
  // src/api/browser-agent-routes.ts
14316
- import { Hono as Hono11 } from "hono";
14550
+ import { Hono as Hono12 } from "hono";
14317
14551
 
14318
14552
  // src/api/browser-agent-db.ts
14319
14553
  import { randomUUID as randomUUID2 } from "crypto";
@@ -15747,7 +15981,7 @@ async function loadOpenSession(id, userId) {
15747
15981
  return row;
15748
15982
  }
15749
15983
  function buildBrowserAgentRoutes() {
15750
- const app2 = new Hono11();
15984
+ const app2 = new Hono12();
15751
15985
  app2.use("*", async (c, next) => {
15752
15986
  await migrateBrowserAgent();
15753
15987
  return next();
@@ -16453,9 +16687,149 @@ if (state.key) { refreshSessions(); if (state.current) selectSession(state.curre
16453
16687
 
16454
16688
  // src/api/stripe-routes.ts
16455
16689
  import Stripe from "stripe";
16456
- import { Hono as Hono12 } from "hono";
16690
+ import { Hono as Hono13 } from "hono";
16691
+
16692
+ // src/api/memory.ts
16693
+ import { createCipheriv, createDecipheriv, randomBytes, scryptSync } from "crypto";
16694
+
16695
+ // src/api/session.ts
16696
+ import { createHmac as createHmac2, timingSafeEqual } from "crypto";
16697
+ var isProduction = () => process.env.NODE_ENV === "production" || process.env.VERCEL === "1";
16698
+ function getSessionSecret() {
16699
+ const configured = process.env.SESSION_SECRET?.trim();
16700
+ if (configured) return configured;
16701
+ if (isProduction()) throw new Error("SESSION_SECRET is not set \u2014 add it to your Vercel env vars (see src/api/env.ts)");
16702
+ return "dev-secret-change-me";
16703
+ }
16704
+ var secret = () => getSessionSecret();
16705
+ function safeEqualHex(a, b) {
16706
+ if (a.length !== b.length) return false;
16707
+ try {
16708
+ return timingSafeEqual(Buffer.from(a, "hex"), Buffer.from(b, "hex"));
16709
+ } catch {
16710
+ return false;
16711
+ }
16712
+ }
16713
+ function signSession(userId) {
16714
+ const payload = String(userId);
16715
+ const sig = createHmac2("sha256", secret()).update(payload).digest("hex");
16716
+ return `${payload}.${sig}`;
16717
+ }
16718
+ function verifySession(token) {
16719
+ const dot = token.lastIndexOf(".");
16720
+ if (dot === -1) return null;
16721
+ const payload = token.slice(0, dot);
16722
+ const sig = token.slice(dot + 1);
16723
+ const expected = createHmac2("sha256", secret()).update(payload).digest("hex");
16724
+ if (!safeEqualHex(sig, expected)) return null;
16725
+ const id = parseInt(payload);
16726
+ return isNaN(id) ? null : id;
16727
+ }
16728
+
16729
+ // src/api/memory.ts
16730
+ var MEMORY_BASE_URL = () => (process.env.MCP_MEMORY_BASE_URL ?? "https://mcp-memory-omega.vercel.app").replace(/\/$/, "");
16731
+ var ADMIN_KEY = () => process.env.MCP_MEMORY_ADMIN_KEY?.trim() ?? "";
16732
+ function isMemoryOperator(email) {
16733
+ if (!email) return false;
16734
+ const ops = (process.env.MEMORY_OPERATOR_IDENTITIES ?? "").split(",").map((s) => s.trim().toLowerCase()).filter(Boolean);
16735
+ return ops.includes(email.trim().toLowerCase());
16736
+ }
16737
+ function encKey() {
16738
+ return scryptSync(getSessionSecret(), "mcp-memory-key-v1", 32);
16739
+ }
16740
+ function encryptMemoryKey(secret2) {
16741
+ const iv = randomBytes(12);
16742
+ const cipher = createCipheriv("aes-256-gcm", encKey(), iv);
16743
+ const enc = Buffer.concat([cipher.update(secret2, "utf8"), cipher.final()]);
16744
+ const tag = cipher.getAuthTag();
16745
+ return `${iv.toString("base64")}:${tag.toString("base64")}:${enc.toString("base64")}`;
16746
+ }
16747
+ function decryptMemoryKey(stored) {
16748
+ try {
16749
+ const [ivB, tagB, dataB] = stored.split(":");
16750
+ const decipher = createDecipheriv("aes-256-gcm", encKey(), Buffer.from(ivB, "base64"));
16751
+ decipher.setAuthTag(Buffer.from(tagB, "base64"));
16752
+ return Buffer.concat([decipher.update(Buffer.from(dataB, "base64")), decipher.final()]).toString("utf8");
16753
+ } catch {
16754
+ return null;
16755
+ }
16756
+ }
16757
+ async function memoryCall(toolName, args, userMemoryKey) {
16758
+ try {
16759
+ const res = await fetch(`${MEMORY_BASE_URL()}/api/mcp/thorbitMemoryServer/tools/${toolName}/execute`, {
16760
+ method: "POST",
16761
+ headers: { "content-type": "application/json" },
16762
+ body: JSON.stringify({ data: { ...args, apiKey: userMemoryKey } })
16763
+ });
16764
+ const body = await res.json().catch(() => null);
16765
+ if (!res.ok) return { ok: false, error: body?.error ?? `memory ${toolName} failed (${res.status})` };
16766
+ if (body?.result) return body.result;
16767
+ return { ok: false, error: body?.error ?? `memory ${toolName} returned no result` };
16768
+ } catch (err) {
16769
+ return { ok: false, error: err?.message ?? "memory call failed" };
16770
+ }
16771
+ }
16772
+ function personalVault(user) {
16773
+ return `mcp-${user.id}`;
16774
+ }
16775
+ function memoryIdentity(user) {
16776
+ return user.email;
16777
+ }
16778
+ function resolveMemoryIdentity(user) {
16779
+ return memoryIdentity(user);
16780
+ }
16781
+ function memoryPlanForReads(user) {
16782
+ if (isMemoryOperator(user.email)) return "unlimited";
16783
+ if (user.memory_plan === "pro" || user.memory_plan === "team") return user.memory_plan;
16784
+ return "free";
16785
+ }
16786
+ async function provisionMemoryForUser(user) {
16787
+ try {
16788
+ const { key } = await getOrCreateUserMemoryKey(user);
16789
+ if (key) await setMemoryProvisioned(user.id, true);
16790
+ } catch {
16791
+ }
16792
+ }
16793
+ async function getOrCreateUserMemoryKey(user) {
16794
+ const creds = await getUserMemoryCreds(user.id);
16795
+ if (creds.memory_key) {
16796
+ const decrypted = decryptMemoryKey(creds.memory_key);
16797
+ if (decrypted) return { key: decrypted };
16798
+ }
16799
+ const admin = ADMIN_KEY();
16800
+ if (!admin) return { key: null, error: "memory not configured (MCP_MEMORY_ADMIN_KEY missing)" };
16801
+ const identity = memoryIdentity(user);
16802
+ const plan = isMemoryOperator(user.email) ? "unlimited" : (await getMemoryPlan(user.id)).plan;
16803
+ const provisioned = await memoryCall(
16804
+ "provisionDefaultsTool",
16805
+ { granteeIdentity: identity, issueKey: true, plan },
16806
+ admin
16807
+ );
16808
+ if (!provisioned.ok || !provisioned.secret) {
16809
+ return { key: null, error: provisioned.error ?? "failed to provision default vaults" };
16810
+ }
16811
+ await setUserMemoryCreds(user.id, encryptMemoryKey(provisioned.secret), identity);
16812
+ return { key: provisioned.secret };
16813
+ }
16814
+ async function syncMemoryKeyPlan(user, plan) {
16815
+ const effectivePlan = isMemoryOperator(user.email) ? "unlimited" : plan;
16816
+ const { key, error } = await getOrCreateUserMemoryKey(user);
16817
+ if (!key) return { ok: false, error: error ?? "memory unavailable" };
16818
+ const listed = await memoryCall(
16819
+ "listKeysTool",
16820
+ {},
16821
+ key
16822
+ );
16823
+ if (!listed.ok || !listed.keys?.length) return { ok: false, error: listed.error ?? "no memory keys found" };
16824
+ const target = listed.keys[0];
16825
+ if (target.plan === effectivePlan) return { ok: true };
16826
+ const res = await memoryCall("setScopeTool", { keyId: target.keyId, plan: effectivePlan }, key);
16827
+ return { ok: res.ok, error: res.error };
16828
+ }
16829
+
16830
+ // src/api/stripe-routes.ts
16457
16831
  var stripe = new Stripe(process.env.STRIPE_SECRET_KEY, { apiVersion: "2026-02-25.clover" });
16458
- var stripeApp = new Hono12();
16832
+ var stripeApp = new Hono13();
16459
16833
  stripeApp.post("/webhooks", async (c) => {
16460
16834
  const sig = c.req.header("stripe-signature");
16461
16835
  const body = await c.req.text();
@@ -16469,6 +16843,8 @@ stripeApp.post("/webhooks", async (c) => {
16469
16843
  if (!isNew) return c.json({ received: true });
16470
16844
  if (event.type === "invoice.paid" || event.type === "invoice.payment_succeeded") {
16471
16845
  const invoice = event.data.object;
16846
+ const memLineId = invoice.lines.data.map(linePriceId).find((id) => id && id in MEMORY_PLANS);
16847
+ if (memLineId) return c.json({ received: true });
16472
16848
  const lineTierId = invoice.lines.data.map(linePriceId).find((id) => id && id in SUBSCRIPTION_TIERS);
16473
16849
  if (lineTierId) {
16474
16850
  const tier = SUBSCRIPTION_TIERS[lineTierId];
@@ -16482,6 +16858,16 @@ stripeApp.post("/webhooks", async (c) => {
16482
16858
  if (event.type === "customer.subscription.created" || event.type === "customer.subscription.updated") {
16483
16859
  const sub = event.data.object;
16484
16860
  const priceId = sub.items.data[0]?.price?.id;
16861
+ if (priceId && priceId in MEMORY_PLANS) {
16862
+ const mem = MEMORY_PLANS[priceId];
16863
+ const user2 = await resolveUser(sub.customer, sub.customer_email);
16864
+ if (user2) {
16865
+ const live = sub.status === "active" || sub.status === "trialing";
16866
+ await setMemoryPlan(user2.id, live ? mem.plan : "free", live ? sub.id : null);
16867
+ await syncMemoryKeyPlan(user2, live ? mem.plan : "free");
16868
+ }
16869
+ return c.json({ received: true });
16870
+ }
16485
16871
  const user = await getUserByStripeCustomerId(sub.customer);
16486
16872
  if (!user) return c.json({ received: true });
16487
16873
  if (priceId && priceId in SUBSCRIPTION_TIERS) {
@@ -16496,6 +16882,14 @@ stripeApp.post("/webhooks", async (c) => {
16496
16882
  if (event.type === "customer.subscription.deleted") {
16497
16883
  const sub = event.data.object;
16498
16884
  const priceId = sub.items.data[0]?.price?.id;
16885
+ if (priceId && priceId in MEMORY_PLANS || await resolveUserByMemorySubscription(sub.id)) {
16886
+ const user2 = await resolveUserByMemorySubscription(sub.id) ?? await getUserByStripeCustomerId(sub.customer);
16887
+ if (user2) {
16888
+ await setMemoryPlan(user2.id, "free", null);
16889
+ await syncMemoryKeyPlan(user2, "free");
16890
+ }
16891
+ return c.json({ received: true });
16892
+ }
16499
16893
  const user = await getUserByStripeCustomerId(sub.customer);
16500
16894
  if (!user) return c.json({ received: true });
16501
16895
  if (priceId && priceId in SUBSCRIPTION_TIERS) {
@@ -16520,6 +16914,417 @@ async function resolveUser(customerId, emailFallback) {
16520
16914
  return user;
16521
16915
  }
16522
16916
 
16917
+ // src/api/oauth-routes.ts
16918
+ import { Hono as Hono14 } from "hono";
16919
+ import { getCookie, setCookie } from "hono/cookie";
16920
+ import { createHash as createHash3, randomBytes as randomBytes2, randomUUID as randomUUID3 } from "crypto";
16921
+ import { importPKCS8, exportJWK, SignJWT } from "jose";
16922
+ var ISSUER = "https://mcpscraper.dev";
16923
+ var RESOURCE = () => process.env.MCP_MEMORY_RESOURCE_URL ?? "https://mcp-memory-omega.vercel.app/mcp/thorbitMemoryServer/mcp";
16924
+ var SCRAPER_RESOURCE = () => (process.env.MCP_SCRAPER_RESOURCE_URL ?? "https://mcpscraper.dev/mcp").replace(/\/$/, "");
16925
+ var MEMORY_SCOPES = ["memory.read", "memory.write", "memory.admin"];
16926
+ var SCRAPER_SCOPES = ["scraper.use"];
16927
+ var SUPPORTED_SCOPES = [...MEMORY_SCOPES, ...SCRAPER_SCOPES];
16928
+ function isScraperResource(resource) {
16929
+ if (!resource) return false;
16930
+ return resource.replace(/\/$/, "") === SCRAPER_RESOURCE();
16931
+ }
16932
+ var ACCESS_TTL_SECONDS = 3600;
16933
+ var REFRESH_TTL_SECONDS = 60 * 60 * 24 * 30;
16934
+ var CODE_TTL_SECONDS = 60;
16935
+ var secureCookies = process.env.NODE_ENV === "production" || process.env.VERCEL === "1";
16936
+ var sessionCookieOptions = {
16937
+ httpOnly: true,
16938
+ secure: secureCookies,
16939
+ path: "/",
16940
+ maxAge: 60 * 60 * 24 * 30,
16941
+ sameSite: "Lax"
16942
+ };
16943
+ var cachedKeys = null;
16944
+ async function getKeys() {
16945
+ if (cachedKeys) return cachedKeys;
16946
+ const raw = process.env.OAUTH_SIGNING_KEY;
16947
+ if (!raw?.trim()) throw new Error("OAUTH_SIGNING_KEY is not set");
16948
+ const pem = raw.includes("BEGIN") ? raw : Buffer.from(raw, "base64").toString("utf8");
16949
+ const privateKey = await importPKCS8(pem, "RS256", { extractable: true });
16950
+ const full = await exportJWK(privateKey);
16951
+ const publicJwk = { kty: full.kty, n: full.n, e: full.e };
16952
+ const kid = createHash3("sha256").update(JSON.stringify({ e: publicJwk.e, kty: publicJwk.kty, n: publicJwk.n })).digest("base64url").slice(0, 16);
16953
+ publicJwk.kid = kid;
16954
+ publicJwk.alg = "RS256";
16955
+ publicJwk.use = "sig";
16956
+ cachedKeys = { privateKey, publicJwk, kid };
16957
+ return cachedKeys;
16958
+ }
16959
+ function isAllowedRedirect(uri) {
16960
+ let u;
16961
+ try {
16962
+ u = new URL(uri);
16963
+ } catch {
16964
+ return false;
16965
+ }
16966
+ if (u.protocol !== "https:") return false;
16967
+ const h = u.hostname.toLowerCase();
16968
+ if (["claude.ai", "www.claude.ai", "claude.com", "www.claude.com"].includes(h)) return true;
16969
+ if (h.endsWith(".claude.ai") || h.endsWith(".claude.com")) return true;
16970
+ const extra = (process.env.OAUTH_EXTRA_REDIRECT_HOSTS ?? "").split(",").map((s) => s.trim().toLowerCase()).filter(Boolean);
16971
+ return extra.includes(h);
16972
+ }
16973
+ function resolvePlan(user) {
16974
+ if (isMemoryOperator(user.email)) return "unlimited";
16975
+ if (user.memory_plan === "pro" || user.memory_plan === "team") return user.memory_plan;
16976
+ return "free";
16977
+ }
16978
+ function negotiateScope(requested, user, resource) {
16979
+ if (isScraperResource(resource)) return "scraper.use";
16980
+ const operator = isMemoryOperator(user.email);
16981
+ const wanted = (requested ?? "").split(/\s+/).map((s) => s.trim()).filter(Boolean);
16982
+ const filtered = wanted.filter((s) => MEMORY_SCOPES.includes(s));
16983
+ const base = filtered.length > 0 ? filtered : ["memory.read", "memory.write"];
16984
+ const allowed = base.filter((s) => s === "memory.admin" ? operator : true);
16985
+ const unique = [...new Set(allowed)];
16986
+ if (operator && !unique.includes("memory.admin")) unique.push("memory.admin");
16987
+ return unique.join(" ");
16988
+ }
16989
+ async function sessionUser(token) {
16990
+ if (!token) return null;
16991
+ const userId = verifySession(token);
16992
+ if (!userId) return null;
16993
+ const user = await getUserById(userId);
16994
+ return user ?? null;
16995
+ }
16996
+ function escapeHtml(value) {
16997
+ return value.replace(/&/g, "&amp;").replace(/</g, "&lt;").replace(/>/g, "&gt;").replace(/"/g, "&quot;").replace(/'/g, "&#39;");
16998
+ }
16999
+ function hiddenFields(p) {
17000
+ const fields = [
17001
+ ["response_type", p.response_type],
17002
+ ["client_id", p.client_id],
17003
+ ["redirect_uri", p.redirect_uri],
17004
+ ["code_challenge", p.code_challenge],
17005
+ ["code_challenge_method", p.code_challenge_method],
17006
+ ["scope", p.scope],
17007
+ ["state", p.state],
17008
+ ["resource", p.resource]
17009
+ ];
17010
+ return fields.map(([k, v]) => `<input type="hidden" name="${k}" value="${escapeHtml(v)}" />`).join("");
17011
+ }
17012
+ function scopeLabel(scope) {
17013
+ const map = {
17014
+ "memory.read": "Read your vaults, notes and search",
17015
+ "memory.write": "Create and edit notes in your vaults",
17016
+ "memory.admin": "Full administrative access",
17017
+ "scraper.use": "Run web scraping, search, maps and social tools (billed to your account)"
17018
+ };
17019
+ return scope.split(/\s+/).filter(Boolean).map((s) => map[s] ?? s).map((l) => `<li>${escapeHtml(l)}</li>`).join("");
17020
+ }
17021
+ function resourceLabel(resource) {
17022
+ return isScraperResource(resource) ? "MCP Scraper" : "mcp-memory";
17023
+ }
17024
+ function pageShell(inner) {
17025
+ return `<!doctype html><html lang="en"><head><meta charset="utf-8" /><meta name="viewport" content="width=device-width,initial-scale=1" /><title>Authorize Claude \xB7 mcp-memory</title><style>
17026
+ :root{color-scheme:dark}
17027
+ *{box-sizing:border-box}
17028
+ body{margin:0;min-height:100vh;display:flex;align-items:center;justify-content:center;background:#0b0d10;color:#e8eaed;font-family:-apple-system,BlinkMacSystemFont,"Segoe UI",Inter,system-ui,sans-serif;padding:24px}
17029
+ .card{width:100%;max-width:420px;background:#14171c;border:1px solid #232830;border-radius:16px;padding:32px}
17030
+ h1{font-size:20px;margin:0 0 4px}
17031
+ p.sub{color:#9aa3ad;font-size:14px;margin:0 0 24px}
17032
+ label{display:block;font-size:13px;color:#9aa3ad;margin:14px 0 6px}
17033
+ input[type=email],input[type=password]{width:100%;padding:11px 13px;background:#0b0d10;border:1px solid #2b313a;border-radius:9px;color:#e8eaed;font-size:14px}
17034
+ input:focus{outline:none;border-color:#5b8cff}
17035
+ button{width:100%;margin-top:20px;padding:12px;border:0;border-radius:9px;font-size:14px;font-weight:600;cursor:pointer}
17036
+ .primary{background:#5b8cff;color:#fff}
17037
+ .ghost{background:transparent;color:#9aa3ad;border:1px solid #2b313a;margin-top:10px}
17038
+ ul{margin:14px 0;padding-left:18px;color:#cbd2da;font-size:13px;line-height:1.7}
17039
+ .err{background:#3a1b1f;border:1px solid #6b2b30;color:#ffb4b4;padding:10px 12px;border-radius:9px;font-size:13px;margin-bottom:16px}
17040
+ .who{font-size:13px;color:#9aa3ad;margin-bottom:8px}
17041
+ </style></head><body><div class="card">${inner}</div></body></html>`;
17042
+ }
17043
+ function renderLogin(p, error) {
17044
+ return pageShell(`
17045
+ <h1>Sign in to ${escapeHtml(resourceLabel(p.resource))}</h1>
17046
+ <p class="sub">Authorize Claude to connect to your ${escapeHtml(resourceLabel(p.resource))} on mcpscraper.dev.</p>
17047
+ ${error ? `<div class="err">${escapeHtml(error)}</div>` : ""}
17048
+ <form method="post" action="/oauth/authorize">
17049
+ ${hiddenFields(p)}
17050
+ <input type="hidden" name="action" value="login" />
17051
+ <label for="email">Email</label>
17052
+ <input id="email" name="email" type="email" autocomplete="username" required />
17053
+ <label for="password">Password</label>
17054
+ <input id="password" name="password" type="password" autocomplete="current-password" required />
17055
+ <button class="primary" type="submit">Sign in</button>
17056
+ </form>`);
17057
+ }
17058
+ function renderConsent(p, email, error) {
17059
+ return pageShell(`
17060
+ <h1>Connect Claude to ${escapeHtml(resourceLabel(p.resource))}</h1>
17061
+ <p class="sub">Allow Claude to access your ${escapeHtml(resourceLabel(p.resource))}?</p>
17062
+ ${error ? `<div class="err">${escapeHtml(error)}</div>` : ""}
17063
+ <div class="who">Signed in as <strong>${escapeHtml(email)}</strong></div>
17064
+ <ul>${scopeLabel(p.scope)}</ul>
17065
+ <form method="post" action="/oauth/authorize">
17066
+ ${hiddenFields(p)}
17067
+ <input type="hidden" name="action" value="approve" />
17068
+ <button class="primary" type="submit">Allow</button>
17069
+ </form>
17070
+ <form method="post" action="/oauth/authorize">
17071
+ ${hiddenFields(p)}
17072
+ <input type="hidden" name="action" value="deny" />
17073
+ <button class="ghost" type="submit">Deny</button>
17074
+ </form>`);
17075
+ }
17076
+ function redirectWithError(redirectUri, state, error) {
17077
+ const u = new URL(redirectUri);
17078
+ u.searchParams.set("error", error);
17079
+ if (state) u.searchParams.set("state", state);
17080
+ return new Response(null, { status: 302, headers: { Location: u.toString() } });
17081
+ }
17082
+ var oauthApp = new Hono14();
17083
+ oauthApp.use("*", async (c, next) => {
17084
+ c.header("Access-Control-Allow-Origin", "*");
17085
+ c.header("Access-Control-Allow-Methods", "GET, POST, OPTIONS");
17086
+ c.header("Access-Control-Allow-Headers", "Content-Type, Authorization, mcp-protocol-version");
17087
+ if (c.req.method === "OPTIONS") return c.body(null, 204);
17088
+ return next();
17089
+ });
17090
+ function authServerMetadata() {
17091
+ return {
17092
+ issuer: ISSUER,
17093
+ authorization_endpoint: `${ISSUER}/oauth/authorize`,
17094
+ token_endpoint: `${ISSUER}/oauth/token`,
17095
+ registration_endpoint: `${ISSUER}/oauth/register`,
17096
+ jwks_uri: `${ISSUER}/.well-known/jwks.json`,
17097
+ scopes_supported: [...SUPPORTED_SCOPES],
17098
+ response_types_supported: ["code"],
17099
+ grant_types_supported: ["authorization_code", "refresh_token"],
17100
+ code_challenge_methods_supported: ["S256"],
17101
+ token_endpoint_auth_methods_supported: ["none"]
17102
+ };
17103
+ }
17104
+ function openidConfiguration() {
17105
+ return {
17106
+ ...authServerMetadata(),
17107
+ subject_types_supported: ["public"],
17108
+ id_token_signing_alg_values_supported: ["RS256"]
17109
+ };
17110
+ }
17111
+ oauthApp.get("/.well-known/oauth-authorization-server", (c) => {
17112
+ console.log("[oauth-dcr] GET /.well-known/oauth-authorization-server ua=%s", c.req.header("user-agent") ?? "");
17113
+ return c.json(authServerMetadata());
17114
+ });
17115
+ oauthApp.get("/.well-known/oauth-authorization-server/*", (c) => {
17116
+ console.log("[oauth-dcr] GET %s (path-aware AS) ua=%s", c.req.path, c.req.header("user-agent") ?? "");
17117
+ return c.json(authServerMetadata());
17118
+ });
17119
+ oauthApp.get("/.well-known/openid-configuration", (c) => {
17120
+ console.log("[oauth-dcr] GET /.well-known/openid-configuration ua=%s", c.req.header("user-agent") ?? "");
17121
+ return c.json(openidConfiguration());
17122
+ });
17123
+ oauthApp.get("/.well-known/openid-configuration/*", (c) => {
17124
+ console.log("[oauth-dcr] GET %s (path-aware OIDC) ua=%s", c.req.path, c.req.header("user-agent") ?? "");
17125
+ return c.json(openidConfiguration());
17126
+ });
17127
+ oauthApp.get("/.well-known/jwks.json", async (c) => {
17128
+ const { publicJwk } = await getKeys();
17129
+ c.header("Cache-Control", "public, max-age=3600");
17130
+ return c.json({ keys: [publicJwk] });
17131
+ });
17132
+ oauthApp.get("/.well-known/oauth-protected-resource", (c) => {
17133
+ return c.json(protectedResourceMetadata());
17134
+ });
17135
+ oauthApp.get("/.well-known/oauth-protected-resource/*", (c) => {
17136
+ return c.json(protectedResourceMetadata());
17137
+ });
17138
+ oauthApp.post("/oauth/register", async (c) => {
17139
+ const rawBody = await c.req.text();
17140
+ console.log("[oauth-dcr] POST /oauth/register ua=%s ct=%s body=%s", c.req.header("user-agent") ?? "", c.req.header("content-type") ?? "", rawBody.slice(0, 2e3));
17141
+ let body;
17142
+ try {
17143
+ body = rawBody ? JSON.parse(rawBody) : null;
17144
+ } catch {
17145
+ body = null;
17146
+ }
17147
+ if (!body) {
17148
+ console.log("[oauth-dcr] register REJECT reason=body_not_json");
17149
+ return c.json({ error: "invalid_client_metadata", error_description: "Body must be JSON" }, 400);
17150
+ }
17151
+ console.log("[oauth-dcr] register body keys=%s redirect_uris=%s", Object.keys(body).join(","), JSON.stringify(body.redirect_uris));
17152
+ const redirectUris = Array.isArray(body.redirect_uris) ? body.redirect_uris.filter((u) => typeof u === "string") : [];
17153
+ if (redirectUris.length === 0) {
17154
+ console.log("[oauth-dcr] register REJECT reason=no_redirect_uris");
17155
+ return c.json({ error: "invalid_redirect_uri", error_description: "redirect_uris is required" }, 400);
17156
+ }
17157
+ for (const uri of redirectUris) {
17158
+ if (!isAllowedRedirect(uri)) {
17159
+ console.log("[oauth-dcr] register REJECT reason=redirect_uri_not_allowed uri=%s", uri);
17160
+ return c.json({ error: "invalid_redirect_uri", error_description: `redirect_uri not allowed: ${uri}` }, 400);
17161
+ }
17162
+ }
17163
+ const clientName = typeof body.client_name === "string" ? body.client_name : null;
17164
+ const clientId = `client_${randomBytes2(16).toString("hex")}`;
17165
+ await registerClient(clientId, redirectUris, clientName);
17166
+ console.log("[oauth-dcr] register OK client_id=%s redirect_uris=%s", clientId, JSON.stringify(redirectUris));
17167
+ return c.json({
17168
+ client_id: clientId,
17169
+ redirect_uris: redirectUris,
17170
+ client_name: clientName ?? void 0,
17171
+ token_endpoint_auth_method: "none",
17172
+ grant_types: ["authorization_code", "refresh_token"],
17173
+ response_types: ["code"],
17174
+ client_id_issued_at: Math.floor(Date.now() / 1e3)
17175
+ }, 201);
17176
+ });
17177
+ function parseAuthParams(get) {
17178
+ return {
17179
+ response_type: get("response_type") ?? "",
17180
+ client_id: get("client_id") ?? "",
17181
+ redirect_uri: get("redirect_uri") ?? "",
17182
+ code_challenge: get("code_challenge") ?? "",
17183
+ code_challenge_method: get("code_challenge_method") ?? "",
17184
+ scope: get("scope") ?? "",
17185
+ state: get("state") ?? "",
17186
+ resource: get("resource") ?? ""
17187
+ };
17188
+ }
17189
+ async function validateAuthRequest(p) {
17190
+ if (p.response_type !== "code") return { ok: false, status: 400, message: "response_type must be code" };
17191
+ if (!p.client_id) return { ok: false, status: 400, message: "client_id is required" };
17192
+ const client2 = await getClient(p.client_id);
17193
+ if (!client2) return { ok: false, status: 400, message: "unknown client_id" };
17194
+ if (!p.redirect_uri || !client2.redirect_uris.includes(p.redirect_uri)) return { ok: false, status: 400, message: "redirect_uri does not match the registered set" };
17195
+ if (!isAllowedRedirect(p.redirect_uri)) return { ok: false, status: 400, message: "redirect_uri host not allowed" };
17196
+ if (!p.code_challenge || p.code_challenge_method !== "S256") return { ok: false, status: 400, message: "PKCE S256 code_challenge is required" };
17197
+ return { ok: true };
17198
+ }
17199
+ oauthApp.get("/oauth/authorize", async (c) => {
17200
+ const p = parseAuthParams((k) => c.req.query(k));
17201
+ console.log("[oauth-dcr] GET /oauth/authorize client_id=%s redirect_uri=%s scope=%s resource=%s", p.client_id, p.redirect_uri, p.scope, p.resource);
17202
+ const valid = await validateAuthRequest(p);
17203
+ if (!valid.ok) {
17204
+ console.log("[oauth-dcr] authorize REJECT reason=%s", valid.message);
17205
+ return c.text(valid.message, valid.status);
17206
+ }
17207
+ const user = await sessionUser(getCookie(c, "session"));
17208
+ if (!user) return c.html(renderLogin(p));
17209
+ const negotiated = { ...p, scope: negotiateScope(p.scope, user, p.resource) };
17210
+ return c.html(renderConsent(negotiated, user.email));
17211
+ });
17212
+ oauthApp.post("/oauth/authorize", async (c) => {
17213
+ const form = await c.req.parseBody();
17214
+ const get = (k) => {
17215
+ const v = form[k];
17216
+ return typeof v === "string" ? v : void 0;
17217
+ };
17218
+ const p = parseAuthParams(get);
17219
+ const action = get("action") ?? "";
17220
+ const valid = await validateAuthRequest(p);
17221
+ if (!valid.ok) return c.text(valid.message, valid.status);
17222
+ if (action === "login") {
17223
+ const email = (get("email") ?? "").trim().toLowerCase();
17224
+ const password = get("password") ?? "";
17225
+ const user2 = email ? await getUserByEmail(email) : void 0;
17226
+ const okPw = !!(user2 && user2.password_hash && verifyPassword(password, user2.password_hash));
17227
+ console.log("[oauth-dcr] LOGIN email=%s found=%s hasHash=%s pwLen=%d ok=%s", email, !!user2, !!(user2 && user2.password_hash), password.length, okPw);
17228
+ if (!okPw) {
17229
+ return c.html(renderLogin(p, "Invalid email or password."));
17230
+ }
17231
+ setCookie(c, "session", signSession(user2.id), sessionCookieOptions);
17232
+ return c.html(renderConsent({ ...p, scope: negotiateScope(p.scope, user2, p.resource) }, user2.email));
17233
+ }
17234
+ const user = await sessionUser(getCookie(c, "session"));
17235
+ if (!user) return c.html(renderLogin(p, "Your session expired. Please sign in again."));
17236
+ if (action === "deny") return redirectWithError(p.redirect_uri, p.state, "access_denied");
17237
+ if (action !== "approve") return c.text("unsupported action", 400);
17238
+ const scope = negotiateScope(p.scope, user, p.resource);
17239
+ const code = `code_${randomBytes2(32).toString("base64url")}`;
17240
+ const expiresAt = new Date(Date.now() + CODE_TTL_SECONDS * 1e3).toISOString();
17241
+ await putCode({
17242
+ code,
17243
+ client_id: p.client_id,
17244
+ redirect_uri: p.redirect_uri,
17245
+ code_challenge: p.code_challenge,
17246
+ identity: user.email,
17247
+ scope,
17248
+ resource: p.resource || RESOURCE(),
17249
+ expires_at: expiresAt
17250
+ });
17251
+ const u = new URL(p.redirect_uri);
17252
+ u.searchParams.set("code", code);
17253
+ if (p.state) u.searchParams.set("state", p.state);
17254
+ return new Response(null, { status: 302, headers: { Location: u.toString() } });
17255
+ });
17256
+ function pkceMatches(verifier, challenge) {
17257
+ if (!verifier) return false;
17258
+ const computed = createHash3("sha256").update(verifier).digest("base64url");
17259
+ return computed === challenge;
17260
+ }
17261
+ async function mintAccessToken(identity, scope, plan, audience) {
17262
+ const { privateKey, kid } = await getKeys();
17263
+ return new SignJWT({ scope, plan }).setProtectedHeader({ alg: "RS256", kid }).setIssuer(ISSUER).setSubject(identity).setAudience(audience).setIssuedAt().setJti(randomUUID3()).setExpirationTime(`${ACCESS_TTL_SECONDS}s`).sign(privateKey);
17264
+ }
17265
+ function tokenErrorResponse(c, error, description, status) {
17266
+ return c.json({ error, error_description: description }, status);
17267
+ }
17268
+ oauthApp.post("/oauth/token", async (c) => {
17269
+ const form = await c.req.parseBody();
17270
+ const get = (k) => {
17271
+ const v = form[k];
17272
+ return typeof v === "string" ? v : void 0;
17273
+ };
17274
+ const grantType = get("grant_type");
17275
+ if (grantType === "authorization_code") {
17276
+ const code = get("code") ?? "";
17277
+ const clientId = get("client_id") ?? "";
17278
+ const redirectUri = get("redirect_uri") ?? "";
17279
+ const verifier = get("code_verifier") ?? "";
17280
+ const record = await consumeCode(code);
17281
+ if (!record) return tokenErrorResponse(c, "invalid_grant", "Authorization code is invalid or already used", 400);
17282
+ if (new Date(record.expires_at).getTime() < Date.now()) return tokenErrorResponse(c, "invalid_grant", "Authorization code has expired", 400);
17283
+ if (record.client_id !== clientId) return tokenErrorResponse(c, "invalid_grant", "client_id mismatch", 400);
17284
+ if (record.redirect_uri !== redirectUri) return tokenErrorResponse(c, "invalid_grant", "redirect_uri mismatch", 400);
17285
+ if (!pkceMatches(verifier, record.code_challenge)) return tokenErrorResponse(c, "invalid_grant", "PKCE verification failed", 400);
17286
+ const user = await getUserByEmail(record.identity);
17287
+ const plan = user ? resolvePlan(user) : "free";
17288
+ const audience = record.resource ?? RESOURCE();
17289
+ const accessToken = await mintAccessToken(record.identity, record.scope, plan, audience);
17290
+ const refreshToken = `rt_${randomBytes2(40).toString("base64url")}`;
17291
+ await putRefresh({
17292
+ refresh_token: refreshToken,
17293
+ client_id: clientId,
17294
+ identity: record.identity,
17295
+ scope: record.scope,
17296
+ resource: audience,
17297
+ expires_at: new Date(Date.now() + REFRESH_TTL_SECONDS * 1e3).toISOString(),
17298
+ revoked: 0
17299
+ });
17300
+ return c.json({ access_token: accessToken, token_type: "Bearer", expires_in: ACCESS_TTL_SECONDS, refresh_token: refreshToken, scope: record.scope });
17301
+ }
17302
+ if (grantType === "refresh_token") {
17303
+ const refreshToken = get("refresh_token") ?? "";
17304
+ const clientId = get("client_id") ?? "";
17305
+ const record = await getRefresh(refreshToken);
17306
+ if (!record || record.revoked) return tokenErrorResponse(c, "invalid_grant", "Refresh token is invalid or revoked", 400);
17307
+ if (new Date(record.expires_at).getTime() < Date.now()) return tokenErrorResponse(c, "invalid_grant", "Refresh token has expired", 400);
17308
+ if (clientId && record.client_id !== clientId) return tokenErrorResponse(c, "invalid_grant", "client_id mismatch", 400);
17309
+ const user = await getUserByEmail(record.identity);
17310
+ const plan = user ? resolvePlan(user) : "free";
17311
+ const audience = record.resource ?? RESOURCE();
17312
+ const accessToken = await mintAccessToken(record.identity, record.scope, plan, audience);
17313
+ const nextRefresh = `rt_${randomBytes2(40).toString("base64url")}`;
17314
+ await rotateRefresh(refreshToken, {
17315
+ refresh_token: nextRefresh,
17316
+ client_id: record.client_id,
17317
+ identity: record.identity,
17318
+ scope: record.scope,
17319
+ resource: audience,
17320
+ expires_at: new Date(Date.now() + REFRESH_TTL_SECONDS * 1e3).toISOString(),
17321
+ revoked: 0
17322
+ });
17323
+ return c.json({ access_token: accessToken, token_type: "Bearer", expires_in: ACCESS_TTL_SECONDS, refresh_token: nextRefresh, scope: record.scope });
17324
+ }
17325
+ return tokenErrorResponse(c, "unsupported_grant_type", "grant_type must be authorization_code or refresh_token", 400);
17326
+ });
17327
+
16523
17328
  // src/api/site-audit-worker.ts
16524
17329
  var MAX_CONCURRENT_SITE_AUDIT = 1;
16525
17330
  async function drainSiteAuditQueue(budget) {
@@ -16555,12 +17360,115 @@ function startSiteAuditWorker() {
16555
17360
 
16556
17361
  // src/api/server.ts
16557
17362
  import { createMiddleware as createMiddleware3 } from "hono/factory";
16558
- import { getCookie, setCookie, deleteCookie } from "hono/cookie";
17363
+ import { getCookie as getCookie2, setCookie as setCookie2, deleteCookie } from "hono/cookie";
16559
17364
  import Stripe2 from "stripe";
16560
17365
 
17366
+ // src/api/scrape-vault-sink.ts
17367
+ var MAX_DEPOSIT_CHARS = 5e6;
17368
+ var SCRAPE_FALLBACK_PREFIX = "scrape-fallback/";
17369
+ var SCRAPE_BLOB_TTL_MS = 24 * 60 * 60 * 1e3;
17370
+ async function depositScrapeToVault(user, opts) {
17371
+ try {
17372
+ const content = (opts.content ?? "").trim();
17373
+ if (!content) return { deposited: false, error: "empty content" };
17374
+ const clipped = content.length > MAX_DEPOSIT_CHARS ? content.slice(0, MAX_DEPOSIT_CHARS) : content;
17375
+ const { key, error } = await getOrCreateUserMemoryKey(user);
17376
+ if (!key) return { deposited: false, error: error ?? "memory unavailable" };
17377
+ const vault = opts.vault?.trim() || personalVault(user);
17378
+ const title = (opts.title?.trim() || opts.source).slice(0, 200);
17379
+ const res = await memoryCall(
17380
+ "libraryIngestTool",
17381
+ { title, content: clipped, source: opts.source, vault },
17382
+ key
17383
+ );
17384
+ if (!res.ok) return { deposited: false, vault, error: res.error ?? "ingest failed" };
17385
+ return { deposited: true, vault, noteId: res.noteId, path: res.path, chunks: res.chunks };
17386
+ } catch (err) {
17387
+ return { deposited: false, error: err?.message ?? "deposit failed" };
17388
+ }
17389
+ }
17390
+ async function uploadFallbackBlob(opts) {
17391
+ try {
17392
+ const content = (opts.content ?? "").trim();
17393
+ if (!content) return null;
17394
+ const slug = (slugify(opts.title || opts.source) || "scrape").slice(0, 60);
17395
+ const key = `${SCRAPE_FALLBACK_PREFIX}${slug}.md`;
17396
+ const doc = `# ${opts.title}
17397
+ Source: ${opts.source}
17398
+
17399
+ ${content}`;
17400
+ const stored = await getBlobStore().put(key, doc, "text/markdown");
17401
+ return { fileUrl: stored.url, fileExpiresAt: new Date(Date.now() + SCRAPE_BLOB_TTL_MS).toISOString() };
17402
+ } catch {
17403
+ return null;
17404
+ }
17405
+ }
17406
+ async function persistScrapeBody(user, opts) {
17407
+ const deposit = await depositScrapeToVault(user, opts);
17408
+ if (deposit.deposited) return deposit;
17409
+ const fallback = await uploadFallbackBlob(opts);
17410
+ if (!fallback) return deposit;
17411
+ return { ...deposit, ...fallback };
17412
+ }
17413
+
17414
+ // src/api/scrape-blob-cleanup.ts
17415
+ import { readdir, stat, unlink } from "fs/promises";
17416
+ import { homedir as homedir4 } from "os";
17417
+ import { join as join7 } from "path";
17418
+ async function cleanupVercel(token, cutoff) {
17419
+ const { list, del } = await import("@vercel/blob");
17420
+ let cursor;
17421
+ let deleted = 0;
17422
+ do {
17423
+ const res = await list({ prefix: SCRAPE_FALLBACK_PREFIX, cursor, token });
17424
+ const expired = res.blobs.filter((b) => new Date(b.uploadedAt).getTime() < cutoff);
17425
+ if (expired.length) {
17426
+ await del(expired.map((b) => b.url), { token });
17427
+ deleted += expired.length;
17428
+ }
17429
+ cursor = res.hasMore ? res.cursor : void 0;
17430
+ } while (cursor);
17431
+ return { deleted, store: "vercel-blob" };
17432
+ }
17433
+ async function cleanupLocal(cutoff) {
17434
+ const baseDir = process.env.MCP_SCRAPER_OUTPUT_DIR?.trim() || join7(homedir4(), "Downloads", "mcp-scraper");
17435
+ const dir = join7(baseDir, "blobs", SCRAPE_FALLBACK_PREFIX.replace(/\/$/, ""));
17436
+ let deleted = 0;
17437
+ let entries;
17438
+ try {
17439
+ entries = await readdir(dir);
17440
+ } catch {
17441
+ return { deleted: 0, store: "local" };
17442
+ }
17443
+ for (const name of entries) {
17444
+ const path5 = join7(dir, name);
17445
+ try {
17446
+ const s = await stat(path5);
17447
+ if (s.isFile() && s.mtimeMs < cutoff) {
17448
+ await unlink(path5);
17449
+ deleted++;
17450
+ }
17451
+ } catch {
17452
+ }
17453
+ }
17454
+ return { deleted, store: "local" };
17455
+ }
17456
+ async function cleanupExpiredScrapeBlobs(maxAgeMs = SCRAPE_BLOB_TTL_MS) {
17457
+ const cutoff = Date.now() - maxAgeMs;
17458
+ try {
17459
+ const token = process.env.BLOB_READ_WRITE_TOKEN;
17460
+ return token ? await cleanupVercel(token, cutoff) : await cleanupLocal(cutoff);
17461
+ } catch {
17462
+ return { deleted: 0, store: "none" };
17463
+ }
17464
+ }
17465
+
16561
17466
  // src/api/credit-operations.ts
16562
- async function grantSignupCredit(_userId) {
16563
- return;
17467
+ async function grantSignupCredit(userId) {
17468
+ if (!FREE_SIGNUP_MC || FREE_SIGNUP_MC <= 0) return;
17469
+ const already = await ledgerExistsForOperation(userId, LedgerOperation.SIGNUP_GRANT);
17470
+ if (already) return;
17471
+ await creditMc(userId, FREE_SIGNUP_MC, LedgerOperation.SIGNUP_GRANT, "Welcome credits");
16564
17472
  }
16565
17473
  async function applyMonthlyFreeRefresh(user) {
16566
17474
  return user;
@@ -16598,46 +17506,168 @@ async function getFreeCreditBreakdown(userId) {
16598
17506
  };
16599
17507
  }
16600
17508
 
16601
- // src/api/session.ts
16602
- import { createHmac as createHmac2, timingSafeEqual } from "crypto";
16603
- var isProduction = () => process.env.NODE_ENV === "production" || process.env.VERCEL === "1";
16604
- function getSessionSecret() {
16605
- const configured = process.env.SESSION_SECRET?.trim();
16606
- if (configured) return configured;
16607
- if (isProduction()) throw new Error("SESSION_SECRET is not set \u2014 add it to your Vercel env vars (see src/api/env.ts)");
16608
- return "dev-secret-change-me";
17509
+ // src/api/memory-db.ts
17510
+ import { neon } from "@neondatabase/serverless";
17511
+ var _sql = null;
17512
+ function sql() {
17513
+ if (_sql) return _sql;
17514
+ const url = process.env.MEMORY_DATABASE_URL?.trim();
17515
+ if (!url) throw new Error("MEMORY_DATABASE_URL is not set (paa-crawler needs mcp-memory Neon URL for reads)");
17516
+ _sql = neon(url);
17517
+ return _sql;
17518
+ }
17519
+ function physicalVault(owner, vault) {
17520
+ return `${owner} ${vault}`;
17521
+ }
17522
+ function isUnlimited(plan) {
17523
+ return plan === "unlimited";
17524
+ }
17525
+ function quotaBytesForPlan(plan) {
17526
+ return MEMORY_PLAN_QUOTA[plan] ?? MEMORY_PLAN_QUOTA.free;
17527
+ }
17528
+ function toGb(bytes) {
17529
+ return Math.round(bytes / 1e9 * 100) / 100;
17530
+ }
17531
+ async function entitledVaultRows(identity) {
17532
+ const rows = await sql().query(
17533
+ `SELECT vault, owner_identity FROM mem_vault_registry WHERE grantee = $1 ORDER BY created_at ASC`,
17534
+ [identity]
17535
+ );
17536
+ const seen = /* @__PURE__ */ new Map();
17537
+ for (const r of rows) {
17538
+ if (seen.has(r.vault)) continue;
17539
+ seen.set(r.vault, { vault: r.vault, owner: r.owner_identity, physical: physicalVault(r.owner_identity, r.vault) });
17540
+ }
17541
+ return [...seen.values()];
17542
+ }
17543
+ async function storageByPhysical(physicals) {
17544
+ if (physicals.length === 0) return /* @__PURE__ */ new Map();
17545
+ const rows = await sql().query(
17546
+ `SELECT vault, COUNT(*)::int AS notes, COALESCE(SUM(octet_length(content)), 0)::bigint AS bytes
17547
+ FROM mem_notes WHERE vault = ANY($1) GROUP BY vault`,
17548
+ [physicals]
17549
+ );
17550
+ const m = /* @__PURE__ */ new Map();
17551
+ for (const r of rows) m.set(r.vault, { notes: Number(r.notes), bytes: Number(r.bytes) });
17552
+ return m;
17553
+ }
17554
+ async function dbListVaults(identity) {
17555
+ const ents = await entitledVaultRows(identity);
17556
+ const storage = await storageByPhysical(ents.map((e) => e.physical));
17557
+ return ents.map((e) => {
17558
+ const st = storage.get(e.physical);
17559
+ const isOwner = e.owner === identity;
17560
+ return {
17561
+ vault: e.vault,
17562
+ role: isOwner ? "owner" : "shared",
17563
+ ...isOwner ? {} : { sharedBy: e.owner },
17564
+ notes: st?.notes ?? 0,
17565
+ bytes: st?.bytes ?? 0
17566
+ };
17567
+ });
16609
17568
  }
16610
- var secret = () => getSessionSecret();
16611
- function safeEqualHex(a, b) {
16612
- if (a.length !== b.length) return false;
16613
- try {
16614
- return timingSafeEqual(Buffer.from(a, "hex"), Buffer.from(b, "hex"));
16615
- } catch {
16616
- return false;
16617
- }
17569
+ async function dbStorage(identity, plan) {
17570
+ const ents = await entitledVaultRows(identity);
17571
+ const storage = await storageByPhysical(ents.map((e) => e.physical));
17572
+ let totalBytes = 0;
17573
+ const perVault = ents.map((e) => {
17574
+ const st = storage.get(e.physical);
17575
+ const bytes = st?.bytes ?? 0;
17576
+ totalBytes += bytes;
17577
+ return { vault: e.vault, bytes, gb: toGb(bytes), notes: st?.notes ?? 0 };
17578
+ });
17579
+ const unlimited = isUnlimited(plan);
17580
+ const quotaBytes = unlimited ? null : quotaBytesForPlan(plan);
17581
+ return {
17582
+ ok: true,
17583
+ totalBytes,
17584
+ totalGb: toGb(totalBytes),
17585
+ quotaBytes,
17586
+ quotaGb: quotaBytes == null ? null : toGb(quotaBytes),
17587
+ unlimited,
17588
+ perVault
17589
+ };
16618
17590
  }
16619
- function signSession(userId) {
16620
- const payload = String(userId);
16621
- const sig = createHmac2("sha256", secret()).update(payload).digest("hex");
16622
- return `${payload}.${sig}`;
17591
+ async function resolvePhysical(identity, vault) {
17592
+ const rows = await sql().query(
17593
+ `SELECT owner_identity FROM mem_vault_registry WHERE grantee = $1 AND vault = $2 LIMIT 1`,
17594
+ [identity, vault]
17595
+ );
17596
+ if (!rows[0]) return null;
17597
+ return physicalVault(rows[0].owner_identity, vault);
17598
+ }
17599
+ async function dbNotes(identity, vault) {
17600
+ const physical = await resolvePhysical(identity, vault);
17601
+ if (!physical) return [];
17602
+ const rows = await sql().query(
17603
+ `SELECT path, title, kind, source, updated_at, octet_length(content) AS bytes, props
17604
+ FROM mem_notes WHERE vault = $1 ORDER BY updated_at DESC`,
17605
+ [physical]
17606
+ );
17607
+ return rows.map((r) => ({
17608
+ path: r.path,
17609
+ title: r.title,
17610
+ kind: r.kind,
17611
+ source: r.source,
17612
+ updatedAt: r.updated_at,
17613
+ bytes: Number(r.bytes),
17614
+ props: r.props && typeof r.props === "object" ? r.props : {}
17615
+ }));
16623
17616
  }
16624
- function verifySession(token) {
16625
- const dot = token.lastIndexOf(".");
16626
- if (dot === -1) return null;
16627
- const payload = token.slice(0, dot);
16628
- const sig = token.slice(dot + 1);
16629
- const expected = createHmac2("sha256", secret()).update(payload).digest("hex");
16630
- if (!safeEqualHex(sig, expected)) return null;
16631
- const id = parseInt(payload);
16632
- return isNaN(id) ? null : id;
17617
+ async function dbNote(identity, vault, path5) {
17618
+ const physical = await resolvePhysical(identity, vault);
17619
+ if (!physical) return null;
17620
+ const rows = await sql().query(
17621
+ `SELECT path, title, content, kind, source, updated_at, captured_at
17622
+ FROM mem_notes WHERE vault = $1 AND path = $2 LIMIT 1`,
17623
+ [physical, path5]
17624
+ );
17625
+ const r = rows[0];
17626
+ if (!r) return null;
17627
+ return {
17628
+ path: r.path,
17629
+ title: r.title,
17630
+ content: r.content,
17631
+ kind: r.kind,
17632
+ source: r.source,
17633
+ updatedAt: r.updated_at,
17634
+ capturedAt: r.captured_at
17635
+ };
17636
+ }
17637
+ var FREE_COST_CAP_USD = 1;
17638
+ async function dbUsage(identity, plan) {
17639
+ const now = /* @__PURE__ */ new Date();
17640
+ const period = `${now.getUTCFullYear()}-${String(now.getUTCMonth() + 1).padStart(2, "0")}`;
17641
+ const rows = await sql().query(
17642
+ `SELECT source, COALESCE(SUM(cost_usd), 0)::float8 AS total
17643
+ FROM mem_usage_ledger WHERE identity = $1 AND period = $2 GROUP BY source`,
17644
+ [identity, period]
17645
+ );
17646
+ const bySource = {};
17647
+ let costUsd = 0;
17648
+ for (const r of rows) {
17649
+ const v = Number(r.total);
17650
+ bySource[r.source] = v;
17651
+ costUsd += v;
17652
+ }
17653
+ return {
17654
+ ok: true,
17655
+ period,
17656
+ plan,
17657
+ costUsd,
17658
+ bySource,
17659
+ freeCapUsd: FREE_COST_CAP_USD,
17660
+ freeCapReached: plan === "free" && costUsd >= FREE_COST_CAP_USD,
17661
+ unlimited: isUnlimited(plan)
17662
+ };
16633
17663
  }
16634
17664
 
16635
17665
  // src/api/server.ts
16636
- var secureCookies = process.env.NODE_ENV === "production" || process.env.VERCEL === "1";
16637
- var isProduction2 = secureCookies;
16638
- var sessionCookieOptions = {
17666
+ var secureCookies2 = process.env.NODE_ENV === "production" || process.env.VERCEL === "1";
17667
+ var isProduction2 = secureCookies2;
17668
+ var sessionCookieOptions2 = {
16639
17669
  httpOnly: true,
16640
- secure: secureCookies,
17670
+ secure: secureCookies2,
16641
17671
  path: "/",
16642
17672
  maxAge: 60 * 60 * 24 * 30,
16643
17673
  sameSite: "Strict"
@@ -16698,7 +17728,7 @@ var adminAuth = createMiddleware3(async (c, next) => {
16698
17728
  return next();
16699
17729
  });
16700
17730
  var sessionAuth = createMiddleware3(async (c, next) => {
16701
- const token = getCookie(c, "session");
17731
+ const token = getCookie2(c, "session");
16702
17732
  if (!token) return c.json({ error: "Not authenticated" }, 401);
16703
17733
  const userId = verifySession(token);
16704
17734
  if (!userId) return c.json({ error: "Session expired" }, 401);
@@ -16709,7 +17739,7 @@ var sessionAuth = createMiddleware3(async (c, next) => {
16709
17739
  c.set("sessionUser", { ...refreshed, balance_mc: balanceMc });
16710
17740
  return next();
16711
17741
  });
16712
- var app = new Hono13();
17742
+ var app = new Hono15();
16713
17743
  var STRIPE_API_VERSION = "2026-02-25.clover";
16714
17744
  function requireStripeSecret() {
16715
17745
  const secret2 = process.env.STRIPE_SECRET_KEY?.trim();
@@ -16737,6 +17767,33 @@ app.use("*", async (c, next) => {
16737
17767
  c.header("X-Content-Type-Options", "nosniff");
16738
17768
  c.header("Referrer-Policy", "same-origin");
16739
17769
  });
17770
+ function opForCostPath(p) {
17771
+ if (p === "/harvest" || p === "/harvest/sync") return "harvest";
17772
+ if (p === "/extract-url") return "page_scrape";
17773
+ if (p === "/extract-site") return "extract_site";
17774
+ if (p === "/map-urls") return "url_map";
17775
+ if (p === "/youtube/harvest") return "yt_channel";
17776
+ if (p === "/youtube/transcribe") return "yt_transcription";
17777
+ if (p === "/maps/search") return "maps_search";
17778
+ if (p === "/maps/place") return "maps_place";
17779
+ if (p.startsWith("/facebook/")) {
17780
+ if (p.endsWith("transcribe")) return "fb_transcribe";
17781
+ if (p.endsWith("/search")) return "fb_search";
17782
+ return "fb_ad";
17783
+ }
17784
+ if (p.startsWith("/instagram/")) return "instagram";
17785
+ if (p.startsWith("/api/internal/site-architecture-auditor")) return "audit_site";
17786
+ if (p.startsWith("/agent/")) return "browser_session";
17787
+ return null;
17788
+ }
17789
+ app.use("*", (c, next) => {
17790
+ const op = opForCostPath(new URL(c.req.url).pathname);
17791
+ if (!op) return next();
17792
+ const probeRunId = c.req.header("x-cost-probe-id") ?? null;
17793
+ const forceHeadless = c.req.header("x-kernel-headless") === "true";
17794
+ return runWithCostContext({ op, probeRunId, forceHeadless }, () => next());
17795
+ });
17796
+ void migrateCostTelemetry();
16740
17797
  app.post("/auth/register", requireAllowedOrigin, async (c) => {
16741
17798
  const { email, password } = await c.req.json();
16742
17799
  const normalizedEmail = email?.trim().toLowerCase();
@@ -16768,8 +17825,16 @@ app.post("/auth/register", requireAllowedOrigin, async (c) => {
16768
17825
  }
16769
17826
  const newKey = await rotateApiKey(user.id);
16770
17827
  await grantSignupCredit(user.id);
17828
+ void (async () => {
17829
+ try {
17830
+ const full = await getUserById(user.id);
17831
+ if (full) await provisionMemoryForUser(full);
17832
+ } catch (err) {
17833
+ console.warn("[auth/register] memory provision failed (will retry on first write):", err instanceof Error ? err.message : String(err));
17834
+ }
17835
+ })();
16771
17836
  const token = signSession(user.id);
16772
- setCookie(c, "session", token, sessionCookieOptions);
17837
+ setCookie2(c, "session", token, sessionCookieOptions2);
16773
17838
  return c.json({ email: user.email, api_key: newKey }, 201);
16774
17839
  } catch (err) {
16775
17840
  const msg = err instanceof Error ? err.message : "";
@@ -16788,11 +17853,11 @@ app.post("/auth/login", requireAllowedOrigin, async (c) => {
16788
17853
  return c.json({ error: "Invalid email or password" }, 401);
16789
17854
  }
16790
17855
  const token = signSession(user.id);
16791
- setCookie(c, "session", token, sessionCookieOptions);
17856
+ setCookie2(c, "session", token, sessionCookieOptions2);
16792
17857
  return c.json({ email: user.email, api_key: user.key_active ? user.api_key : null });
16793
17858
  });
16794
17859
  app.post("/auth/logout", requireAllowedOrigin, (c) => {
16795
- deleteCookie(c, "session", { path: "/", secure: secureCookies, sameSite: "Strict" });
17860
+ deleteCookie(c, "session", { path: "/", secure: secureCookies2, sameSite: "Strict" });
16796
17861
  return c.json({ ok: true });
16797
17862
  });
16798
17863
  app.post("/auth/forgot-password", requireAllowedOrigin, async (c) => {
@@ -16846,16 +17911,16 @@ app.get("/catalog", (c) => {
16846
17911
  return c.json(CATALOG);
16847
17912
  });
16848
17913
  app.get("/me", async (c) => {
16849
- const token = getCookie(c, "session");
17914
+ const token = getCookie2(c, "session");
16850
17915
  if (!token) return c.json({ authenticated: false });
16851
17916
  const userId = verifySession(token);
16852
17917
  if (!userId) {
16853
- deleteCookie(c, "session", { path: "/", secure: secureCookies, sameSite: "Strict" });
17918
+ deleteCookie(c, "session", { path: "/", secure: secureCookies2, sameSite: "Strict" });
16854
17919
  return c.json({ authenticated: false });
16855
17920
  }
16856
17921
  const foundUser = await getUserById(userId);
16857
17922
  if (!foundUser) {
16858
- deleteCookie(c, "session", { path: "/", secure: secureCookies, sameSite: "Strict" });
17923
+ deleteCookie(c, "session", { path: "/", secure: secureCookies2, sameSite: "Strict" });
16859
17924
  return c.json({ authenticated: false });
16860
17925
  }
16861
17926
  const refreshed = await applyMonthlyFreeRefresh(foundUser);
@@ -16874,6 +17939,8 @@ app.get("/me", async (c) => {
16874
17939
  has_concurrency_sub: !!user.concurrency_stripe_sub_id,
16875
17940
  subscription_tier: user.subscription_tier,
16876
17941
  subscription_concurrency: user.subscription_concurrency,
17942
+ memory_plan: user.memory_plan ?? "free",
17943
+ has_memory_sub: !!user.memory_subscription_id,
16877
17944
  ...stats
16878
17945
  });
16879
17946
  });
@@ -16887,6 +17954,227 @@ app.delete("/api-key", requireAllowedOrigin, sessionAuth, async (c) => {
16887
17954
  await revokeApiKey(user.id);
16888
17955
  return c.json({ ok: true });
16889
17956
  });
17957
+ function maybeProvisionInBackground(user, vaultCount) {
17958
+ if (vaultCount > 0 || user.memory_provisioned === 1) return;
17959
+ void (async () => {
17960
+ try {
17961
+ if (await getMemoryProvisioned(user.id)) return;
17962
+ await provisionMemoryForUser(user);
17963
+ } catch (err) {
17964
+ console.warn("[memory] background provision failed:", err instanceof Error ? err.message : String(err));
17965
+ }
17966
+ })();
17967
+ }
17968
+ app.get("/memory/overview", auth2, async (c) => {
17969
+ const user = c.get("user");
17970
+ const identity = resolveMemoryIdentity(user);
17971
+ const plan = memoryPlanForReads(user);
17972
+ const requestedVault = c.req.query("vault")?.trim() || null;
17973
+ try {
17974
+ const [vaults, storage, usage] = await Promise.all([
17975
+ dbListVaults(identity),
17976
+ dbStorage(identity, plan),
17977
+ dbUsage(identity, plan)
17978
+ ]);
17979
+ maybeProvisionInBackground(user, vaults.length);
17980
+ const activeVault = requestedVault && vaults.some((v) => v.vault === requestedVault) ? requestedVault : vaults[0]?.vault ?? null;
17981
+ const notes = activeVault ? await dbNotes(identity, activeVault) : [];
17982
+ const billing = {
17983
+ ok: true,
17984
+ memory_plan: user.memory_plan ?? "free",
17985
+ has_memory_sub: !!user.memory_subscription_id,
17986
+ free_quota_gb: MEMORY_PLAN_QUOTA.free / 1e9,
17987
+ plans: Object.values(MEMORY_PLANS).map((p) => ({ plan: p.plan, label: p.label, price_id: p.price_id, interval: p.interval, monthly_usd: p.monthly_usd, quota_gb: p.quota_bytes / 1e9 }))
17988
+ };
17989
+ return c.json({ ok: true, vaults, storage, activeVault, notes, usage, billing });
17990
+ } catch (err) {
17991
+ const message = err instanceof Error ? err.message : "memory overview failed";
17992
+ console.error("[memory/overview]", message);
17993
+ return c.json({ ok: false, error: message }, 502);
17994
+ }
17995
+ });
17996
+ app.get("/memory/vaults", auth2, async (c) => {
17997
+ const user = c.get("user");
17998
+ try {
17999
+ const vaults = await dbListVaults(resolveMemoryIdentity(user));
18000
+ maybeProvisionInBackground(user, vaults.length);
18001
+ return c.json({ ok: true, vaults });
18002
+ } catch (err) {
18003
+ return c.json({ ok: false, error: err instanceof Error ? err.message : "could not load vaults" }, 502);
18004
+ }
18005
+ });
18006
+ app.get("/memory/connect", auth2, async (c) => {
18007
+ const { key, error } = await getOrCreateUserMemoryKey(c.get("user"));
18008
+ if (!key) return c.json({ error: error ?? "memory unavailable" }, 502);
18009
+ const command = `MCP_MEMORY_KEY=${key} npx -y -p @vilovieta/mcp-memory@latest mcp-memory`;
18010
+ const config = {
18011
+ mcpServers: {
18012
+ "mcp-memory": {
18013
+ command: "npx",
18014
+ args: ["-y", "-p", "@vilovieta/mcp-memory@latest", "mcp-memory"],
18015
+ env: { MCP_MEMORY_KEY: key }
18016
+ }
18017
+ }
18018
+ };
18019
+ return c.json({ ok: true, command, config });
18020
+ });
18021
+ app.post("/memory/vault", auth2, async (c) => {
18022
+ const { key, error } = await getOrCreateUserMemoryKey(c.get("user"));
18023
+ if (!key) return c.json({ error: error ?? "memory unavailable" }, 502);
18024
+ const body = await c.req.json().catch(() => ({}));
18025
+ const vault = body.vault?.trim();
18026
+ if (!vault) return c.json({ error: "vault name required" }, 400);
18027
+ const result = await memoryCall("addVaultTool", { vault }, key);
18028
+ return c.json(result, result.ok ? 200 : 502);
18029
+ });
18030
+ app.get("/memory/notes", auth2, async (c) => {
18031
+ const user = c.get("user");
18032
+ const identity = resolveMemoryIdentity(user);
18033
+ try {
18034
+ let vault = c.req.query("vault")?.trim() || null;
18035
+ if (!vault) {
18036
+ const vaults = await dbListVaults(identity);
18037
+ vault = vaults[0]?.vault ?? null;
18038
+ }
18039
+ if (!vault) return c.json({ ok: true, notes: [], vault: null });
18040
+ const notes = await dbNotes(identity, vault);
18041
+ return c.json({ ok: true, vault, notes });
18042
+ } catch (err) {
18043
+ return c.json({ ok: false, error: err instanceof Error ? err.message : "could not load notes" }, 502);
18044
+ }
18045
+ });
18046
+ app.get("/memory/note", auth2, async (c) => {
18047
+ const user = c.get("user");
18048
+ const vault = c.req.query("vault")?.trim();
18049
+ const path5 = c.req.query("path");
18050
+ if (!path5) return c.json({ error: "path required" }, 400);
18051
+ if (!vault) return c.json({ ok: false, error: "vault required" }, 400);
18052
+ try {
18053
+ const note = await dbNote(resolveMemoryIdentity(user), vault, path5);
18054
+ if (!note) return c.json({ ok: false, error: "note not found" }, 404);
18055
+ return c.json({ ok: true, note });
18056
+ } catch (err) {
18057
+ return c.json({ ok: false, error: err instanceof Error ? err.message : "could not load note" }, 502);
18058
+ }
18059
+ });
18060
+ app.put("/memory/note", auth2, async (c) => {
18061
+ const { key, error } = await getOrCreateUserMemoryKey(c.get("user"));
18062
+ if (!key) return c.json({ error: error ?? "memory unavailable" }, 502);
18063
+ const body = await c.req.json().catch(() => ({}));
18064
+ if (!body.path) return c.json({ error: "path required" }, 400);
18065
+ const result = await memoryCall("putTool", { vault: body.vault, path: body.path, title: body.title, content: body.content }, key);
18066
+ return c.json(result, result.ok ? 200 : 502);
18067
+ });
18068
+ app.delete("/memory/note", auth2, async (c) => {
18069
+ const { key, error } = await getOrCreateUserMemoryKey(c.get("user"));
18070
+ if (!key) return c.json({ error: error ?? "memory unavailable" }, 502);
18071
+ const vault = c.req.query("vault");
18072
+ const path5 = c.req.query("path");
18073
+ if (!path5) return c.json({ error: "path required" }, 400);
18074
+ const result = await memoryCall("deleteNoteTool", { vault, path: path5 }, key);
18075
+ return c.json(result, result.ok ? 200 : 502);
18076
+ });
18077
+ app.get("/memory/note/download", auth2, async (c) => {
18078
+ const user = c.get("user");
18079
+ const vault = c.req.query("vault")?.trim();
18080
+ const path5 = c.req.query("path");
18081
+ if (!path5) return c.json({ error: "path required" }, 400);
18082
+ if (!vault) return c.json({ error: "vault required" }, 400);
18083
+ const note = await dbNote(resolveMemoryIdentity(user), vault, path5);
18084
+ if (!note) return c.json({ error: "note not found" }, 404);
18085
+ const filename = (path5.split("/").pop() || "note").replace(/"/g, "");
18086
+ return new Response(note.content ?? "", {
18087
+ status: 200,
18088
+ headers: {
18089
+ "Content-Type": "text/markdown; charset=utf-8",
18090
+ "Content-Disposition": `attachment; filename="${filename.endsWith(".md") ? filename : filename + ".md"}"`
18091
+ }
18092
+ });
18093
+ });
18094
+ app.get("/memory/storage", auth2, async (c) => {
18095
+ const user = c.get("user");
18096
+ try {
18097
+ const result = await dbStorage(resolveMemoryIdentity(user), memoryPlanForReads(user));
18098
+ return c.json(result);
18099
+ } catch (err) {
18100
+ return c.json({ ok: false, error: err instanceof Error ? err.message : "could not load storage" }, 502);
18101
+ }
18102
+ });
18103
+ app.get("/memory/billing", auth2, async (c) => {
18104
+ const user = c.get("user");
18105
+ const plans = Object.values(MEMORY_PLANS).map((p) => ({ plan: p.plan, label: p.label, price_id: p.price_id, interval: p.interval, monthly_usd: p.monthly_usd, quota_gb: p.quota_bytes / 1e9 }));
18106
+ return c.json({
18107
+ ok: true,
18108
+ memory_plan: user.memory_plan ?? "free",
18109
+ has_memory_sub: !!user.memory_subscription_id,
18110
+ free_quota_gb: MEMORY_PLAN_QUOTA.free / 1e9,
18111
+ plans
18112
+ });
18113
+ });
18114
+ app.post("/memory/checkout", auth2, async (c) => {
18115
+ try {
18116
+ const user = c.get("user");
18117
+ const body = await c.req.json().catch(() => ({}));
18118
+ const priceId = body.priceId?.trim();
18119
+ if (!priceId || !MEMORY_PLANS[priceId]) return c.json({ error: "Invalid priceId \u2014 must be a memory plan price." }, 400);
18120
+ const secret2 = process.env.STRIPE_SECRET_KEY?.trim();
18121
+ if (!secret2) return c.json({ error: "Stripe is not configured." }, 503);
18122
+ const stripeClient = new Stripe2(secret2, { apiVersion: STRIPE_API_VERSION });
18123
+ let customerId = user.stripe_customer_id;
18124
+ if (!customerId) {
18125
+ const customer = await stripeClient.customers.create({ email: user.email });
18126
+ customerId = customer.id;
18127
+ await setStripeCustomerId(user.id, customerId);
18128
+ }
18129
+ const session = await stripeClient.checkout.sessions.create({
18130
+ mode: "subscription",
18131
+ customer: customerId,
18132
+ line_items: [{ price: priceId, quantity: 1 }],
18133
+ client_reference_id: String(user.id),
18134
+ metadata: { product: "mcp-memory", userId: String(user.id) },
18135
+ subscription_data: { metadata: { product: "mcp-memory", userId: String(user.id) } },
18136
+ automatic_tax: { enabled: true },
18137
+ billing_address_collection: "required",
18138
+ customer_update: { address: "auto", name: "auto" },
18139
+ tax_id_collection: { enabled: true },
18140
+ success_url: `${appOrigin()}/memory?billing=success`,
18141
+ cancel_url: `${appOrigin()}/memory?billing=cancel`
18142
+ });
18143
+ if (!session.url) return c.json({ error: "Stripe did not return a checkout URL." }, 502);
18144
+ return c.json({ url: session.url });
18145
+ } catch (err) {
18146
+ const message = err instanceof Error ? err.message : "Unable to start memory checkout.";
18147
+ console.error("[memory/checkout]", message);
18148
+ return c.json({ error: message }, 500);
18149
+ }
18150
+ });
18151
+ app.post("/memory/portal", auth2, async (c) => {
18152
+ try {
18153
+ const user = c.get("user");
18154
+ if (!user.stripe_customer_id) return c.json({ error: "No billing account yet \u2014 subscribe first." }, 409);
18155
+ const secret2 = process.env.STRIPE_SECRET_KEY?.trim();
18156
+ if (!secret2) return c.json({ error: "Stripe is not configured." }, 503);
18157
+ const stripeClient = new Stripe2(secret2, { apiVersion: STRIPE_API_VERSION });
18158
+ const session = await stripeClient.billingPortal.sessions.create({
18159
+ customer: user.stripe_customer_id,
18160
+ return_url: `${appOrigin()}/memory`
18161
+ });
18162
+ return c.json({ url: session.url });
18163
+ } catch (err) {
18164
+ const message = err instanceof Error ? err.message : "Unable to open billing portal.";
18165
+ console.error("[memory/portal]", message);
18166
+ return c.json({ error: message }, 500);
18167
+ }
18168
+ });
18169
+ app.get("/memory/usage", auth2, async (c) => {
18170
+ const user = c.get("user");
18171
+ try {
18172
+ const result = await dbUsage(resolveMemoryIdentity(user), memoryPlanForReads(user));
18173
+ return c.json(result);
18174
+ } catch (err) {
18175
+ return c.json({ ok: false, error: err instanceof Error ? err.message : "could not load usage" }, 502);
18176
+ }
18177
+ });
16890
18178
  var SYNC_HARVEST_TIMEOUT_OVERRIDE_MS = (() => {
16891
18179
  const raw = process.env.SYNC_HARVEST_TIMEOUT_MS;
16892
18180
  const parsed = raw === void 0 ? NaN : Number(raw);
@@ -17013,6 +18301,7 @@ app.post("/harvest/sync", auth2, async (c) => {
17013
18301
  format: "json",
17014
18302
  outputDir: "/tmp/paa-output-api",
17015
18303
  signal: syncSignal,
18304
+ softDeadlineMs: Date.now() + Math.floor(syncTimeoutMs * 0.8),
17016
18305
  onAttemptEvent: recordAttempt
17017
18306
  });
17018
18307
  await completeJob(jobId, result);
@@ -17165,8 +18454,14 @@ app.post("/extract-url", auth2, async (c) => {
17165
18454
  screenshotMeta = { base64: screenshotBuf.toString("base64"), sizeBytes: screenshotBuf.length, device };
17166
18455
  }
17167
18456
  const mediaMeta = downloadMedia ? await harvestPageMedia(result.bodyHtml, canonicalUrl, { types: mediaTypes ?? ["image", "video", "audio"], outputDir: null }) : null;
18457
+ const memory = bodyResult.data.depositToVault ? await persistScrapeBody(user, {
18458
+ title: result.title ?? canonicalUrl,
18459
+ content: result.bodyMarkdown ?? "",
18460
+ source: canonicalUrl,
18461
+ vault: bodyResult.data.vaultName
18462
+ }) : void 0;
17168
18463
  await logRequestEvent({ userId: user.id, source: "extract_url", status: "done", query: canonicalUrl, resultCount: result.headings.length, result });
17169
- return c.json({ ...result, screenshot: screenshotMeta, branding: brandingData, media: mediaMeta });
18464
+ return c.json({ ...result, screenshot: screenshotMeta, branding: brandingData, media: mediaMeta, memory });
17170
18465
  } catch (err) {
17171
18466
  const msg = err instanceof Error ? err.message : String(err);
17172
18467
  if (debited) await creditMc(user.id, MC_COSTS.page_scrape, LedgerOperation.EXTRACT_URL, "failed call");
@@ -17584,16 +18879,17 @@ app.get("/cron/tick", async (c) => {
17584
18879
  if (!process.env.CRON_SECRET || secret2 !== `Bearer ${process.env.CRON_SECRET}`) {
17585
18880
  return c.json({ error: "Unauthorized" }, 401);
17586
18881
  }
17587
- const { drainQueue } = await import("./worker-UM5UM6O5.js");
18882
+ const { drainQueue } = await import("./worker-UC6D2756.js");
17588
18883
  const budget = { maxJobs: 10, deadlineMs: Date.now() + 28e4 };
17589
18884
  const workflowDispatchResult = await dispatchDueWorkflowSchedules(`${new URL(c.req.url).protocol}//${new URL(c.req.url).host}`);
17590
- const [results, sweepResult, reapResult, expiredResult] = await Promise.all([
18885
+ const [results, sweepResult, reapResult, expiredResult, blobCleanup] = await Promise.all([
17591
18886
  drainQueue(budget),
17592
18887
  runMonthlyRefreshSweep(),
17593
18888
  reapIdleBrowserSessions(120).catch(() => ({ reaped: 0 })),
17594
- expireOldLots().catch(() => ({ expired_lots: 0, expired_mc: 0 }))
18889
+ expireOldLots().catch(() => ({ expired_lots: 0, expired_mc: 0 })),
18890
+ cleanupExpiredScrapeBlobs().catch(() => ({ deleted: 0, store: "none" }))
17595
18891
  ]);
17596
- return c.json({ drained: results.length, results, sweepResult, reaped: reapResult, expired: expiredResult, workflowDispatch: workflowDispatchResult });
18892
+ return c.json({ drained: results.length, results, sweepResult, reaped: reapResult, expired: expiredResult, blobCleanup, workflowDispatch: workflowDispatchResult });
17597
18893
  });
17598
18894
  app.on(["GET", "POST", "PUT"], "/api/inngest", serveInngest({ client: inngest, functions: [siteAuditFn, siteExtractFn] }));
17599
18895
  app.route("/api/internal/site-architecture-auditor", siteAuditApp);
@@ -17601,6 +18897,7 @@ app.route("/youtube", youtubeApp);
17601
18897
  app.route("/screenshot", screenshotApp);
17602
18898
  app.route("/facebook", facebookAdApp);
17603
18899
  app.route("/instagram", instagramApp);
18900
+ app.route("/reddit", redditApp);
17604
18901
  app.route("/maps", mapsApp);
17605
18902
  app.route("/directory", directoryApp);
17606
18903
  app.route("/workflows", workflowApp);
@@ -17623,6 +18920,7 @@ app.get("/console/auth/:id/go", async (c) => {
17623
18920
  });
17624
18921
  app.get("/console/:id", (c) => c.html(renderConsoleHtml(c.req.param("id"))));
17625
18922
  app.route("/stripe", stripeApp);
18923
+ app.route("/", oauthApp);
17626
18924
  if (!process.env.INNGEST_EVENT_KEY) {
17627
18925
  startSiteAuditWorker();
17628
18926
  }
@@ -17729,4 +19027,4 @@ app.get("/blog/:slug/", (c) => {
17729
19027
  export {
17730
19028
  app
17731
19029
  };
17732
- //# sourceMappingURL=server-2SRARAO5.js.map
19030
+ //# sourceMappingURL=server-RRCKVBHG.js.map