mcp-scraper 0.3.24 → 0.3.26

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (66) hide show
  1. package/dist/bin/api-server.cjs +8220 -5991
  2. package/dist/bin/api-server.cjs.map +1 -1
  3. package/dist/bin/api-server.js +3 -3
  4. package/dist/bin/mcp-scraper-cli.cjs +320 -58
  5. package/dist/bin/mcp-scraper-cli.cjs.map +1 -1
  6. package/dist/bin/mcp-scraper-cli.js +4 -3
  7. package/dist/bin/mcp-scraper-cli.js.map +1 -1
  8. package/dist/bin/mcp-scraper-install.cjs +3 -3
  9. package/dist/bin/mcp-scraper-install.cjs.map +1 -1
  10. package/dist/bin/mcp-scraper-install.js +2 -2
  11. package/dist/bin/mcp-stdio-server.cjs +186 -49
  12. package/dist/bin/mcp-stdio-server.cjs.map +1 -1
  13. package/dist/bin/mcp-stdio-server.js +5 -5
  14. package/dist/bin/paa-harvest.cjs +284 -10
  15. package/dist/bin/paa-harvest.cjs.map +1 -1
  16. package/dist/bin/paa-harvest.js +5 -3
  17. package/dist/bin/paa-harvest.js.map +1 -1
  18. package/dist/{chunk-GZ4W72WF.js → chunk-2GHI2D6Q.js} +184 -48
  19. package/dist/chunk-2GHI2D6Q.js.map +1 -0
  20. package/dist/{chunk-ZAP7MYVO.js → chunk-3PRO376E.js} +2 -3
  21. package/dist/chunk-3PRO376E.js.map +1 -0
  22. package/dist/{chunk-LFATOGDF.js → chunk-BP27CZ5Q.js} +4 -3
  23. package/dist/chunk-BP27CZ5Q.js.map +1 -0
  24. package/dist/{chunk-DE7WP4UU.js → chunk-DOBQN3EY.js} +3 -3
  25. package/dist/chunk-DOBQN3EY.js.map +1 -0
  26. package/dist/{chunk-IGOEAKC7.js → chunk-G7PQ64LM.js} +274 -9
  27. package/dist/chunk-G7PQ64LM.js.map +1 -0
  28. package/dist/chunk-K443GQY5.js +24 -0
  29. package/dist/chunk-K443GQY5.js.map +1 -0
  30. package/dist/chunk-MS2UZ6HR.js +7 -0
  31. package/dist/chunk-MS2UZ6HR.js.map +1 -0
  32. package/dist/{chunk-H2R232HK.js → chunk-NNEIXK5L.js} +307 -62
  33. package/dist/chunk-NNEIXK5L.js.map +1 -0
  34. package/dist/{chunk-3KYRG7O7.js → chunk-TRNSXR46.js} +185 -2
  35. package/dist/chunk-TRNSXR46.js.map +1 -0
  36. package/dist/{chunk-JTWQGPKF.js → chunk-WYCER2HW.js} +33 -10
  37. package/dist/chunk-WYCER2HW.js.map +1 -0
  38. package/dist/{db-EG5ETPTY.js → db-FP2ABUU4.js} +32 -2
  39. package/dist/index.cjs +284 -10
  40. package/dist/index.cjs.map +1 -1
  41. package/dist/index.d.cts +7 -3
  42. package/dist/index.d.ts +7 -3
  43. package/dist/index.js +5 -3
  44. package/dist/index.js.map +1 -1
  45. package/dist/{server-2SRARAO5.js → server-HIGHKTEV.js} +1563 -275
  46. package/dist/server-HIGHKTEV.js.map +1 -0
  47. package/dist/{worker-UM5UM6O5.js → worker-UC6D2756.js} +7 -6
  48. package/dist/{worker-UM5UM6O5.js.map → worker-UC6D2756.js.map} +1 -1
  49. package/docs/kernel-proxy-future-enhancements.md +80 -0
  50. package/docs/spec-browser-routing.md +108 -0
  51. package/docs/spec-mcp-tools.md +146 -0
  52. package/docs/spec-server-instructions.md +94 -0
  53. package/docs/specs/audit-visual-demo.md +309 -0
  54. package/package.json +5 -2
  55. package/dist/chunk-3KYRG7O7.js.map +0 -1
  56. package/dist/chunk-DE7WP4UU.js.map +0 -1
  57. package/dist/chunk-GZ4W72WF.js.map +0 -1
  58. package/dist/chunk-H2R232HK.js.map +0 -1
  59. package/dist/chunk-H7R4PPMG.js +0 -7
  60. package/dist/chunk-H7R4PPMG.js.map +0 -1
  61. package/dist/chunk-IGOEAKC7.js.map +0 -1
  62. package/dist/chunk-JTWQGPKF.js.map +0 -1
  63. package/dist/chunk-LFATOGDF.js.map +0 -1
  64. package/dist/chunk-ZAP7MYVO.js.map +0 -1
  65. package/dist/server-2SRARAO5.js.map +0 -1
  66. /package/dist/{db-EG5ETPTY.js.map → db-FP2ABUU4.js.map} +0 -0
@@ -21,26 +21,30 @@ import {
21
21
  renderLinkReport,
22
22
  sanitizeAttempts,
23
23
  sanitizeHarvestResult
24
- } from "./chunk-GZ4W72WF.js";
24
+ } from "./chunk-2GHI2D6Q.js";
25
25
  import {
26
26
  csvRecords,
27
27
  listWorkflowDefinitions,
28
28
  rowsToCsv,
29
29
  runWorkflow,
30
30
  runWorkflowStep,
31
+ slugify,
31
32
  workflowDefinition,
32
33
  workflowOutputBaseDir,
33
34
  workflowStepCount,
34
35
  workflowSupportsSteps
35
- } from "./chunk-H2R232HK.js";
36
- import "./chunk-H7R4PPMG.js";
36
+ } from "./chunk-NNEIXK5L.js";
37
+ import "./chunk-MS2UZ6HR.js";
37
38
  import {
38
39
  BROWSER_OPEN_MIN_BALANCE_MC,
39
40
  CONCURRENCY_PRICE_ID,
40
41
  CREDIT_COST_CATALOG,
42
+ FREE_SIGNUP_MC,
41
43
  LedgerOperation,
42
44
  MC_COSTS,
43
45
  MC_PER_CREDIT,
46
+ MEMORY_PLANS,
47
+ MEMORY_PLAN_QUOTA,
44
48
  SUBSCRIPTION_TIERS,
45
49
  SUBSCRIPTION_TIER_BY_KEY,
46
50
  browserActiveCostMc,
@@ -50,15 +54,20 @@ import {
50
54
  harvestProblemResponse,
51
55
  insufficientBalanceResponse,
52
56
  serializeHarvestProblem
53
- } from "./chunk-JTWQGPKF.js";
57
+ } from "./chunk-WYCER2HW.js";
54
58
  import {
55
59
  BrowserDriver,
56
60
  MapsSelectors,
57
61
  buildYouTubeChannelVideosUrl,
62
+ currentCostContext,
58
63
  deleteKernelProxyId,
59
64
  harvest,
60
- resolveKernelProxyId
61
- } from "./chunk-IGOEAKC7.js";
65
+ migrateCostTelemetry,
66
+ recordKernelSession,
67
+ recordVendorUsage,
68
+ resolveKernelProxyId,
69
+ runWithCostContext
70
+ } from "./chunk-G7PQ64LM.js";
62
71
  import {
63
72
  CaptchaError,
64
73
  RECAPTCHA_INSTRUCTIONS,
@@ -67,7 +76,8 @@ import {
67
76
  browserServiceProfileSaveChanges,
68
77
  browserServiceProxyId,
69
78
  sanitizeVendorName
70
- } from "./chunk-ZAP7MYVO.js";
79
+ } from "./chunk-3PRO376E.js";
80
+ import "./chunk-K443GQY5.js";
71
81
  import {
72
82
  DEFAULT_MAPS_PROXY_MODE,
73
83
  DEFAULT_PROXY_MODE,
@@ -77,7 +87,7 @@ import {
77
87
  RawMapsHoursRowSchema,
78
88
  RawMapsOverviewSchema,
79
89
  RawMapsReviewStatsSchema
80
- } from "./chunk-LFATOGDF.js";
90
+ } from "./chunk-BP27CZ5Q.js";
81
91
  import {
82
92
  SiteAuditJobRowSchema,
83
93
  advanceWorkflowRunStep,
@@ -86,6 +96,7 @@ import {
86
96
  checkRateLimit,
87
97
  completeJob,
88
98
  completeWorkflowRunRecord,
99
+ consumeCode,
89
100
  consumePasswordResetToken,
90
101
  countActiveUsers,
91
102
  createJob,
@@ -101,13 +112,18 @@ import {
101
112
  expireOldLots,
102
113
  failJob,
103
114
  failWorkflowRunRecord,
115
+ getClient,
104
116
  getDb,
105
117
  getJob,
106
118
  getLedger,
119
+ getMemoryPlan,
120
+ getMemoryProvisioned,
121
+ getRefresh,
107
122
  getUserByApiKey,
108
123
  getUserByEmail,
109
124
  getUserById,
110
125
  getUserByStripeCustomerId,
126
+ getUserMemoryCreds,
111
127
  getUserStats,
112
128
  getWorkflowArtifact,
113
129
  getWorkflowRun,
@@ -127,19 +143,27 @@ import {
127
143
  markWorkflowRunRunning,
128
144
  markWorkflowScheduleRan,
129
145
  patchWorkflowSchedule,
146
+ putCode,
147
+ putRefresh,
130
148
  reconcileBalanceMc,
131
149
  recordStripeEvent,
132
150
  recordWorkflowWebhookDelivery,
151
+ registerClient,
133
152
  replaceWorkflowArtifacts,
153
+ resolveUserByMemorySubscription,
134
154
  revokeApiKey,
135
155
  rotateApiKey,
156
+ rotateRefresh,
136
157
  setConcurrencySubId,
137
158
  setExtraConcurrencySlots,
159
+ setMemoryPlan,
160
+ setMemoryProvisioned,
138
161
  setPassword,
139
162
  setStripeCustomerId,
140
163
  setSubscriptionTier,
164
+ setUserMemoryCreds,
141
165
  verifyPassword
142
- } from "./chunk-3KYRG7O7.js";
166
+ } from "./chunk-TRNSXR46.js";
143
167
 
144
168
  // src/blog/registry.ts
145
169
  var posts = [
@@ -3511,7 +3535,9 @@ async function fetchWithKernel(url) {
3511
3535
  const apiKey = browserServiceApiKey();
3512
3536
  if (!apiKey) throw new Error("Browser backend API key not set");
3513
3537
  const client2 = new Kernel({ apiKey });
3514
- const kb = await client2.browsers.create({ stealth: true, timeout_seconds: 60 });
3538
+ const openedAtMs = Date.now();
3539
+ const useHeadless = currentCostContext()?.forceHeadful !== true;
3540
+ const kb = await client2.browsers.create({ stealth: true, ...useHeadless ? { headless: true } : {}, timeout_seconds: 60 });
3515
3541
  const browser = await chromium.connectOverCDP(kb.cdp_ws_url);
3516
3542
  try {
3517
3543
  const context = browser.contexts()[0] ?? await browser.newContext({
@@ -3526,6 +3552,7 @@ async function fetchWithKernel(url) {
3526
3552
  });
3527
3553
  await client2.browsers.deleteByID(kb.session_id).catch(() => {
3528
3554
  });
3555
+ void recordKernelSession({ kernelSessionId: kb.session_id, source: "kernel_fetch", stealth: typeof kb.stealth === "boolean" ? kb.stealth : null, headlessSent: useHeadless ? true : null, proxyUsed: typeof kb.proxy_id === "string" ? Boolean(kb.proxy_id) : null, fallback: true, openedAtMs, closedAtMs: Date.now() });
3529
3556
  }
3530
3557
  }
3531
3558
 
@@ -4127,6 +4154,7 @@ async function captureLocal(url, device, wantScreenshot, wantBranding) {
4127
4154
  }
4128
4155
  async function captureKernel(url, kernelApiKey, device, wantScreenshot, wantBranding) {
4129
4156
  const kernel = new Kernel2({ apiKey: kernelApiKey });
4157
+ const openedAtMs = Date.now();
4130
4158
  const session = await kernel.browsers.create({ stealth: true, timeout_seconds: 240 });
4131
4159
  const browser = await chromium2.connectOverCDP(session.cdp_ws_url);
4132
4160
  const vp = device === "mobile" ? MOBILE_VIEWPORT : DESKTOP_VIEWPORT;
@@ -4146,6 +4174,7 @@ async function captureKernel(url, kernelApiKey, device, wantScreenshot, wantBran
4146
4174
  return result;
4147
4175
  } finally {
4148
4176
  await browser.close();
4177
+ void recordKernelSession({ kernelSessionId: session.session_id, source: "screenshot", stealth: typeof session.stealth === "boolean" ? session.stealth : null, headlessSent: null, proxyUsed: typeof session.proxy_id === "string" ? Boolean(session.proxy_id) : null, openedAtMs, closedAtMs: Date.now() });
4149
4178
  }
4150
4179
  }
4151
4180
  async function captureScreenshot(url, kernelApiKey, device = "desktop") {
@@ -4800,14 +4829,19 @@ async function crawlWithRotation(urls, opts) {
4800
4829
  let kb = null;
4801
4830
  let browser = null;
4802
4831
  let inBrowser = 0;
4832
+ let kbOpenedAtMs = 0;
4803
4833
  const retire = () => {
4804
- const b = browser, k = kb;
4834
+ const b = browser, k = kb, openedAtMs = kbOpenedAtMs;
4805
4835
  browser = null;
4806
4836
  kb = null;
4837
+ kbOpenedAtMs = 0;
4807
4838
  if (b) void b.close().catch(() => {
4808
4839
  });
4809
- if (k) void client2.browsers.deleteByID(k.session_id).catch(() => {
4810
- });
4840
+ if (k) {
4841
+ void client2.browsers.deleteByID(k.session_id).catch(() => {
4842
+ });
4843
+ void recordKernelSession({ kernelSessionId: k.session_id, source: "rotating_crawl", stealth: typeof k.stealth === "boolean" ? k.stealth : null, headlessSent: headless, proxyUsed: typeof k.proxy_id === "string" ? Boolean(k.proxy_id) : null, openedAtMs, closedAtMs: Date.now() });
4844
+ }
4811
4845
  };
4812
4846
  try {
4813
4847
  while (true) {
@@ -4819,6 +4853,7 @@ async function crawlWithRotation(urls, opts) {
4819
4853
  if (!browser || inBrowser >= urlsPerBrowser) {
4820
4854
  retire();
4821
4855
  kb = await createBrowserWithRetry(client2, headless);
4856
+ kbOpenedAtMs = Date.now();
4822
4857
  browser = await withTimeout(chromium3.connectOverCDP(kb.cdp_ws_url), 3e4);
4823
4858
  inBrowser = 0;
4824
4859
  }
@@ -5226,7 +5261,7 @@ async function extractSite(opts) {
5226
5261
  }
5227
5262
 
5228
5263
  // src/api/server.ts
5229
- import { Hono as Hono13 } from "hono";
5264
+ import { Hono as Hono15 } from "hono";
5230
5265
  import { serve as serveInngest } from "inngest/hono";
5231
5266
 
5232
5267
  // src/inngest/client.ts
@@ -5301,6 +5336,7 @@ var DeepInfraLlmClient = class {
5301
5336
  });
5302
5337
  if (!resp.ok) throw new Error(`DeepInfra error ${resp.status}: ${await resp.text()}`);
5303
5338
  const data = await resp.json();
5339
+ void recordVendorUsage({ vendor: "deepinfra_qwen", model: this.model, units: data.usage?.completion_tokens ?? 0, unitType: "tokens" });
5304
5340
  return data.choices[0].message.content;
5305
5341
  }
5306
5342
  async completeJson(prompt) {
@@ -8012,16 +8048,6 @@ var CATALOG = {
8012
8048
  isSpecial: true,
8013
8049
  tagline: "Generate a rank-tracker blueprint",
8014
8050
  params: [{ key: "domain", label: "Domain", type: "string", required: true }]
8015
- },
8016
- {
8017
- key: "deep_research_workflow",
8018
- label: "Deep Research",
8019
- kind: "simple",
8020
- endpoint: "/workflows/run",
8021
- mcpName: "workflow_run",
8022
- isSpecial: true,
8023
- tagline: "Multi-source, fact-checked research report",
8024
- params: [{ key: "question", label: "Question", type: "string", required: true }]
8025
8051
  }
8026
8052
  ]
8027
8053
  }
@@ -8924,6 +8950,8 @@ async function attemptKernelWhisper(videoId, kernelApiKey, kernelProxyId, falKey
8924
8950
  timestamp: c.timestamp,
8925
8951
  text: c.text.trim()
8926
8952
  }));
8953
+ const audioSec = chunks.length ? chunks[chunks.length - 1]?.timestamp?.[1] ?? 0 : 0;
8954
+ void recordVendorUsage({ vendor: "fal_wizper", model: "fal-ai/wizper", units: audioSec / 60, unitType: "audio_min" });
8927
8955
  const text = data.text ?? chunks.map((c) => c.text).join(" ");
8928
8956
  if (!text) return null;
8929
8957
  return { videoId, text, chunks, durationMs: Date.now() - start, method: "browser-whisper" };
@@ -9007,7 +9035,9 @@ var ExtractUrlBodySchema = z11.object({
9007
9035
  extractBranding: z11.boolean().optional(),
9008
9036
  downloadMedia: z11.boolean().optional(),
9009
9037
  mediaTypes: z11.array(z11.enum(["image", "video", "audio"])).optional(),
9010
- allowLocal: z11.boolean().optional()
9038
+ allowLocal: z11.boolean().optional(),
9039
+ depositToVault: z11.boolean().optional(),
9040
+ vaultName: z11.string().trim().min(1).max(120).optional()
9011
9041
  });
9012
9042
  var MapUrlsBodySchema = z11.object({
9013
9043
  url: z11.string().min(1, "url is required"),
@@ -10351,6 +10381,8 @@ async function transcribeMediaUrl(mediaUrl, markdownTitle = "# Media Transcript"
10351
10381
  const text = data.text ?? "";
10352
10382
  const chunks = data.chunks ?? [];
10353
10383
  const durationMs = Date.now() - startMs;
10384
+ const audioSec = chunks.length ? chunks[chunks.length - 1]?.timestamp?.[1] ?? 0 : 0;
10385
+ void recordVendorUsage({ vendor: "fal_wizper", model: "fal-ai/wizper", units: audioSec / 60, unitType: "audio_min" });
10354
10386
  return {
10355
10387
  text,
10356
10388
  chunks,
@@ -10410,21 +10442,24 @@ function buildPageIntelUrl(body, country) {
10410
10442
  if (body.pageId?.trim()) return `https://www.facebook.com/ads/library/?active_status=all&ad_type=all&country=${country}&is_targeted_country=false&media_type=all&search_type=page&view_all_page_id=${body.pageId.trim()}`;
10411
10443
  return `https://www.facebook.com/ads/library/?active_status=all&ad_type=all&country=${country}&q=${encodeURIComponent(body.query.trim())}&search_type=keyword_unordered`;
10412
10444
  }
10445
+ function configuredKernelProxy() {
10446
+ return (process.env.BROWSER_SERVICE_PROXY_ID ?? process.env.KERNEL_PROXY_ID)?.trim() || void 0;
10447
+ }
10413
10448
  function kernelLaunchOpts() {
10414
- return { headless: true, kernelApiKey: browserServiceApiKey(), kernelProxyId: browserServiceProxyId(), viewport: { width: 1280, height: 900 }, locale: "en-US" };
10449
+ return { headless: true, kernelApiKey: browserServiceApiKey(), kernelProxyId: configuredKernelProxy(), viewport: { width: 1280, height: 900 }, locale: "en-US" };
10415
10450
  }
10416
10451
  async function kernelLaunchOptsResidential() {
10417
- let proxyId = browserServiceProxyId();
10452
+ let proxyId = configuredKernelProxy();
10418
10453
  try {
10419
10454
  const resolution = await resolveKernelProxyId({
10420
10455
  kernelApiKey: browserServiceApiKey(),
10421
10456
  proxyMode: DEFAULT_PROXY_MODE,
10422
- configuredKernelProxyId: browserServiceProxyId(),
10457
+ configuredKernelProxyId: configuredKernelProxy(),
10423
10458
  gl: "us"
10424
10459
  });
10425
10460
  if (resolution.kernelProxyId) proxyId = resolution.kernelProxyId;
10426
10461
  } catch {
10427
- proxyId = browserServiceProxyId();
10462
+ proxyId = configuredKernelProxy();
10428
10463
  }
10429
10464
  return { headless: true, kernelApiKey: browserServiceApiKey(), kernelProxyId: proxyId, viewport: { width: 1280, height: 900 }, locale: "en-US" };
10430
10465
  }
@@ -11162,10 +11197,14 @@ function resolveProfileName(body) {
11162
11197
  const explicit = body.profile?.trim();
11163
11198
  return explicit || browserServiceProfileName();
11164
11199
  }
11200
+ function configuredKernelProxy2() {
11201
+ return (process.env.BROWSER_SERVICE_PROXY_ID ?? process.env.KERNEL_PROXY_ID)?.trim() || void 0;
11202
+ }
11165
11203
  async function kernelLaunchOptsDirect(profileName, saveProfileChanges) {
11166
11204
  return {
11167
11205
  headless: true,
11168
11206
  kernelApiKey: browserServiceApiKey(),
11207
+ kernelProxyId: configuredKernelProxy2(),
11169
11208
  ...profileName ? { kernelProfileName: profileName } : {},
11170
11209
  ...typeof saveProfileChanges === "boolean" ? { kernelProfileSaveChanges: saveProfileChanges } : {},
11171
11210
  viewport: { width: 1280, height: 900 },
@@ -11420,8 +11459,136 @@ instagramApp.post("/media-download", createApiKeyAuth(), async (c) => {
11420
11459
  }
11421
11460
  });
11422
11461
 
11423
- // src/api/maps-routes.ts
11462
+ // src/api/reddit-routes.ts
11424
11463
  import { Hono as Hono6 } from "hono";
11464
+ import { z as z15 } from "zod";
11465
+ var RedditThreadBodySchema = z15.object({
11466
+ url: z15.string().trim().min(1, "url is required"),
11467
+ maxComments: z15.number().int().min(1).max(2e3).optional()
11468
+ });
11469
+ function invalidRequest3(message) {
11470
+ return { error_code: "invalid_request", message };
11471
+ }
11472
+ function toOldReddit(url) {
11473
+ try {
11474
+ const u = new URL(url.trim());
11475
+ if (!/(^|\.)reddit\.com$/.test(u.hostname.toLowerCase()) && u.hostname.toLowerCase() !== "redd.it") return null;
11476
+ u.hostname = "old.reddit.com";
11477
+ u.protocol = "https:";
11478
+ return u.toString();
11479
+ } catch {
11480
+ return null;
11481
+ }
11482
+ }
11483
+ async function residentialProxyId(attemptIndex) {
11484
+ try {
11485
+ const r = await resolveKernelProxyId({
11486
+ kernelApiKey: browserServiceApiKey(),
11487
+ proxyMode: "location",
11488
+ location: "Austin, TX",
11489
+ gl: "us",
11490
+ attemptIndex,
11491
+ fresh: true
11492
+ });
11493
+ return r.kernelProxyId;
11494
+ } catch {
11495
+ return void 0;
11496
+ }
11497
+ }
11498
+ var PARSE_REDDIT = `(() => {
11499
+ const txt = el => ((el && el.innerText) || '').trim();
11500
+ const attr = (el, a) => ((el && el.getAttribute(a)) || '').trim();
11501
+ const bodyText = (document.body && document.body.innerText) || '';
11502
+ const blocked = /whoa there|blocked by network|you've been blocked|network (policy|security)|log in to your reddit account/i.test(bodyText);
11503
+ const post = document.querySelector('#siteTable .thing.link') || document.querySelector('.thing.link');
11504
+ const title = txt(document.querySelector('a.title')) || (document.title || '');
11505
+ const author = attr(post, 'data-author');
11506
+ const scoreEl = post && post.querySelector('.score.unvoted');
11507
+ const score = scoreEl ? ((scoreEl.getAttribute('title') || scoreEl.textContent || '').trim()) : '';
11508
+ const postBody = txt(post && post.querySelector('.usertext-body .md'));
11509
+ const comments = [];
11510
+ document.querySelectorAll('.commentarea .comment').forEach(c => {
11511
+ const entry = c.querySelector(':scope > .entry');
11512
+ if (!entry) return;
11513
+ const a = attr(c, 'data-author');
11514
+ const b = txt(entry.querySelector('.usertext-body .md'));
11515
+ if (!b) return;
11516
+ const s = ((entry.querySelector('.score.unvoted') || {}).textContent || '').trim();
11517
+ let depth = 0, p = c.parentElement;
11518
+ while (p) { if (p.classList && p.classList.contains('comment')) depth++; p = p.parentElement; }
11519
+ comments.push({ author: a, score: s, depth: depth, body: b });
11520
+ });
11521
+ return { title: title, author: author, score: score, postBody: postBody, blocked: blocked, numComments: comments.length, comments: comments };
11522
+ })()`;
11523
+ var redditApp = new Hono6();
11524
+ redditApp.post("/thread", createApiKeyAuth(), async (c) => {
11525
+ const raw = await c.req.json().catch(() => ({}));
11526
+ const parsed = RedditThreadBodySchema.safeParse(raw);
11527
+ if (!parsed.success) return c.json(invalidRequest3(parsed.error.issues[0]?.message ?? "Invalid request"), 400);
11528
+ const body = parsed.data;
11529
+ const oldUrl = toOldReddit(body.url);
11530
+ if (!oldUrl) return c.json(invalidRequest3("url must be a reddit.com link"), 400);
11531
+ const user = c.get("user");
11532
+ const gate = await acquireConcurrencyGate(user, "reddit_thread", {
11533
+ reuseLockId: c.req.header("x-mcp-scraper-concurrency-lock"),
11534
+ metadata: { url: body.url }
11535
+ });
11536
+ if (!gate.ok) return c.json(concurrencyLimitExceededResponse(gate), 429, { "Retry-After": String(gate.retryAfterSeconds) });
11537
+ let debited = false;
11538
+ let refunded = false;
11539
+ try {
11540
+ const { ok, balance_mc } = await debitMc(user.id, MC_COSTS.reddit_thread, LedgerOperation.REDDIT_THREAD, body.url);
11541
+ if (!ok) return c.json(insufficientBalanceResponse(balance_mc, MC_COSTS.reddit_thread), 402);
11542
+ debited = true;
11543
+ let result = null;
11544
+ for (let attempt = 0; attempt < 3 && !result; attempt++) {
11545
+ const proxyId = await residentialProxyId(attempt);
11546
+ const driver = new BrowserDriver();
11547
+ try {
11548
+ await driver.launch({
11549
+ headless: true,
11550
+ headlessMode: "headful",
11551
+ kernelApiKey: browserServiceApiKey(),
11552
+ kernelProxyId: proxyId,
11553
+ viewport: { width: 1280, height: 900 },
11554
+ locale: "en-US"
11555
+ });
11556
+ const page = driver.getPage();
11557
+ await page.goto(oldUrl, { waitUntil: "domcontentloaded", timeout: 45e3 });
11558
+ await page.waitForTimeout(2500);
11559
+ const data = await page.evaluate(PARSE_REDDIT);
11560
+ if (!data.blocked && (data.postBody || data.comments.length > 0)) {
11561
+ result = { ...data, sourceUrl: body.url, oldRedditUrl: oldUrl, attempts: attempt + 1 };
11562
+ }
11563
+ } catch {
11564
+ } finally {
11565
+ await driver.close();
11566
+ }
11567
+ }
11568
+ if (!result) {
11569
+ await creditMc(user.id, MC_COSTS.reddit_thread, LedgerOperation.REDDIT_THREAD_REFUND, "reddit blocked/empty after retries");
11570
+ refunded = true;
11571
+ await logRequestEvent({ userId: user.id, source: "reddit_thread", status: "failed", query: body.url, error: "reddit blocked/empty after retries" });
11572
+ return c.json({ error: "Reddit blocked the request after retries (refunded)" }, 503);
11573
+ }
11574
+ if (body.maxComments && Array.isArray(result.comments)) {
11575
+ result.comments = result.comments.slice(0, body.maxComments);
11576
+ result.numComments = result.comments.length;
11577
+ }
11578
+ await logRequestEvent({ userId: user.id, source: "reddit_thread", status: "done", query: body.url, resultCount: Number(result.numComments ?? 0), result });
11579
+ return c.json(result);
11580
+ } catch (err) {
11581
+ if (debited && !refunded) await creditMc(user.id, MC_COSTS.reddit_thread, LedgerOperation.REDDIT_THREAD_REFUND, "failed call");
11582
+ const msg = err instanceof Error ? err.message : String(err);
11583
+ await logRequestEvent({ userId: user.id, source: "reddit_thread", status: "failed", query: body.url, error: msg });
11584
+ return c.json({ error: msg }, 500);
11585
+ } finally {
11586
+ await releaseConcurrencyGate(gate.lockId);
11587
+ }
11588
+ });
11589
+
11590
+ // src/api/maps-routes.ts
11591
+ import { Hono as Hono7 } from "hono";
11425
11592
 
11426
11593
  // src/extractor/MapsNavigator.ts
11427
11594
  var MapsNavigator = class {
@@ -11596,7 +11763,7 @@ var MapsReviewCollector = class {
11596
11763
  };
11597
11764
 
11598
11765
  // src/extractor/MapsExtractor.ts
11599
- import { z as z15 } from "zod";
11766
+ import { z as z16 } from "zod";
11600
11767
  var MapsExtractor = class {
11601
11768
  constructor(driver) {
11602
11769
  this.driver = driver;
@@ -11782,7 +11949,7 @@ var MapsExtractor = class {
11782
11949
  });
11783
11950
  return rows;
11784
11951
  }, { hoursTable: MapsSelectors.hoursTable, hoursTableAlt: MapsSelectors.hoursTableAlt });
11785
- const result = z15.array(RawMapsHoursRowSchema).safeParse(raw);
11952
+ const result = z16.array(RawMapsHoursRowSchema).safeParse(raw);
11786
11953
  if (!result.success) {
11787
11954
  console.warn("[MapsExtractor] hours parse failed", result.error.flatten());
11788
11955
  return [];
@@ -11846,7 +12013,7 @@ var MapsExtractor = class {
11846
12013
  });
11847
12014
  return results;
11848
12015
  });
11849
- const result = z15.array(RawMapsAboutAttributeSchema).safeParse(raw);
12016
+ const result = z16.array(RawMapsAboutAttributeSchema).safeParse(raw);
11850
12017
  if (!result.success) {
11851
12018
  console.warn("[MapsExtractor] about parse failed", result.error.flatten());
11852
12019
  return [];
@@ -12107,7 +12274,7 @@ async function cleanupDisposableProxy(kernelApiKey, proxyId, eventName) {
12107
12274
  }
12108
12275
  async function runMapsSearchWithRotation(options) {
12109
12276
  const attempts = [];
12110
- const maxAttempts = mapsSearchMaxAttemptsForProxyMode(options.proxyMode);
12277
+ const maxAttempts = mapsSearchMaxAttemptsForProxyMode(options.proxyMode) + 1;
12111
12278
  const started = Date.now();
12112
12279
  let lastError = null;
12113
12280
  for (let attemptIndex = 0; attemptIndex < maxAttempts; attemptIndex += 1) {
@@ -12129,12 +12296,15 @@ async function runMapsSearchWithRotation(options) {
12129
12296
  fresh: options.proxyMode === "location"
12130
12297
  });
12131
12298
  disposableProxyId = resolution.disposableProxyId;
12132
- const result = await extractor.extract({
12299
+ const extractArgs = {
12133
12300
  ...options,
12134
12301
  proxyZip,
12135
12302
  kernelProxyId: options.proxyMode === "none" ? void 0 : resolution.kernelProxyId,
12136
12303
  kernelProxyResolution: resolution.resolution
12137
- });
12304
+ };
12305
+ const isHeadfulEscalation = attemptIndex === maxAttempts - 1;
12306
+ const baseCtx = currentCostContext();
12307
+ const result = isHeadfulEscalation && baseCtx ? await runWithCostContext({ ...baseCtx, forceHeadful: true }, () => extractor.extract(extractArgs)) : await extractor.extract(extractArgs);
12138
12308
  const attempt = attemptFromDebug({
12139
12309
  attemptIndex,
12140
12310
  maxAttempts,
@@ -12219,7 +12389,7 @@ function mapsErrorResponse(c, err, errorCode) {
12219
12389
  attempts: rotationError?.attempts ?? void 0
12220
12390
  }, retryable ? 503 : 500);
12221
12391
  }
12222
- var mapsApp = new Hono6();
12392
+ var mapsApp = new Hono7();
12223
12393
  mapsApp.post("/search", createApiKeyAuth(), async (c) => {
12224
12394
  const user = c.get("user");
12225
12395
  const body = await c.req.json().catch(() => ({}));
@@ -12357,12 +12527,12 @@ mapsApp.post("/place", createApiKeyAuth(), async (c) => {
12357
12527
  });
12358
12528
 
12359
12529
  // src/api/directory-routes.ts
12360
- import { Hono as Hono7 } from "hono";
12530
+ import { Hono as Hono8 } from "hono";
12361
12531
 
12362
12532
  // src/directory/directory-workflow.ts
12363
12533
  import { mkdir as mkdir2, writeFile } from "fs/promises";
12364
12534
  import { join as join6 } from "path";
12365
- import { z as z16 } from "zod";
12535
+ import { z as z17 } from "zod";
12366
12536
 
12367
12537
  // src/directory/location-db.ts
12368
12538
  import { access, readFile } from "fs/promises";
@@ -12523,24 +12693,24 @@ async function resolveDirectoryMarkets(options) {
12523
12693
  }
12524
12694
 
12525
12695
  // src/directory/directory-workflow.ts
12526
- var DirectoryWorkflowOptionsSchema = z16.object({
12527
- query: z16.string().min(1),
12528
- state: z16.string().min(2).default("TN"),
12529
- minPopulation: z16.number().int().min(0).default(1e5),
12530
- populationYear: z16.union(POPULATION_YEARS.map((year) => z16.literal(year))).default(2025),
12531
- maxCities: z16.number().int().min(1).max(100).default(25),
12532
- maxResultsPerCity: z16.number().int().min(1).max(50).default(50),
12533
- concurrency: z16.number().int().min(1).max(5).default(5),
12534
- includeZipGroups: z16.boolean().default(true),
12535
- usZipsCsvPath: z16.string().optional(),
12536
- saveCsv: z16.boolean().default(true),
12537
- gl: z16.string().length(2).default("us"),
12538
- hl: z16.string().length(2).default("en"),
12539
- proxyMode: z16.enum(["location", "configured", "none"]).default(DEFAULT_MAPS_PROXY_MODE),
12540
- proxyZip: z16.string().regex(/^\d{5}$/).optional(),
12541
- debug: z16.boolean().default(false),
12542
- headless: z16.boolean().default(true),
12543
- kernelApiKey: z16.string().optional()
12696
+ var DirectoryWorkflowOptionsSchema = z17.object({
12697
+ query: z17.string().min(1),
12698
+ state: z17.string().min(2).default("TN"),
12699
+ minPopulation: z17.number().int().min(0).default(1e5),
12700
+ populationYear: z17.union(POPULATION_YEARS.map((year) => z17.literal(year))).default(2025),
12701
+ maxCities: z17.number().int().min(1).max(100).default(25),
12702
+ maxResultsPerCity: z17.number().int().min(1).max(50).default(50),
12703
+ concurrency: z17.number().int().min(1).max(5).default(5),
12704
+ includeZipGroups: z17.boolean().default(true),
12705
+ usZipsCsvPath: z17.string().optional(),
12706
+ saveCsv: z17.boolean().default(true),
12707
+ gl: z17.string().length(2).default("us"),
12708
+ hl: z17.string().length(2).default("en"),
12709
+ proxyMode: z17.enum(["location", "configured", "none"]).default(DEFAULT_MAPS_PROXY_MODE),
12710
+ proxyZip: z17.string().regex(/^\d{5}$/).optional(),
12711
+ debug: z17.boolean().default(false),
12712
+ headless: z17.boolean().default(true),
12713
+ kernelApiKey: z17.string().optional()
12544
12714
  });
12545
12715
  function errorMessage(err) {
12546
12716
  return err instanceof Error ? err.message : String(err);
@@ -12761,7 +12931,7 @@ async function runDirectoryWorkflowFromPlan(options, plan) {
12761
12931
  }
12762
12932
 
12763
12933
  // src/api/directory-routes.ts
12764
- var directoryApp = new Hono7();
12934
+ var directoryApp = new Hono8();
12765
12935
  directoryApp.post("/run", createApiKeyAuth(), async (c) => {
12766
12936
  const user = c.get("user");
12767
12937
  const body = await c.req.json().catch(() => ({}));
@@ -12834,35 +13004,35 @@ directoryApp.post("/run", createApiKeyAuth(), async (c) => {
12834
13004
  // src/api/workflow-routes.ts
12835
13005
  import { createHmac } from "crypto";
12836
13006
  import { readFile as readFile2 } from "fs/promises";
12837
- import { Hono as Hono8 } from "hono";
12838
- import { z as z17 } from "zod";
12839
- var workflowApp = new Hono8();
12840
- var WorkflowInputSchema = z17.record(z17.unknown()).default({});
12841
- var WorkflowIdSchema = z17.string().min(1);
12842
- var CadenceSchema = z17.enum(["daily", "weekly", "monthly"]);
12843
- var ScheduleStatusSchema = z17.enum(["active", "paused"]);
12844
- var RunBodySchema = z17.object({
13007
+ import { Hono as Hono9 } from "hono";
13008
+ import { z as z18 } from "zod";
13009
+ var workflowApp = new Hono9();
13010
+ var WorkflowInputSchema = z18.record(z18.unknown()).default({});
13011
+ var WorkflowIdSchema = z18.string().min(1);
13012
+ var CadenceSchema = z18.enum(["daily", "weekly", "monthly"]);
13013
+ var ScheduleStatusSchema = z18.enum(["active", "paused"]);
13014
+ var RunBodySchema = z18.object({
12845
13015
  workflowId: WorkflowIdSchema,
12846
13016
  input: WorkflowInputSchema,
12847
- webhookUrl: z17.string().url().optional()
13017
+ webhookUrl: z18.string().url().optional()
12848
13018
  });
12849
- var ScheduleCreateSchema = z17.object({
13019
+ var ScheduleCreateSchema = z18.object({
12850
13020
  workflowId: WorkflowIdSchema,
12851
- name: z17.string().min(1).max(120).optional(),
13021
+ name: z18.string().min(1).max(120).optional(),
12852
13022
  input: WorkflowInputSchema,
12853
13023
  cadence: CadenceSchema.default("weekly"),
12854
- timezone: z17.string().min(1).max(64).default("UTC"),
12855
- webhookUrl: z17.string().url().optional(),
12856
- nextRunAt: z17.string().datetime().optional()
13024
+ timezone: z18.string().min(1).max(64).default("UTC"),
13025
+ webhookUrl: z18.string().url().optional(),
13026
+ nextRunAt: z18.string().datetime().optional()
12857
13027
  });
12858
- var SchedulePatchSchema = z17.object({
12859
- name: z17.string().min(1).max(120).optional(),
13028
+ var SchedulePatchSchema = z18.object({
13029
+ name: z18.string().min(1).max(120).optional(),
12860
13030
  status: ScheduleStatusSchema.optional(),
12861
13031
  input: WorkflowInputSchema.optional(),
12862
13032
  cadence: CadenceSchema.optional(),
12863
- timezone: z17.string().min(1).max(64).optional(),
12864
- webhookUrl: z17.string().url().nullable().optional(),
12865
- nextRunAt: z17.string().datetime().nullable().optional()
13033
+ timezone: z18.string().min(1).max(64).optional(),
13034
+ webhookUrl: z18.string().url().nullable().optional(),
13035
+ nextRunAt: z18.string().datetime().nullable().optional()
12866
13036
  });
12867
13037
  function hostedWorkflowOutputDir() {
12868
13038
  return workflowOutputBaseDir(process.env.MCP_SCRAPER_WORKFLOW_OUTPUT_DIR?.trim() || "/tmp/mcp-scraper-workflows");
@@ -13366,7 +13536,7 @@ workflowApp.post("/cron/dispatch", async (c) => {
13366
13536
  });
13367
13537
 
13368
13538
  // src/api/serp-intelligence-routes.ts
13369
- import { Hono as Hono9 } from "hono";
13539
+ import { Hono as Hono10 } from "hono";
13370
13540
 
13371
13541
  // src/serp-intelligence/page-snapshot-extractor.ts
13372
13542
  import { createHash as createHash2 } from "crypto";
@@ -13678,7 +13848,7 @@ async function capturePageSnapshots(targets, options = {}) {
13678
13848
  }
13679
13849
 
13680
13850
  // src/serp-intelligence/schemas.ts
13681
- import { z as z18 } from "zod";
13851
+ import { z as z19 } from "zod";
13682
13852
  var SerpIntelligenceDeviceValues = ["desktop", "mobile"];
13683
13853
  var SerpIntelligenceProxyModeValues = ["location", "configured", "none"];
13684
13854
  var SerpIntelligenceAttemptOutcomeValues = [
@@ -13740,171 +13910,171 @@ function isPublicHttpUrl(value) {
13740
13910
  return false;
13741
13911
  }
13742
13912
  }
13743
- var SerpIntelligencePublicHttpUrlSchema = z18.string().url().refine(isPublicHttpUrl, "url must be a public HTTP or HTTPS URL");
13744
- var SerpIntelligenceCaptureBodySchema = z18.object({
13745
- query: z18.string().trim().min(1, "query is required"),
13746
- location: z18.string().trim().min(1).optional(),
13747
- gl: z18.string().trim().length(2).default("us"),
13748
- hl: z18.string().trim().length(2).default("en"),
13749
- device: z18.enum(SerpIntelligenceDeviceValues).default("desktop"),
13750
- proxyMode: z18.enum(SerpIntelligenceProxyModeValues).default(DEFAULT_PROXY_MODE),
13751
- proxyZip: z18.string().regex(/^\d{5}$/).optional(),
13752
- pages: z18.number().int().min(1).max(2).default(1),
13753
- debug: z18.boolean().default(false),
13754
- includePageSnapshots: z18.boolean().default(false),
13755
- pageSnapshotLimit: z18.number().int().min(0).max(10).default(0)
13913
+ var SerpIntelligencePublicHttpUrlSchema = z19.string().url().refine(isPublicHttpUrl, "url must be a public HTTP or HTTPS URL");
13914
+ var SerpIntelligenceCaptureBodySchema = z19.object({
13915
+ query: z19.string().trim().min(1, "query is required"),
13916
+ location: z19.string().trim().min(1).optional(),
13917
+ gl: z19.string().trim().length(2).default("us"),
13918
+ hl: z19.string().trim().length(2).default("en"),
13919
+ device: z19.enum(SerpIntelligenceDeviceValues).default("desktop"),
13920
+ proxyMode: z19.enum(SerpIntelligenceProxyModeValues).default(DEFAULT_PROXY_MODE),
13921
+ proxyZip: z19.string().regex(/^\d{5}$/).optional(),
13922
+ pages: z19.number().int().min(1).max(2).default(1),
13923
+ debug: z19.boolean().default(false),
13924
+ includePageSnapshots: z19.boolean().default(false),
13925
+ pageSnapshotLimit: z19.number().int().min(0).max(10).default(0)
13756
13926
  }).strict();
13757
- var SerpIntelligencePageSnapshotRequestSchema = z18.object({
13927
+ var SerpIntelligencePageSnapshotRequestSchema = z19.object({
13758
13928
  url: SerpIntelligencePublicHttpUrlSchema,
13759
- sourceKind: z18.enum(SerpPageSnapshotSourceKindValues).default("configured_target"),
13760
- sourcePosition: z18.number().int().min(1).optional()
13929
+ sourceKind: z19.enum(SerpPageSnapshotSourceKindValues).default("configured_target"),
13930
+ sourcePosition: z19.number().int().min(1).optional()
13761
13931
  }).strict();
13762
- var SerpIntelligencePageSnapshotsBodySchema = z18.object({
13763
- urls: z18.array(SerpIntelligencePublicHttpUrlSchema).min(1).max(25),
13764
- targets: z18.array(SerpIntelligencePageSnapshotRequestSchema).min(1).max(25).optional(),
13765
- maxConcurrency: z18.number().int().min(1).max(5).default(2),
13766
- timeoutMs: z18.number().int().min(1e3).max(6e4).default(15e3),
13767
- debug: z18.boolean().default(false)
13932
+ var SerpIntelligencePageSnapshotsBodySchema = z19.object({
13933
+ urls: z19.array(SerpIntelligencePublicHttpUrlSchema).min(1).max(25),
13934
+ targets: z19.array(SerpIntelligencePageSnapshotRequestSchema).min(1).max(25).optional(),
13935
+ maxConcurrency: z19.number().int().min(1).max(5).default(2),
13936
+ timeoutMs: z19.number().int().min(1e3).max(6e4).default(15e3),
13937
+ debug: z19.boolean().default(false)
13768
13938
  }).strict();
13769
- var SerpIntelligenceAICitationSchema = z18.object({
13770
- text: z18.string(),
13771
- href: z18.string()
13939
+ var SerpIntelligenceAICitationSchema = z19.object({
13940
+ text: z19.string(),
13941
+ href: z19.string()
13772
13942
  }).strict();
13773
- var SerpIntelligenceOrganicResultSchema = z18.object({
13774
- position: z18.number().int().min(1),
13775
- title: z18.string(),
13776
- url: z18.string(),
13777
- domain: z18.string(),
13778
- cite: z18.string().nullable(),
13779
- snippet: z18.string().nullable(),
13780
- isRedditStyle: z18.boolean(),
13781
- inlineRating: z18.object({
13782
- value: z18.string(),
13783
- count: z18.string()
13943
+ var SerpIntelligenceOrganicResultSchema = z19.object({
13944
+ position: z19.number().int().min(1),
13945
+ title: z19.string(),
13946
+ url: z19.string(),
13947
+ domain: z19.string(),
13948
+ cite: z19.string().nullable(),
13949
+ snippet: z19.string().nullable(),
13950
+ isRedditStyle: z19.boolean(),
13951
+ inlineRating: z19.object({
13952
+ value: z19.string(),
13953
+ count: z19.string()
13784
13954
  }).strict().nullable()
13785
13955
  }).strict();
13786
- var SerpIntelligenceLocationEvidenceSchema = z18.object({
13787
- status: z18.enum(SerpIntelligenceLocalizationStatusValues),
13788
- expected: z18.object({
13789
- city: z18.string(),
13790
- regionCode: z18.string().nullable(),
13791
- canonicalLocation: z18.string()
13956
+ var SerpIntelligenceLocationEvidenceSchema = z19.object({
13957
+ status: z19.enum(SerpIntelligenceLocalizationStatusValues),
13958
+ expected: z19.object({
13959
+ city: z19.string(),
13960
+ regionCode: z19.string().nullable(),
13961
+ canonicalLocation: z19.string()
13792
13962
  }).strict().nullable(),
13793
- candidates: z18.array(z18.object({
13794
- city: z18.string(),
13795
- regionCode: z18.string(),
13796
- count: z18.number().int().min(0),
13797
- examples: z18.array(z18.string())
13963
+ candidates: z19.array(z19.object({
13964
+ city: z19.string(),
13965
+ regionCode: z19.string(),
13966
+ count: z19.number().int().min(0),
13967
+ examples: z19.array(z19.string())
13798
13968
  }).strict())
13799
13969
  }).strict();
13800
- var SerpIntelligenceHarvestResultSchema = z18.object({
13801
- seed: z18.string(),
13802
- location: z18.string().nullable(),
13803
- extractedAt: z18.string(),
13804
- totalQuestions: z18.number().int().min(0),
13805
- surface: z18.enum(["web", "aim", "unknown"]),
13806
- aiOverview: z18.object({
13807
- detected: z18.boolean(),
13808
- text: z18.string().nullable(),
13809
- citations: z18.array(SerpIntelligenceAICitationSchema),
13810
- expanded: z18.boolean().optional(),
13811
- fullyExpanded: z18.boolean().optional(),
13812
- sections: z18.array(z18.string()).optional()
13970
+ var SerpIntelligenceHarvestResultSchema = z19.object({
13971
+ seed: z19.string(),
13972
+ location: z19.string().nullable(),
13973
+ extractedAt: z19.string(),
13974
+ totalQuestions: z19.number().int().min(0),
13975
+ surface: z19.enum(["web", "aim", "unknown"]),
13976
+ aiOverview: z19.object({
13977
+ detected: z19.boolean(),
13978
+ text: z19.string().nullable(),
13979
+ citations: z19.array(SerpIntelligenceAICitationSchema),
13980
+ expanded: z19.boolean().optional(),
13981
+ fullyExpanded: z19.boolean().optional(),
13982
+ sections: z19.array(z19.string()).optional()
13813
13983
  }).strict(),
13814
- aiMode: z18.object({
13815
- detected: z18.boolean(),
13816
- text: z18.string().nullable(),
13817
- citations: z18.array(SerpIntelligenceAICitationSchema)
13984
+ aiMode: z19.object({
13985
+ detected: z19.boolean(),
13986
+ text: z19.string().nullable(),
13987
+ citations: z19.array(SerpIntelligenceAICitationSchema)
13818
13988
  }).strict(),
13819
- tree: z18.array(z18.unknown()),
13820
- flat: z18.array(z18.unknown()),
13821
- videos: z18.array(z18.unknown()),
13822
- forums: z18.array(z18.unknown()),
13823
- organicResults: z18.array(SerpIntelligenceOrganicResultSchema),
13824
- localPack: z18.array(z18.unknown()),
13825
- entityIds: z18.object({
13826
- entities: z18.array(z18.object({
13827
- name: z18.string(),
13828
- kgId: z18.string().nullable(),
13829
- cid: z18.string().nullable(),
13830
- gcid: z18.string().nullable()
13989
+ tree: z19.array(z19.unknown()),
13990
+ flat: z19.array(z19.unknown()),
13991
+ videos: z19.array(z19.unknown()),
13992
+ forums: z19.array(z19.unknown()),
13993
+ organicResults: z19.array(SerpIntelligenceOrganicResultSchema),
13994
+ localPack: z19.array(z19.unknown()),
13995
+ entityIds: z19.object({
13996
+ entities: z19.array(z19.object({
13997
+ name: z19.string(),
13998
+ kgId: z19.string().nullable(),
13999
+ cid: z19.string().nullable(),
14000
+ gcid: z19.string().nullable()
13831
14001
  }).strict()),
13832
- kgIds: z18.array(z18.string()),
13833
- cids: z18.array(z18.string()),
13834
- gcids: z18.array(z18.string())
14002
+ kgIds: z19.array(z19.string()),
14003
+ cids: z19.array(z19.string()),
14004
+ gcids: z19.array(z19.string())
13835
14005
  }).strict(),
13836
- stats: z18.object({
13837
- seed: z18.string(),
13838
- totalQuestions: z18.number().int().min(0),
13839
- maxDepthReached: z18.number().int().min(0),
13840
- durationMs: z18.number().min(0),
13841
- errorCount: z18.number().int().min(0)
14006
+ stats: z19.object({
14007
+ seed: z19.string(),
14008
+ totalQuestions: z19.number().int().min(0),
14009
+ maxDepthReached: z19.number().int().min(0),
14010
+ durationMs: z19.number().min(0),
14011
+ errorCount: z19.number().int().min(0)
13842
14012
  }).strict(),
13843
- diagnostics: z18.object({
13844
- completionStatus: z18.enum(["paa_found", "no_paa", "serp_only"]),
13845
- problem: z18.null(),
13846
- warnings: z18.array(z18.unknown()).optional(),
13847
- debug: z18.object({
14013
+ diagnostics: z19.object({
14014
+ completionStatus: z19.enum(["paa_found", "no_paa", "serp_only"]),
14015
+ problem: z19.null(),
14016
+ warnings: z19.array(z19.unknown()).optional(),
14017
+ debug: z19.object({
13848
14018
  locationEvidence: SerpIntelligenceLocationEvidenceSchema.optional()
13849
14019
  }).passthrough().optional()
13850
14020
  }).passthrough(),
13851
- whatPeopleSaying: z18.array(z18.unknown())
14021
+ whatPeopleSaying: z19.array(z19.unknown())
13852
14022
  }).strict();
13853
- var SerpIntelligenceCaptureAttemptSchema = z18.object({
13854
- attemptNumber: z18.number().int().min(1),
13855
- outcome: z18.enum(SerpIntelligenceAttemptOutcomeValues),
13856
- startedAt: z18.string().optional(),
13857
- completedAt: z18.string().optional(),
13858
- durationMs: z18.number().min(0).optional(),
13859
- problemCode: z18.string().optional(),
13860
- message: z18.string().optional(),
13861
- kernelSessionId: z18.string().nullable().optional(),
13862
- cleanupSucceeded: z18.boolean().nullable().optional()
14023
+ var SerpIntelligenceCaptureAttemptSchema = z19.object({
14024
+ attemptNumber: z19.number().int().min(1),
14025
+ outcome: z19.enum(SerpIntelligenceAttemptOutcomeValues),
14026
+ startedAt: z19.string().optional(),
14027
+ completedAt: z19.string().optional(),
14028
+ durationMs: z19.number().min(0).optional(),
14029
+ problemCode: z19.string().optional(),
14030
+ message: z19.string().optional(),
14031
+ kernelSessionId: z19.string().nullable().optional(),
14032
+ cleanupSucceeded: z19.boolean().nullable().optional()
13863
14033
  }).strict();
13864
- var SerpPageSnapshotCaptureSchema = z18.object({
14034
+ var SerpPageSnapshotCaptureSchema = z19.object({
13865
14035
  url: SerpIntelligencePublicHttpUrlSchema,
13866
14036
  requestedUrl: SerpIntelligencePublicHttpUrlSchema,
13867
14037
  finalUrl: SerpIntelligencePublicHttpUrlSchema.nullable(),
13868
- sourceKind: z18.enum(SerpPageSnapshotSourceKindValues),
13869
- sourcePosition: z18.number().int().min(1).nullable(),
13870
- status: z18.enum(SerpPageFetchStatusValues),
13871
- fetchedVia: z18.enum(SerpPageFetchedViaValues).nullable(),
13872
- httpStatus: z18.number().int().min(100).max(599).nullable(),
13873
- contentType: z18.string().nullable(),
13874
- title: z18.string().nullable(),
14038
+ sourceKind: z19.enum(SerpPageSnapshotSourceKindValues),
14039
+ sourcePosition: z19.number().int().min(1).nullable(),
14040
+ status: z19.enum(SerpPageFetchStatusValues),
14041
+ fetchedVia: z19.enum(SerpPageFetchedViaValues).nullable(),
14042
+ httpStatus: z19.number().int().min(100).max(599).nullable(),
14043
+ contentType: z19.string().nullable(),
14044
+ title: z19.string().nullable(),
13875
14045
  canonicalUrl: SerpIntelligencePublicHttpUrlSchema.nullable(),
13876
- metaDescription: z18.string().nullable(),
13877
- headings: z18.array(z18.object({
13878
- level: z18.number().int().min(1).max(6),
13879
- text: z18.string()
14046
+ metaDescription: z19.string().nullable(),
14047
+ headings: z19.array(z19.object({
14048
+ level: z19.number().int().min(1).max(6),
14049
+ text: z19.string()
13880
14050
  }).strict()).default([]),
13881
- artifact: z18.object({
13882
- htmlBlobUrl: z18.string().url().nullable(),
13883
- textBlobUrl: z18.string().url().nullable(),
13884
- markdownBlobUrl: z18.string().url().nullable(),
13885
- screenshotBlobUrl: z18.string().url().nullable(),
13886
- contentSha256: z18.string().nullable(),
13887
- capturedAt: z18.string().nullable()
14051
+ artifact: z19.object({
14052
+ htmlBlobUrl: z19.string().url().nullable(),
14053
+ textBlobUrl: z19.string().url().nullable(),
14054
+ markdownBlobUrl: z19.string().url().nullable(),
14055
+ screenshotBlobUrl: z19.string().url().nullable(),
14056
+ contentSha256: z19.string().nullable(),
14057
+ capturedAt: z19.string().nullable()
13888
14058
  }).strict(),
13889
- error: z18.object({
13890
- code: z18.string(),
13891
- message: z18.string()
14059
+ error: z19.object({
14060
+ code: z19.string(),
14061
+ message: z19.string()
13892
14062
  }).strict().nullable()
13893
14063
  }).strict();
13894
- var SerpIntelligenceCaptureResponseSchema = z18.object({
14064
+ var SerpIntelligenceCaptureResponseSchema = z19.object({
13895
14065
  harvestResult: SerpIntelligenceHarvestResultSchema,
13896
- attempts: z18.array(SerpIntelligenceCaptureAttemptSchema),
14066
+ attempts: z19.array(SerpIntelligenceCaptureAttemptSchema),
13897
14067
  locationEvidence: SerpIntelligenceLocationEvidenceSchema.nullable(),
13898
- pageSnapshotArtifacts: z18.array(SerpPageSnapshotCaptureSchema),
13899
- billing: z18.object({
13900
- creditsUsed: z18.number().min(0).optional(),
13901
- requestId: z18.string().optional(),
13902
- jobId: z18.string().optional()
14068
+ pageSnapshotArtifacts: z19.array(SerpPageSnapshotCaptureSchema),
14069
+ billing: z19.object({
14070
+ creditsUsed: z19.number().min(0).optional(),
14071
+ requestId: z19.string().optional(),
14072
+ jobId: z19.string().optional()
13903
14073
  }).strict().optional()
13904
14074
  }).strict();
13905
- var SerpIntelligencePageSnapshotsResponseSchema = z18.object({
13906
- pageSnapshotArtifacts: z18.array(SerpPageSnapshotCaptureSchema),
13907
- attempts: z18.array(SerpIntelligenceCaptureAttemptSchema).default([])
14075
+ var SerpIntelligencePageSnapshotsResponseSchema = z19.object({
14076
+ pageSnapshotArtifacts: z19.array(SerpPageSnapshotCaptureSchema),
14077
+ attempts: z19.array(SerpIntelligenceCaptureAttemptSchema).default([])
13908
14078
  }).strict();
13909
14079
 
13910
14080
  // src/serp-intelligence/serp-capture-service.ts
@@ -14076,7 +14246,7 @@ var SERP_INTELLIGENCE_RATE_LIMIT = 60;
14076
14246
  var SERP_INTELLIGENCE_RATE_WINDOW_SECONDS = 60;
14077
14247
  var POST_CAPTURE_ROUTE_LABEL = "POST /capture";
14078
14248
  var POST_PAGE_SNAPSHOTS_ROUTE_LABEL = "POST /page-snapshots";
14079
- var serpIntelligenceApp = new Hono9();
14249
+ var serpIntelligenceApp = new Hono10();
14080
14250
  serpIntelligenceApp.use("*", createApiKeyAuth());
14081
14251
  function structuredError(input) {
14082
14252
  return {
@@ -14245,41 +14415,95 @@ serpIntelligenceApp.post("/page-snapshots", async (c) => {
14245
14415
  });
14246
14416
 
14247
14417
  // src/mcp/mcp-routes.ts
14248
- import { Hono as Hono10 } from "hono";
14418
+ import { Hono as Hono11 } from "hono";
14249
14419
  import { WebStandardStreamableHTTPServerTransport } from "@modelcontextprotocol/sdk/server/webStandardStreamableHttp.js";
14420
+
14421
+ // src/mcp/mcp-oauth.ts
14422
+ import { createRemoteJWKSet, jwtVerify } from "jose";
14423
+ function oauthIssuer() {
14424
+ return (process.env.OAUTH_ISSUER ?? "https://mcpscraper.dev").replace(/\/$/, "");
14425
+ }
14426
+ function oauthResourceUrl() {
14427
+ return (process.env.MCP_SCRAPER_RESOURCE_URL ?? "https://mcpscraper.dev/mcp").replace(/\/$/, "");
14428
+ }
14429
+ function oauthResourceBaseUrl() {
14430
+ return (process.env.OAUTH_RS_BASE_URL ?? "https://mcpscraper.dev").replace(/\/$/, "");
14431
+ }
14432
+ function protectedResourceMetadata() {
14433
+ return {
14434
+ resource: oauthResourceUrl(),
14435
+ authorization_servers: [oauthIssuer()],
14436
+ scopes_supported: ["scraper.use"],
14437
+ bearer_methods_supported: ["header"]
14438
+ };
14439
+ }
14440
+ function wwwAuthenticateValue() {
14441
+ return `Bearer resource_metadata="${oauthResourceBaseUrl()}/.well-known/oauth-protected-resource"`;
14442
+ }
14443
+ function isBearerJwt(token) {
14444
+ return !token.startsWith("sk_") && token.split(".").length === 3;
14445
+ }
14446
+ var jwksSingleton = null;
14447
+ function getJwks() {
14448
+ if (jwksSingleton) return jwksSingleton;
14449
+ jwksSingleton = createRemoteJWKSet(new URL(`${oauthIssuer()}/.well-known/jwks.json`));
14450
+ return jwksSingleton;
14451
+ }
14452
+ async function verifyScraperOAuthJwt(token) {
14453
+ const { payload } = await jwtVerify(token, getJwks(), {
14454
+ issuer: oauthIssuer(),
14455
+ audience: oauthResourceUrl()
14456
+ });
14457
+ return payload;
14458
+ }
14459
+
14460
+ // src/mcp/mcp-routes.ts
14250
14461
  configureReportSaving(false);
14251
- function mcpAuthError() {
14462
+ function mcpAuthError(message = "authentication required") {
14252
14463
  const body = JSON.stringify({
14253
14464
  jsonrpc: "2.0",
14254
14465
  id: null,
14255
14466
  error: {
14256
14467
  code: -32001,
14257
- message: "authentication required"
14468
+ message
14258
14469
  }
14259
14470
  });
14260
14471
  return new Response(body, {
14261
14472
  status: 401,
14262
14473
  headers: {
14263
14474
  "Content-Type": "application/json",
14264
- "WWW-Authenticate": 'Bearer realm="mcp-scraper", error="invalid_token", error_description="Pass an MCP Scraper API key as x-api-key or Bearer token"'
14475
+ "WWW-Authenticate": wwwAuthenticateValue()
14265
14476
  }
14266
14477
  });
14267
14478
  }
14268
14479
  async function requireMcpCallerKey(c) {
14269
14480
  const xApiKey = c.req.header("x-api-key")?.trim();
14270
14481
  const authHeader = c.req.header("Authorization")?.trim();
14271
- const bearerKey = authHeader?.startsWith("Bearer ") ? authHeader.slice(7).trim() : void 0;
14272
- const callerKey = xApiKey ?? bearerKey;
14482
+ const bearerToken = authHeader?.startsWith("Bearer ") ? authHeader.slice(7).trim() : void 0;
14483
+ if (!xApiKey && bearerToken && isBearerJwt(bearerToken)) {
14484
+ let identity;
14485
+ try {
14486
+ const payload = await verifyScraperOAuthJwt(bearerToken);
14487
+ identity = typeof payload.sub === "string" ? payload.sub : "";
14488
+ } catch {
14489
+ return mcpAuthError("Bearer token is invalid or expired");
14490
+ }
14491
+ if (!identity) return mcpAuthError("Bearer token has no subject");
14492
+ const user2 = await getUserByEmail(identity);
14493
+ if (!user2 || !user2.api_key || user2.key_active !== 1) return mcpAuthError("No active MCP Scraper account for this identity");
14494
+ return user2.api_key;
14495
+ }
14496
+ const callerKey = xApiKey ?? bearerToken;
14273
14497
  if (!callerKey) return mcpAuthError();
14274
14498
  const user = await getUserByApiKey(callerKey);
14275
14499
  if (!user) return mcpAuthError();
14276
14500
  return callerKey;
14277
14501
  }
14278
- var mcpApp = new Hono10();
14502
+ var mcpApp = new Hono11();
14279
14503
  function registerSerpIntelligenceCaptureTools(server, executor) {
14280
14504
  server.registerTool("capture_serp_snapshot", {
14281
14505
  title: "SERP Intelligence Snapshot",
14282
- description: "Capture a structured SERP Intelligence Google snapshot through POST /serp-intelligence/capture, the same product capture path used by Phoenix. Split query from location and infer gl/hl. Omit proxyMode for normal use; the service defaults to the configured browser-service proxy without city/ZIP targeting for the highest general success rate. Use proxyMode location only when the user explicitly needs city/ZIP-targeted residential proxy evidence. Use none only for direct-network debugging. Set debug true when investigating location evidence, proxy behavior, CAPTCHA, or capture reliability.",
14506
+ description: "Capture a structured SERP Intelligence Google snapshot through POST /serp-intelligence/capture, the same product capture path used by Phoenix. Split query from location and infer gl/hl. Routing is automatic \u2014 leave proxyMode unset. Set debug true when investigating location evidence, proxy behavior, CAPTCHA, or capture reliability.",
14283
14507
  inputSchema: CaptureSerpSnapshotInputSchema,
14284
14508
  outputSchema: CaptureSerpSnapshotOutputSchema,
14285
14509
  annotations: liveWebToolAnnotations("SERP Intelligence Snapshot")
@@ -14313,7 +14537,7 @@ mcpApp.all("/", async (c) => {
14313
14537
  });
14314
14538
 
14315
14539
  // src/api/browser-agent-routes.ts
14316
- import { Hono as Hono11 } from "hono";
14540
+ import { Hono as Hono12 } from "hono";
14317
14541
 
14318
14542
  // src/api/browser-agent-db.ts
14319
14543
  import { randomUUID as randomUUID2 } from "crypto";
@@ -15747,7 +15971,7 @@ async function loadOpenSession(id, userId) {
15747
15971
  return row;
15748
15972
  }
15749
15973
  function buildBrowserAgentRoutes() {
15750
- const app2 = new Hono11();
15974
+ const app2 = new Hono12();
15751
15975
  app2.use("*", async (c, next) => {
15752
15976
  await migrateBrowserAgent();
15753
15977
  return next();
@@ -16453,9 +16677,149 @@ if (state.key) { refreshSessions(); if (state.current) selectSession(state.curre
16453
16677
 
16454
16678
  // src/api/stripe-routes.ts
16455
16679
  import Stripe from "stripe";
16456
- import { Hono as Hono12 } from "hono";
16680
+ import { Hono as Hono13 } from "hono";
16681
+
16682
+ // src/api/memory.ts
16683
+ import { createCipheriv, createDecipheriv, randomBytes, scryptSync } from "crypto";
16684
+
16685
+ // src/api/session.ts
16686
+ import { createHmac as createHmac2, timingSafeEqual } from "crypto";
16687
+ var isProduction = () => process.env.NODE_ENV === "production" || process.env.VERCEL === "1";
16688
+ function getSessionSecret() {
16689
+ const configured = process.env.SESSION_SECRET?.trim();
16690
+ if (configured) return configured;
16691
+ if (isProduction()) throw new Error("SESSION_SECRET is not set \u2014 add it to your Vercel env vars (see src/api/env.ts)");
16692
+ return "dev-secret-change-me";
16693
+ }
16694
+ var secret = () => getSessionSecret();
16695
+ function safeEqualHex(a, b) {
16696
+ if (a.length !== b.length) return false;
16697
+ try {
16698
+ return timingSafeEqual(Buffer.from(a, "hex"), Buffer.from(b, "hex"));
16699
+ } catch {
16700
+ return false;
16701
+ }
16702
+ }
16703
+ function signSession(userId) {
16704
+ const payload = String(userId);
16705
+ const sig = createHmac2("sha256", secret()).update(payload).digest("hex");
16706
+ return `${payload}.${sig}`;
16707
+ }
16708
+ function verifySession(token) {
16709
+ const dot = token.lastIndexOf(".");
16710
+ if (dot === -1) return null;
16711
+ const payload = token.slice(0, dot);
16712
+ const sig = token.slice(dot + 1);
16713
+ const expected = createHmac2("sha256", secret()).update(payload).digest("hex");
16714
+ if (!safeEqualHex(sig, expected)) return null;
16715
+ const id = parseInt(payload);
16716
+ return isNaN(id) ? null : id;
16717
+ }
16718
+
16719
+ // src/api/memory.ts
16720
+ var MEMORY_BASE_URL = () => (process.env.MCP_MEMORY_BASE_URL ?? "https://mcp-memory-omega.vercel.app").replace(/\/$/, "");
16721
+ var ADMIN_KEY = () => process.env.MCP_MEMORY_ADMIN_KEY?.trim() ?? "";
16722
+ function isMemoryOperator(email) {
16723
+ if (!email) return false;
16724
+ const ops = (process.env.MEMORY_OPERATOR_IDENTITIES ?? "").split(",").map((s) => s.trim().toLowerCase()).filter(Boolean);
16725
+ return ops.includes(email.trim().toLowerCase());
16726
+ }
16727
+ function encKey() {
16728
+ return scryptSync(getSessionSecret(), "mcp-memory-key-v1", 32);
16729
+ }
16730
+ function encryptMemoryKey(secret2) {
16731
+ const iv = randomBytes(12);
16732
+ const cipher = createCipheriv("aes-256-gcm", encKey(), iv);
16733
+ const enc = Buffer.concat([cipher.update(secret2, "utf8"), cipher.final()]);
16734
+ const tag = cipher.getAuthTag();
16735
+ return `${iv.toString("base64")}:${tag.toString("base64")}:${enc.toString("base64")}`;
16736
+ }
16737
+ function decryptMemoryKey(stored) {
16738
+ try {
16739
+ const [ivB, tagB, dataB] = stored.split(":");
16740
+ const decipher = createDecipheriv("aes-256-gcm", encKey(), Buffer.from(ivB, "base64"));
16741
+ decipher.setAuthTag(Buffer.from(tagB, "base64"));
16742
+ return Buffer.concat([decipher.update(Buffer.from(dataB, "base64")), decipher.final()]).toString("utf8");
16743
+ } catch {
16744
+ return null;
16745
+ }
16746
+ }
16747
+ async function memoryCall(toolName, args, userMemoryKey) {
16748
+ try {
16749
+ const res = await fetch(`${MEMORY_BASE_URL()}/api/mcp/thorbitMemoryServer/tools/${toolName}/execute`, {
16750
+ method: "POST",
16751
+ headers: { "content-type": "application/json" },
16752
+ body: JSON.stringify({ data: { ...args, apiKey: userMemoryKey } })
16753
+ });
16754
+ const body = await res.json().catch(() => null);
16755
+ if (!res.ok) return { ok: false, error: body?.error ?? `memory ${toolName} failed (${res.status})` };
16756
+ if (body?.result) return body.result;
16757
+ return { ok: false, error: body?.error ?? `memory ${toolName} returned no result` };
16758
+ } catch (err) {
16759
+ return { ok: false, error: err?.message ?? "memory call failed" };
16760
+ }
16761
+ }
16762
+ function personalVault(user) {
16763
+ return `mcp-${user.id}`;
16764
+ }
16765
+ function memoryIdentity(user) {
16766
+ return user.email;
16767
+ }
16768
+ function resolveMemoryIdentity(user) {
16769
+ return memoryIdentity(user);
16770
+ }
16771
+ function memoryPlanForReads(user) {
16772
+ if (isMemoryOperator(user.email)) return "unlimited";
16773
+ if (user.memory_plan === "pro" || user.memory_plan === "team") return user.memory_plan;
16774
+ return "free";
16775
+ }
16776
+ async function provisionMemoryForUser(user) {
16777
+ try {
16778
+ const { key } = await getOrCreateUserMemoryKey(user);
16779
+ if (key) await setMemoryProvisioned(user.id, true);
16780
+ } catch {
16781
+ }
16782
+ }
16783
+ async function getOrCreateUserMemoryKey(user) {
16784
+ const creds = await getUserMemoryCreds(user.id);
16785
+ if (creds.memory_key) {
16786
+ const decrypted = decryptMemoryKey(creds.memory_key);
16787
+ if (decrypted) return { key: decrypted };
16788
+ }
16789
+ const admin = ADMIN_KEY();
16790
+ if (!admin) return { key: null, error: "memory not configured (MCP_MEMORY_ADMIN_KEY missing)" };
16791
+ const identity = memoryIdentity(user);
16792
+ const plan = isMemoryOperator(user.email) ? "unlimited" : (await getMemoryPlan(user.id)).plan;
16793
+ const provisioned = await memoryCall(
16794
+ "provisionDefaultsTool",
16795
+ { granteeIdentity: identity, issueKey: true, plan },
16796
+ admin
16797
+ );
16798
+ if (!provisioned.ok || !provisioned.secret) {
16799
+ return { key: null, error: provisioned.error ?? "failed to provision default vaults" };
16800
+ }
16801
+ await setUserMemoryCreds(user.id, encryptMemoryKey(provisioned.secret), identity);
16802
+ return { key: provisioned.secret };
16803
+ }
16804
+ async function syncMemoryKeyPlan(user, plan) {
16805
+ const effectivePlan = isMemoryOperator(user.email) ? "unlimited" : plan;
16806
+ const { key, error } = await getOrCreateUserMemoryKey(user);
16807
+ if (!key) return { ok: false, error: error ?? "memory unavailable" };
16808
+ const listed = await memoryCall(
16809
+ "listKeysTool",
16810
+ {},
16811
+ key
16812
+ );
16813
+ if (!listed.ok || !listed.keys?.length) return { ok: false, error: listed.error ?? "no memory keys found" };
16814
+ const target = listed.keys[0];
16815
+ if (target.plan === effectivePlan) return { ok: true };
16816
+ const res = await memoryCall("setScopeTool", { keyId: target.keyId, plan: effectivePlan }, key);
16817
+ return { ok: res.ok, error: res.error };
16818
+ }
16819
+
16820
+ // src/api/stripe-routes.ts
16457
16821
  var stripe = new Stripe(process.env.STRIPE_SECRET_KEY, { apiVersion: "2026-02-25.clover" });
16458
- var stripeApp = new Hono12();
16822
+ var stripeApp = new Hono13();
16459
16823
  stripeApp.post("/webhooks", async (c) => {
16460
16824
  const sig = c.req.header("stripe-signature");
16461
16825
  const body = await c.req.text();
@@ -16469,6 +16833,8 @@ stripeApp.post("/webhooks", async (c) => {
16469
16833
  if (!isNew) return c.json({ received: true });
16470
16834
  if (event.type === "invoice.paid" || event.type === "invoice.payment_succeeded") {
16471
16835
  const invoice = event.data.object;
16836
+ const memLineId = invoice.lines.data.map(linePriceId).find((id) => id && id in MEMORY_PLANS);
16837
+ if (memLineId) return c.json({ received: true });
16472
16838
  const lineTierId = invoice.lines.data.map(linePriceId).find((id) => id && id in SUBSCRIPTION_TIERS);
16473
16839
  if (lineTierId) {
16474
16840
  const tier = SUBSCRIPTION_TIERS[lineTierId];
@@ -16482,6 +16848,16 @@ stripeApp.post("/webhooks", async (c) => {
16482
16848
  if (event.type === "customer.subscription.created" || event.type === "customer.subscription.updated") {
16483
16849
  const sub = event.data.object;
16484
16850
  const priceId = sub.items.data[0]?.price?.id;
16851
+ if (priceId && priceId in MEMORY_PLANS) {
16852
+ const mem = MEMORY_PLANS[priceId];
16853
+ const user2 = await resolveUser(sub.customer, sub.customer_email);
16854
+ if (user2) {
16855
+ const live = sub.status === "active" || sub.status === "trialing";
16856
+ await setMemoryPlan(user2.id, live ? mem.plan : "free", live ? sub.id : null);
16857
+ await syncMemoryKeyPlan(user2, live ? mem.plan : "free");
16858
+ }
16859
+ return c.json({ received: true });
16860
+ }
16485
16861
  const user = await getUserByStripeCustomerId(sub.customer);
16486
16862
  if (!user) return c.json({ received: true });
16487
16863
  if (priceId && priceId in SUBSCRIPTION_TIERS) {
@@ -16496,6 +16872,14 @@ stripeApp.post("/webhooks", async (c) => {
16496
16872
  if (event.type === "customer.subscription.deleted") {
16497
16873
  const sub = event.data.object;
16498
16874
  const priceId = sub.items.data[0]?.price?.id;
16875
+ if (priceId && priceId in MEMORY_PLANS || await resolveUserByMemorySubscription(sub.id)) {
16876
+ const user2 = await resolveUserByMemorySubscription(sub.id) ?? await getUserByStripeCustomerId(sub.customer);
16877
+ if (user2) {
16878
+ await setMemoryPlan(user2.id, "free", null);
16879
+ await syncMemoryKeyPlan(user2, "free");
16880
+ }
16881
+ return c.json({ received: true });
16882
+ }
16499
16883
  const user = await getUserByStripeCustomerId(sub.customer);
16500
16884
  if (!user) return c.json({ received: true });
16501
16885
  if (priceId && priceId in SUBSCRIPTION_TIERS) {
@@ -16520,6 +16904,417 @@ async function resolveUser(customerId, emailFallback) {
16520
16904
  return user;
16521
16905
  }
16522
16906
 
16907
+ // src/api/oauth-routes.ts
16908
+ import { Hono as Hono14 } from "hono";
16909
+ import { getCookie, setCookie } from "hono/cookie";
16910
+ import { createHash as createHash3, randomBytes as randomBytes2, randomUUID as randomUUID3 } from "crypto";
16911
+ import { importPKCS8, exportJWK, SignJWT } from "jose";
16912
+ var ISSUER = "https://mcpscraper.dev";
16913
+ var RESOURCE = () => process.env.MCP_MEMORY_RESOURCE_URL ?? "https://mcp-memory-omega.vercel.app/mcp/thorbitMemoryServer/mcp";
16914
+ var SCRAPER_RESOURCE = () => (process.env.MCP_SCRAPER_RESOURCE_URL ?? "https://mcpscraper.dev/mcp").replace(/\/$/, "");
16915
+ var MEMORY_SCOPES = ["memory.read", "memory.write", "memory.admin"];
16916
+ var SCRAPER_SCOPES = ["scraper.use"];
16917
+ var SUPPORTED_SCOPES = [...MEMORY_SCOPES, ...SCRAPER_SCOPES];
16918
+ function isScraperResource(resource) {
16919
+ if (!resource) return false;
16920
+ return resource.replace(/\/$/, "") === SCRAPER_RESOURCE();
16921
+ }
16922
+ var ACCESS_TTL_SECONDS = 3600;
16923
+ var REFRESH_TTL_SECONDS = 60 * 60 * 24 * 30;
16924
+ var CODE_TTL_SECONDS = 60;
16925
+ var secureCookies = process.env.NODE_ENV === "production" || process.env.VERCEL === "1";
16926
+ var sessionCookieOptions = {
16927
+ httpOnly: true,
16928
+ secure: secureCookies,
16929
+ path: "/",
16930
+ maxAge: 60 * 60 * 24 * 30,
16931
+ sameSite: "Lax"
16932
+ };
16933
+ var cachedKeys = null;
16934
+ async function getKeys() {
16935
+ if (cachedKeys) return cachedKeys;
16936
+ const raw = process.env.OAUTH_SIGNING_KEY;
16937
+ if (!raw?.trim()) throw new Error("OAUTH_SIGNING_KEY is not set");
16938
+ const pem = raw.includes("BEGIN") ? raw : Buffer.from(raw, "base64").toString("utf8");
16939
+ const privateKey = await importPKCS8(pem, "RS256", { extractable: true });
16940
+ const full = await exportJWK(privateKey);
16941
+ const publicJwk = { kty: full.kty, n: full.n, e: full.e };
16942
+ const kid = createHash3("sha256").update(JSON.stringify({ e: publicJwk.e, kty: publicJwk.kty, n: publicJwk.n })).digest("base64url").slice(0, 16);
16943
+ publicJwk.kid = kid;
16944
+ publicJwk.alg = "RS256";
16945
+ publicJwk.use = "sig";
16946
+ cachedKeys = { privateKey, publicJwk, kid };
16947
+ return cachedKeys;
16948
+ }
16949
+ function isAllowedRedirect(uri) {
16950
+ let u;
16951
+ try {
16952
+ u = new URL(uri);
16953
+ } catch {
16954
+ return false;
16955
+ }
16956
+ if (u.protocol !== "https:") return false;
16957
+ const h = u.hostname.toLowerCase();
16958
+ if (["claude.ai", "www.claude.ai", "claude.com", "www.claude.com"].includes(h)) return true;
16959
+ if (h.endsWith(".claude.ai") || h.endsWith(".claude.com")) return true;
16960
+ const extra = (process.env.OAUTH_EXTRA_REDIRECT_HOSTS ?? "").split(",").map((s) => s.trim().toLowerCase()).filter(Boolean);
16961
+ return extra.includes(h);
16962
+ }
16963
+ function resolvePlan(user) {
16964
+ if (isMemoryOperator(user.email)) return "unlimited";
16965
+ if (user.memory_plan === "pro" || user.memory_plan === "team") return user.memory_plan;
16966
+ return "free";
16967
+ }
16968
+ function negotiateScope(requested, user, resource) {
16969
+ if (isScraperResource(resource)) return "scraper.use";
16970
+ const operator = isMemoryOperator(user.email);
16971
+ const wanted = (requested ?? "").split(/\s+/).map((s) => s.trim()).filter(Boolean);
16972
+ const filtered = wanted.filter((s) => MEMORY_SCOPES.includes(s));
16973
+ const base = filtered.length > 0 ? filtered : ["memory.read", "memory.write"];
16974
+ const allowed = base.filter((s) => s === "memory.admin" ? operator : true);
16975
+ const unique = [...new Set(allowed)];
16976
+ if (operator && !unique.includes("memory.admin")) unique.push("memory.admin");
16977
+ return unique.join(" ");
16978
+ }
16979
+ async function sessionUser(token) {
16980
+ if (!token) return null;
16981
+ const userId = verifySession(token);
16982
+ if (!userId) return null;
16983
+ const user = await getUserById(userId);
16984
+ return user ?? null;
16985
+ }
16986
+ function escapeHtml(value) {
16987
+ return value.replace(/&/g, "&amp;").replace(/</g, "&lt;").replace(/>/g, "&gt;").replace(/"/g, "&quot;").replace(/'/g, "&#39;");
16988
+ }
16989
+ function hiddenFields(p) {
16990
+ const fields = [
16991
+ ["response_type", p.response_type],
16992
+ ["client_id", p.client_id],
16993
+ ["redirect_uri", p.redirect_uri],
16994
+ ["code_challenge", p.code_challenge],
16995
+ ["code_challenge_method", p.code_challenge_method],
16996
+ ["scope", p.scope],
16997
+ ["state", p.state],
16998
+ ["resource", p.resource]
16999
+ ];
17000
+ return fields.map(([k, v]) => `<input type="hidden" name="${k}" value="${escapeHtml(v)}" />`).join("");
17001
+ }
17002
+ function scopeLabel(scope) {
17003
+ const map = {
17004
+ "memory.read": "Read your vaults, notes and search",
17005
+ "memory.write": "Create and edit notes in your vaults",
17006
+ "memory.admin": "Full administrative access",
17007
+ "scraper.use": "Run web scraping, search, maps and social tools (billed to your account)"
17008
+ };
17009
+ return scope.split(/\s+/).filter(Boolean).map((s) => map[s] ?? s).map((l) => `<li>${escapeHtml(l)}</li>`).join("");
17010
+ }
17011
+ function resourceLabel(resource) {
17012
+ return isScraperResource(resource) ? "MCP Scraper" : "mcp-memory";
17013
+ }
17014
+ function pageShell(inner) {
17015
+ return `<!doctype html><html lang="en"><head><meta charset="utf-8" /><meta name="viewport" content="width=device-width,initial-scale=1" /><title>Authorize Claude \xB7 mcp-memory</title><style>
17016
+ :root{color-scheme:dark}
17017
+ *{box-sizing:border-box}
17018
+ body{margin:0;min-height:100vh;display:flex;align-items:center;justify-content:center;background:#0b0d10;color:#e8eaed;font-family:-apple-system,BlinkMacSystemFont,"Segoe UI",Inter,system-ui,sans-serif;padding:24px}
17019
+ .card{width:100%;max-width:420px;background:#14171c;border:1px solid #232830;border-radius:16px;padding:32px}
17020
+ h1{font-size:20px;margin:0 0 4px}
17021
+ p.sub{color:#9aa3ad;font-size:14px;margin:0 0 24px}
17022
+ label{display:block;font-size:13px;color:#9aa3ad;margin:14px 0 6px}
17023
+ input[type=email],input[type=password]{width:100%;padding:11px 13px;background:#0b0d10;border:1px solid #2b313a;border-radius:9px;color:#e8eaed;font-size:14px}
17024
+ input:focus{outline:none;border-color:#5b8cff}
17025
+ button{width:100%;margin-top:20px;padding:12px;border:0;border-radius:9px;font-size:14px;font-weight:600;cursor:pointer}
17026
+ .primary{background:#5b8cff;color:#fff}
17027
+ .ghost{background:transparent;color:#9aa3ad;border:1px solid #2b313a;margin-top:10px}
17028
+ ul{margin:14px 0;padding-left:18px;color:#cbd2da;font-size:13px;line-height:1.7}
17029
+ .err{background:#3a1b1f;border:1px solid #6b2b30;color:#ffb4b4;padding:10px 12px;border-radius:9px;font-size:13px;margin-bottom:16px}
17030
+ .who{font-size:13px;color:#9aa3ad;margin-bottom:8px}
17031
+ </style></head><body><div class="card">${inner}</div></body></html>`;
17032
+ }
17033
+ function renderLogin(p, error) {
17034
+ return pageShell(`
17035
+ <h1>Sign in to ${escapeHtml(resourceLabel(p.resource))}</h1>
17036
+ <p class="sub">Authorize Claude to connect to your ${escapeHtml(resourceLabel(p.resource))} on mcpscraper.dev.</p>
17037
+ ${error ? `<div class="err">${escapeHtml(error)}</div>` : ""}
17038
+ <form method="post" action="/oauth/authorize">
17039
+ ${hiddenFields(p)}
17040
+ <input type="hidden" name="action" value="login" />
17041
+ <label for="email">Email</label>
17042
+ <input id="email" name="email" type="email" autocomplete="username" required />
17043
+ <label for="password">Password</label>
17044
+ <input id="password" name="password" type="password" autocomplete="current-password" required />
17045
+ <button class="primary" type="submit">Sign in</button>
17046
+ </form>`);
17047
+ }
17048
+ function renderConsent(p, email, error) {
17049
+ return pageShell(`
17050
+ <h1>Connect Claude to ${escapeHtml(resourceLabel(p.resource))}</h1>
17051
+ <p class="sub">Allow Claude to access your ${escapeHtml(resourceLabel(p.resource))}?</p>
17052
+ ${error ? `<div class="err">${escapeHtml(error)}</div>` : ""}
17053
+ <div class="who">Signed in as <strong>${escapeHtml(email)}</strong></div>
17054
+ <ul>${scopeLabel(p.scope)}</ul>
17055
+ <form method="post" action="/oauth/authorize">
17056
+ ${hiddenFields(p)}
17057
+ <input type="hidden" name="action" value="approve" />
17058
+ <button class="primary" type="submit">Allow</button>
17059
+ </form>
17060
+ <form method="post" action="/oauth/authorize">
17061
+ ${hiddenFields(p)}
17062
+ <input type="hidden" name="action" value="deny" />
17063
+ <button class="ghost" type="submit">Deny</button>
17064
+ </form>`);
17065
+ }
17066
+ function redirectWithError(redirectUri, state, error) {
17067
+ const u = new URL(redirectUri);
17068
+ u.searchParams.set("error", error);
17069
+ if (state) u.searchParams.set("state", state);
17070
+ return new Response(null, { status: 302, headers: { Location: u.toString() } });
17071
+ }
17072
+ var oauthApp = new Hono14();
17073
+ oauthApp.use("*", async (c, next) => {
17074
+ c.header("Access-Control-Allow-Origin", "*");
17075
+ c.header("Access-Control-Allow-Methods", "GET, POST, OPTIONS");
17076
+ c.header("Access-Control-Allow-Headers", "Content-Type, Authorization, mcp-protocol-version");
17077
+ if (c.req.method === "OPTIONS") return c.body(null, 204);
17078
+ return next();
17079
+ });
17080
+ function authServerMetadata() {
17081
+ return {
17082
+ issuer: ISSUER,
17083
+ authorization_endpoint: `${ISSUER}/oauth/authorize`,
17084
+ token_endpoint: `${ISSUER}/oauth/token`,
17085
+ registration_endpoint: `${ISSUER}/oauth/register`,
17086
+ jwks_uri: `${ISSUER}/.well-known/jwks.json`,
17087
+ scopes_supported: [...SUPPORTED_SCOPES],
17088
+ response_types_supported: ["code"],
17089
+ grant_types_supported: ["authorization_code", "refresh_token"],
17090
+ code_challenge_methods_supported: ["S256"],
17091
+ token_endpoint_auth_methods_supported: ["none"]
17092
+ };
17093
+ }
17094
+ function openidConfiguration() {
17095
+ return {
17096
+ ...authServerMetadata(),
17097
+ subject_types_supported: ["public"],
17098
+ id_token_signing_alg_values_supported: ["RS256"]
17099
+ };
17100
+ }
17101
+ oauthApp.get("/.well-known/oauth-authorization-server", (c) => {
17102
+ console.log("[oauth-dcr] GET /.well-known/oauth-authorization-server ua=%s", c.req.header("user-agent") ?? "");
17103
+ return c.json(authServerMetadata());
17104
+ });
17105
+ oauthApp.get("/.well-known/oauth-authorization-server/*", (c) => {
17106
+ console.log("[oauth-dcr] GET %s (path-aware AS) ua=%s", c.req.path, c.req.header("user-agent") ?? "");
17107
+ return c.json(authServerMetadata());
17108
+ });
17109
+ oauthApp.get("/.well-known/openid-configuration", (c) => {
17110
+ console.log("[oauth-dcr] GET /.well-known/openid-configuration ua=%s", c.req.header("user-agent") ?? "");
17111
+ return c.json(openidConfiguration());
17112
+ });
17113
+ oauthApp.get("/.well-known/openid-configuration/*", (c) => {
17114
+ console.log("[oauth-dcr] GET %s (path-aware OIDC) ua=%s", c.req.path, c.req.header("user-agent") ?? "");
17115
+ return c.json(openidConfiguration());
17116
+ });
17117
+ oauthApp.get("/.well-known/jwks.json", async (c) => {
17118
+ const { publicJwk } = await getKeys();
17119
+ c.header("Cache-Control", "public, max-age=3600");
17120
+ return c.json({ keys: [publicJwk] });
17121
+ });
17122
+ oauthApp.get("/.well-known/oauth-protected-resource", (c) => {
17123
+ return c.json(protectedResourceMetadata());
17124
+ });
17125
+ oauthApp.get("/.well-known/oauth-protected-resource/*", (c) => {
17126
+ return c.json(protectedResourceMetadata());
17127
+ });
17128
+ oauthApp.post("/oauth/register", async (c) => {
17129
+ const rawBody = await c.req.text();
17130
+ console.log("[oauth-dcr] POST /oauth/register ua=%s ct=%s body=%s", c.req.header("user-agent") ?? "", c.req.header("content-type") ?? "", rawBody.slice(0, 2e3));
17131
+ let body;
17132
+ try {
17133
+ body = rawBody ? JSON.parse(rawBody) : null;
17134
+ } catch {
17135
+ body = null;
17136
+ }
17137
+ if (!body) {
17138
+ console.log("[oauth-dcr] register REJECT reason=body_not_json");
17139
+ return c.json({ error: "invalid_client_metadata", error_description: "Body must be JSON" }, 400);
17140
+ }
17141
+ console.log("[oauth-dcr] register body keys=%s redirect_uris=%s", Object.keys(body).join(","), JSON.stringify(body.redirect_uris));
17142
+ const redirectUris = Array.isArray(body.redirect_uris) ? body.redirect_uris.filter((u) => typeof u === "string") : [];
17143
+ if (redirectUris.length === 0) {
17144
+ console.log("[oauth-dcr] register REJECT reason=no_redirect_uris");
17145
+ return c.json({ error: "invalid_redirect_uri", error_description: "redirect_uris is required" }, 400);
17146
+ }
17147
+ for (const uri of redirectUris) {
17148
+ if (!isAllowedRedirect(uri)) {
17149
+ console.log("[oauth-dcr] register REJECT reason=redirect_uri_not_allowed uri=%s", uri);
17150
+ return c.json({ error: "invalid_redirect_uri", error_description: `redirect_uri not allowed: ${uri}` }, 400);
17151
+ }
17152
+ }
17153
+ const clientName = typeof body.client_name === "string" ? body.client_name : null;
17154
+ const clientId = `client_${randomBytes2(16).toString("hex")}`;
17155
+ await registerClient(clientId, redirectUris, clientName);
17156
+ console.log("[oauth-dcr] register OK client_id=%s redirect_uris=%s", clientId, JSON.stringify(redirectUris));
17157
+ return c.json({
17158
+ client_id: clientId,
17159
+ redirect_uris: redirectUris,
17160
+ client_name: clientName ?? void 0,
17161
+ token_endpoint_auth_method: "none",
17162
+ grant_types: ["authorization_code", "refresh_token"],
17163
+ response_types: ["code"],
17164
+ client_id_issued_at: Math.floor(Date.now() / 1e3)
17165
+ }, 201);
17166
+ });
17167
+ function parseAuthParams(get) {
17168
+ return {
17169
+ response_type: get("response_type") ?? "",
17170
+ client_id: get("client_id") ?? "",
17171
+ redirect_uri: get("redirect_uri") ?? "",
17172
+ code_challenge: get("code_challenge") ?? "",
17173
+ code_challenge_method: get("code_challenge_method") ?? "",
17174
+ scope: get("scope") ?? "",
17175
+ state: get("state") ?? "",
17176
+ resource: get("resource") ?? ""
17177
+ };
17178
+ }
17179
+ async function validateAuthRequest(p) {
17180
+ if (p.response_type !== "code") return { ok: false, status: 400, message: "response_type must be code" };
17181
+ if (!p.client_id) return { ok: false, status: 400, message: "client_id is required" };
17182
+ const client2 = await getClient(p.client_id);
17183
+ if (!client2) return { ok: false, status: 400, message: "unknown client_id" };
17184
+ if (!p.redirect_uri || !client2.redirect_uris.includes(p.redirect_uri)) return { ok: false, status: 400, message: "redirect_uri does not match the registered set" };
17185
+ if (!isAllowedRedirect(p.redirect_uri)) return { ok: false, status: 400, message: "redirect_uri host not allowed" };
17186
+ if (!p.code_challenge || p.code_challenge_method !== "S256") return { ok: false, status: 400, message: "PKCE S256 code_challenge is required" };
17187
+ return { ok: true };
17188
+ }
17189
+ oauthApp.get("/oauth/authorize", async (c) => {
17190
+ const p = parseAuthParams((k) => c.req.query(k));
17191
+ console.log("[oauth-dcr] GET /oauth/authorize client_id=%s redirect_uri=%s scope=%s resource=%s", p.client_id, p.redirect_uri, p.scope, p.resource);
17192
+ const valid = await validateAuthRequest(p);
17193
+ if (!valid.ok) {
17194
+ console.log("[oauth-dcr] authorize REJECT reason=%s", valid.message);
17195
+ return c.text(valid.message, valid.status);
17196
+ }
17197
+ const user = await sessionUser(getCookie(c, "session"));
17198
+ if (!user) return c.html(renderLogin(p));
17199
+ const negotiated = { ...p, scope: negotiateScope(p.scope, user, p.resource) };
17200
+ return c.html(renderConsent(negotiated, user.email));
17201
+ });
17202
+ oauthApp.post("/oauth/authorize", async (c) => {
17203
+ const form = await c.req.parseBody();
17204
+ const get = (k) => {
17205
+ const v = form[k];
17206
+ return typeof v === "string" ? v : void 0;
17207
+ };
17208
+ const p = parseAuthParams(get);
17209
+ const action = get("action") ?? "";
17210
+ const valid = await validateAuthRequest(p);
17211
+ if (!valid.ok) return c.text(valid.message, valid.status);
17212
+ if (action === "login") {
17213
+ const email = (get("email") ?? "").trim().toLowerCase();
17214
+ const password = get("password") ?? "";
17215
+ const user2 = email ? await getUserByEmail(email) : void 0;
17216
+ const okPw = !!(user2 && user2.password_hash && verifyPassword(password, user2.password_hash));
17217
+ console.log("[oauth-dcr] LOGIN email=%s found=%s hasHash=%s pwLen=%d ok=%s", email, !!user2, !!(user2 && user2.password_hash), password.length, okPw);
17218
+ if (!okPw) {
17219
+ return c.html(renderLogin(p, "Invalid email or password."));
17220
+ }
17221
+ setCookie(c, "session", signSession(user2.id), sessionCookieOptions);
17222
+ return c.html(renderConsent({ ...p, scope: negotiateScope(p.scope, user2, p.resource) }, user2.email));
17223
+ }
17224
+ const user = await sessionUser(getCookie(c, "session"));
17225
+ if (!user) return c.html(renderLogin(p, "Your session expired. Please sign in again."));
17226
+ if (action === "deny") return redirectWithError(p.redirect_uri, p.state, "access_denied");
17227
+ if (action !== "approve") return c.text("unsupported action", 400);
17228
+ const scope = negotiateScope(p.scope, user, p.resource);
17229
+ const code = `code_${randomBytes2(32).toString("base64url")}`;
17230
+ const expiresAt = new Date(Date.now() + CODE_TTL_SECONDS * 1e3).toISOString();
17231
+ await putCode({
17232
+ code,
17233
+ client_id: p.client_id,
17234
+ redirect_uri: p.redirect_uri,
17235
+ code_challenge: p.code_challenge,
17236
+ identity: user.email,
17237
+ scope,
17238
+ resource: p.resource || RESOURCE(),
17239
+ expires_at: expiresAt
17240
+ });
17241
+ const u = new URL(p.redirect_uri);
17242
+ u.searchParams.set("code", code);
17243
+ if (p.state) u.searchParams.set("state", p.state);
17244
+ return new Response(null, { status: 302, headers: { Location: u.toString() } });
17245
+ });
17246
+ function pkceMatches(verifier, challenge) {
17247
+ if (!verifier) return false;
17248
+ const computed = createHash3("sha256").update(verifier).digest("base64url");
17249
+ return computed === challenge;
17250
+ }
17251
+ async function mintAccessToken(identity, scope, plan, audience) {
17252
+ const { privateKey, kid } = await getKeys();
17253
+ return new SignJWT({ scope, plan }).setProtectedHeader({ alg: "RS256", kid }).setIssuer(ISSUER).setSubject(identity).setAudience(audience).setIssuedAt().setJti(randomUUID3()).setExpirationTime(`${ACCESS_TTL_SECONDS}s`).sign(privateKey);
17254
+ }
17255
+ function tokenErrorResponse(c, error, description, status) {
17256
+ return c.json({ error, error_description: description }, status);
17257
+ }
17258
+ oauthApp.post("/oauth/token", async (c) => {
17259
+ const form = await c.req.parseBody();
17260
+ const get = (k) => {
17261
+ const v = form[k];
17262
+ return typeof v === "string" ? v : void 0;
17263
+ };
17264
+ const grantType = get("grant_type");
17265
+ if (grantType === "authorization_code") {
17266
+ const code = get("code") ?? "";
17267
+ const clientId = get("client_id") ?? "";
17268
+ const redirectUri = get("redirect_uri") ?? "";
17269
+ const verifier = get("code_verifier") ?? "";
17270
+ const record = await consumeCode(code);
17271
+ if (!record) return tokenErrorResponse(c, "invalid_grant", "Authorization code is invalid or already used", 400);
17272
+ if (new Date(record.expires_at).getTime() < Date.now()) return tokenErrorResponse(c, "invalid_grant", "Authorization code has expired", 400);
17273
+ if (record.client_id !== clientId) return tokenErrorResponse(c, "invalid_grant", "client_id mismatch", 400);
17274
+ if (record.redirect_uri !== redirectUri) return tokenErrorResponse(c, "invalid_grant", "redirect_uri mismatch", 400);
17275
+ if (!pkceMatches(verifier, record.code_challenge)) return tokenErrorResponse(c, "invalid_grant", "PKCE verification failed", 400);
17276
+ const user = await getUserByEmail(record.identity);
17277
+ const plan = user ? resolvePlan(user) : "free";
17278
+ const audience = record.resource ?? RESOURCE();
17279
+ const accessToken = await mintAccessToken(record.identity, record.scope, plan, audience);
17280
+ const refreshToken = `rt_${randomBytes2(40).toString("base64url")}`;
17281
+ await putRefresh({
17282
+ refresh_token: refreshToken,
17283
+ client_id: clientId,
17284
+ identity: record.identity,
17285
+ scope: record.scope,
17286
+ resource: audience,
17287
+ expires_at: new Date(Date.now() + REFRESH_TTL_SECONDS * 1e3).toISOString(),
17288
+ revoked: 0
17289
+ });
17290
+ return c.json({ access_token: accessToken, token_type: "Bearer", expires_in: ACCESS_TTL_SECONDS, refresh_token: refreshToken, scope: record.scope });
17291
+ }
17292
+ if (grantType === "refresh_token") {
17293
+ const refreshToken = get("refresh_token") ?? "";
17294
+ const clientId = get("client_id") ?? "";
17295
+ const record = await getRefresh(refreshToken);
17296
+ if (!record || record.revoked) return tokenErrorResponse(c, "invalid_grant", "Refresh token is invalid or revoked", 400);
17297
+ if (new Date(record.expires_at).getTime() < Date.now()) return tokenErrorResponse(c, "invalid_grant", "Refresh token has expired", 400);
17298
+ if (clientId && record.client_id !== clientId) return tokenErrorResponse(c, "invalid_grant", "client_id mismatch", 400);
17299
+ const user = await getUserByEmail(record.identity);
17300
+ const plan = user ? resolvePlan(user) : "free";
17301
+ const audience = record.resource ?? RESOURCE();
17302
+ const accessToken = await mintAccessToken(record.identity, record.scope, plan, audience);
17303
+ const nextRefresh = `rt_${randomBytes2(40).toString("base64url")}`;
17304
+ await rotateRefresh(refreshToken, {
17305
+ refresh_token: nextRefresh,
17306
+ client_id: record.client_id,
17307
+ identity: record.identity,
17308
+ scope: record.scope,
17309
+ resource: audience,
17310
+ expires_at: new Date(Date.now() + REFRESH_TTL_SECONDS * 1e3).toISOString(),
17311
+ revoked: 0
17312
+ });
17313
+ return c.json({ access_token: accessToken, token_type: "Bearer", expires_in: ACCESS_TTL_SECONDS, refresh_token: nextRefresh, scope: record.scope });
17314
+ }
17315
+ return tokenErrorResponse(c, "unsupported_grant_type", "grant_type must be authorization_code or refresh_token", 400);
17316
+ });
17317
+
16523
17318
  // src/api/site-audit-worker.ts
16524
17319
  var MAX_CONCURRENT_SITE_AUDIT = 1;
16525
17320
  async function drainSiteAuditQueue(budget) {
@@ -16555,12 +17350,115 @@ function startSiteAuditWorker() {
16555
17350
 
16556
17351
  // src/api/server.ts
16557
17352
  import { createMiddleware as createMiddleware3 } from "hono/factory";
16558
- import { getCookie, setCookie, deleteCookie } from "hono/cookie";
17353
+ import { getCookie as getCookie2, setCookie as setCookie2, deleteCookie } from "hono/cookie";
16559
17354
  import Stripe2 from "stripe";
16560
17355
 
17356
+ // src/api/scrape-vault-sink.ts
17357
+ var MAX_DEPOSIT_CHARS = 5e6;
17358
+ var SCRAPE_FALLBACK_PREFIX = "scrape-fallback/";
17359
+ var SCRAPE_BLOB_TTL_MS = 24 * 60 * 60 * 1e3;
17360
+ async function depositScrapeToVault(user, opts) {
17361
+ try {
17362
+ const content = (opts.content ?? "").trim();
17363
+ if (!content) return { deposited: false, error: "empty content" };
17364
+ const clipped = content.length > MAX_DEPOSIT_CHARS ? content.slice(0, MAX_DEPOSIT_CHARS) : content;
17365
+ const { key, error } = await getOrCreateUserMemoryKey(user);
17366
+ if (!key) return { deposited: false, error: error ?? "memory unavailable" };
17367
+ const vault = opts.vault?.trim() || personalVault(user);
17368
+ const title = (opts.title?.trim() || opts.source).slice(0, 200);
17369
+ const res = await memoryCall(
17370
+ "libraryIngestTool",
17371
+ { title, content: clipped, source: opts.source, vault },
17372
+ key
17373
+ );
17374
+ if (!res.ok) return { deposited: false, vault, error: res.error ?? "ingest failed" };
17375
+ return { deposited: true, vault, noteId: res.noteId, path: res.path, chunks: res.chunks };
17376
+ } catch (err) {
17377
+ return { deposited: false, error: err?.message ?? "deposit failed" };
17378
+ }
17379
+ }
17380
+ async function uploadFallbackBlob(opts) {
17381
+ try {
17382
+ const content = (opts.content ?? "").trim();
17383
+ if (!content) return null;
17384
+ const slug = (slugify(opts.title || opts.source) || "scrape").slice(0, 60);
17385
+ const key = `${SCRAPE_FALLBACK_PREFIX}${slug}.md`;
17386
+ const doc = `# ${opts.title}
17387
+ Source: ${opts.source}
17388
+
17389
+ ${content}`;
17390
+ const stored = await getBlobStore().put(key, doc, "text/markdown");
17391
+ return { fileUrl: stored.url, fileExpiresAt: new Date(Date.now() + SCRAPE_BLOB_TTL_MS).toISOString() };
17392
+ } catch {
17393
+ return null;
17394
+ }
17395
+ }
17396
+ async function persistScrapeBody(user, opts) {
17397
+ const deposit = await depositScrapeToVault(user, opts);
17398
+ if (deposit.deposited) return deposit;
17399
+ const fallback = await uploadFallbackBlob(opts);
17400
+ if (!fallback) return deposit;
17401
+ return { ...deposit, ...fallback };
17402
+ }
17403
+
17404
+ // src/api/scrape-blob-cleanup.ts
17405
+ import { readdir, stat, unlink } from "fs/promises";
17406
+ import { homedir as homedir4 } from "os";
17407
+ import { join as join7 } from "path";
17408
+ async function cleanupVercel(token, cutoff) {
17409
+ const { list, del } = await import("@vercel/blob");
17410
+ let cursor;
17411
+ let deleted = 0;
17412
+ do {
17413
+ const res = await list({ prefix: SCRAPE_FALLBACK_PREFIX, cursor, token });
17414
+ const expired = res.blobs.filter((b) => new Date(b.uploadedAt).getTime() < cutoff);
17415
+ if (expired.length) {
17416
+ await del(expired.map((b) => b.url), { token });
17417
+ deleted += expired.length;
17418
+ }
17419
+ cursor = res.hasMore ? res.cursor : void 0;
17420
+ } while (cursor);
17421
+ return { deleted, store: "vercel-blob" };
17422
+ }
17423
+ async function cleanupLocal(cutoff) {
17424
+ const baseDir = process.env.MCP_SCRAPER_OUTPUT_DIR?.trim() || join7(homedir4(), "Downloads", "mcp-scraper");
17425
+ const dir = join7(baseDir, "blobs", SCRAPE_FALLBACK_PREFIX.replace(/\/$/, ""));
17426
+ let deleted = 0;
17427
+ let entries;
17428
+ try {
17429
+ entries = await readdir(dir);
17430
+ } catch {
17431
+ return { deleted: 0, store: "local" };
17432
+ }
17433
+ for (const name of entries) {
17434
+ const path5 = join7(dir, name);
17435
+ try {
17436
+ const s = await stat(path5);
17437
+ if (s.isFile() && s.mtimeMs < cutoff) {
17438
+ await unlink(path5);
17439
+ deleted++;
17440
+ }
17441
+ } catch {
17442
+ }
17443
+ }
17444
+ return { deleted, store: "local" };
17445
+ }
17446
+ async function cleanupExpiredScrapeBlobs(maxAgeMs = SCRAPE_BLOB_TTL_MS) {
17447
+ const cutoff = Date.now() - maxAgeMs;
17448
+ try {
17449
+ const token = process.env.BLOB_READ_WRITE_TOKEN;
17450
+ return token ? await cleanupVercel(token, cutoff) : await cleanupLocal(cutoff);
17451
+ } catch {
17452
+ return { deleted: 0, store: "none" };
17453
+ }
17454
+ }
17455
+
16561
17456
  // src/api/credit-operations.ts
16562
- async function grantSignupCredit(_userId) {
16563
- return;
17457
+ async function grantSignupCredit(userId) {
17458
+ if (!FREE_SIGNUP_MC || FREE_SIGNUP_MC <= 0) return;
17459
+ const already = await ledgerExistsForOperation(userId, LedgerOperation.SIGNUP_GRANT);
17460
+ if (already) return;
17461
+ await creditMc(userId, FREE_SIGNUP_MC, LedgerOperation.SIGNUP_GRANT, "Welcome credits");
16564
17462
  }
16565
17463
  async function applyMonthlyFreeRefresh(user) {
16566
17464
  return user;
@@ -16598,46 +17496,168 @@ async function getFreeCreditBreakdown(userId) {
16598
17496
  };
16599
17497
  }
16600
17498
 
16601
- // src/api/session.ts
16602
- import { createHmac as createHmac2, timingSafeEqual } from "crypto";
16603
- var isProduction = () => process.env.NODE_ENV === "production" || process.env.VERCEL === "1";
16604
- function getSessionSecret() {
16605
- const configured = process.env.SESSION_SECRET?.trim();
16606
- if (configured) return configured;
16607
- if (isProduction()) throw new Error("SESSION_SECRET is not set \u2014 add it to your Vercel env vars (see src/api/env.ts)");
16608
- return "dev-secret-change-me";
17499
+ // src/api/memory-db.ts
17500
+ import { neon } from "@neondatabase/serverless";
17501
+ var _sql = null;
17502
+ function sql() {
17503
+ if (_sql) return _sql;
17504
+ const url = process.env.MEMORY_DATABASE_URL?.trim();
17505
+ if (!url) throw new Error("MEMORY_DATABASE_URL is not set (paa-crawler needs mcp-memory Neon URL for reads)");
17506
+ _sql = neon(url);
17507
+ return _sql;
17508
+ }
17509
+ function physicalVault(owner, vault) {
17510
+ return `${owner} ${vault}`;
17511
+ }
17512
+ function isUnlimited(plan) {
17513
+ return plan === "unlimited";
17514
+ }
17515
+ function quotaBytesForPlan(plan) {
17516
+ return MEMORY_PLAN_QUOTA[plan] ?? MEMORY_PLAN_QUOTA.free;
17517
+ }
17518
+ function toGb(bytes) {
17519
+ return Math.round(bytes / 1e9 * 100) / 100;
17520
+ }
17521
+ async function entitledVaultRows(identity) {
17522
+ const rows = await sql().query(
17523
+ `SELECT vault, owner_identity FROM mem_vault_registry WHERE grantee = $1 ORDER BY created_at ASC`,
17524
+ [identity]
17525
+ );
17526
+ const seen = /* @__PURE__ */ new Map();
17527
+ for (const r of rows) {
17528
+ if (seen.has(r.vault)) continue;
17529
+ seen.set(r.vault, { vault: r.vault, owner: r.owner_identity, physical: physicalVault(r.owner_identity, r.vault) });
17530
+ }
17531
+ return [...seen.values()];
17532
+ }
17533
+ async function storageByPhysical(physicals) {
17534
+ if (physicals.length === 0) return /* @__PURE__ */ new Map();
17535
+ const rows = await sql().query(
17536
+ `SELECT vault, COUNT(*)::int AS notes, COALESCE(SUM(octet_length(content)), 0)::bigint AS bytes
17537
+ FROM mem_notes WHERE vault = ANY($1) GROUP BY vault`,
17538
+ [physicals]
17539
+ );
17540
+ const m = /* @__PURE__ */ new Map();
17541
+ for (const r of rows) m.set(r.vault, { notes: Number(r.notes), bytes: Number(r.bytes) });
17542
+ return m;
17543
+ }
17544
+ async function dbListVaults(identity) {
17545
+ const ents = await entitledVaultRows(identity);
17546
+ const storage = await storageByPhysical(ents.map((e) => e.physical));
17547
+ return ents.map((e) => {
17548
+ const st = storage.get(e.physical);
17549
+ const isOwner = e.owner === identity;
17550
+ return {
17551
+ vault: e.vault,
17552
+ role: isOwner ? "owner" : "shared",
17553
+ ...isOwner ? {} : { sharedBy: e.owner },
17554
+ notes: st?.notes ?? 0,
17555
+ bytes: st?.bytes ?? 0
17556
+ };
17557
+ });
16609
17558
  }
16610
- var secret = () => getSessionSecret();
16611
- function safeEqualHex(a, b) {
16612
- if (a.length !== b.length) return false;
16613
- try {
16614
- return timingSafeEqual(Buffer.from(a, "hex"), Buffer.from(b, "hex"));
16615
- } catch {
16616
- return false;
16617
- }
17559
+ async function dbStorage(identity, plan) {
17560
+ const ents = await entitledVaultRows(identity);
17561
+ const storage = await storageByPhysical(ents.map((e) => e.physical));
17562
+ let totalBytes = 0;
17563
+ const perVault = ents.map((e) => {
17564
+ const st = storage.get(e.physical);
17565
+ const bytes = st?.bytes ?? 0;
17566
+ totalBytes += bytes;
17567
+ return { vault: e.vault, bytes, gb: toGb(bytes), notes: st?.notes ?? 0 };
17568
+ });
17569
+ const unlimited = isUnlimited(plan);
17570
+ const quotaBytes = unlimited ? null : quotaBytesForPlan(plan);
17571
+ return {
17572
+ ok: true,
17573
+ totalBytes,
17574
+ totalGb: toGb(totalBytes),
17575
+ quotaBytes,
17576
+ quotaGb: quotaBytes == null ? null : toGb(quotaBytes),
17577
+ unlimited,
17578
+ perVault
17579
+ };
16618
17580
  }
16619
- function signSession(userId) {
16620
- const payload = String(userId);
16621
- const sig = createHmac2("sha256", secret()).update(payload).digest("hex");
16622
- return `${payload}.${sig}`;
17581
+ async function resolvePhysical(identity, vault) {
17582
+ const rows = await sql().query(
17583
+ `SELECT owner_identity FROM mem_vault_registry WHERE grantee = $1 AND vault = $2 LIMIT 1`,
17584
+ [identity, vault]
17585
+ );
17586
+ if (!rows[0]) return null;
17587
+ return physicalVault(rows[0].owner_identity, vault);
17588
+ }
17589
+ async function dbNotes(identity, vault) {
17590
+ const physical = await resolvePhysical(identity, vault);
17591
+ if (!physical) return [];
17592
+ const rows = await sql().query(
17593
+ `SELECT path, title, kind, source, updated_at, octet_length(content) AS bytes, props
17594
+ FROM mem_notes WHERE vault = $1 ORDER BY updated_at DESC`,
17595
+ [physical]
17596
+ );
17597
+ return rows.map((r) => ({
17598
+ path: r.path,
17599
+ title: r.title,
17600
+ kind: r.kind,
17601
+ source: r.source,
17602
+ updatedAt: r.updated_at,
17603
+ bytes: Number(r.bytes),
17604
+ props: r.props && typeof r.props === "object" ? r.props : {}
17605
+ }));
16623
17606
  }
16624
- function verifySession(token) {
16625
- const dot = token.lastIndexOf(".");
16626
- if (dot === -1) return null;
16627
- const payload = token.slice(0, dot);
16628
- const sig = token.slice(dot + 1);
16629
- const expected = createHmac2("sha256", secret()).update(payload).digest("hex");
16630
- if (!safeEqualHex(sig, expected)) return null;
16631
- const id = parseInt(payload);
16632
- return isNaN(id) ? null : id;
17607
+ async function dbNote(identity, vault, path5) {
17608
+ const physical = await resolvePhysical(identity, vault);
17609
+ if (!physical) return null;
17610
+ const rows = await sql().query(
17611
+ `SELECT path, title, content, kind, source, updated_at, captured_at
17612
+ FROM mem_notes WHERE vault = $1 AND path = $2 LIMIT 1`,
17613
+ [physical, path5]
17614
+ );
17615
+ const r = rows[0];
17616
+ if (!r) return null;
17617
+ return {
17618
+ path: r.path,
17619
+ title: r.title,
17620
+ content: r.content,
17621
+ kind: r.kind,
17622
+ source: r.source,
17623
+ updatedAt: r.updated_at,
17624
+ capturedAt: r.captured_at
17625
+ };
17626
+ }
17627
+ var FREE_COST_CAP_USD = 1;
17628
+ async function dbUsage(identity, plan) {
17629
+ const now = /* @__PURE__ */ new Date();
17630
+ const period = `${now.getUTCFullYear()}-${String(now.getUTCMonth() + 1).padStart(2, "0")}`;
17631
+ const rows = await sql().query(
17632
+ `SELECT source, COALESCE(SUM(cost_usd), 0)::float8 AS total
17633
+ FROM mem_usage_ledger WHERE identity = $1 AND period = $2 GROUP BY source`,
17634
+ [identity, period]
17635
+ );
17636
+ const bySource = {};
17637
+ let costUsd = 0;
17638
+ for (const r of rows) {
17639
+ const v = Number(r.total);
17640
+ bySource[r.source] = v;
17641
+ costUsd += v;
17642
+ }
17643
+ return {
17644
+ ok: true,
17645
+ period,
17646
+ plan,
17647
+ costUsd,
17648
+ bySource,
17649
+ freeCapUsd: FREE_COST_CAP_USD,
17650
+ freeCapReached: plan === "free" && costUsd >= FREE_COST_CAP_USD,
17651
+ unlimited: isUnlimited(plan)
17652
+ };
16633
17653
  }
16634
17654
 
16635
17655
  // src/api/server.ts
16636
- var secureCookies = process.env.NODE_ENV === "production" || process.env.VERCEL === "1";
16637
- var isProduction2 = secureCookies;
16638
- var sessionCookieOptions = {
17656
+ var secureCookies2 = process.env.NODE_ENV === "production" || process.env.VERCEL === "1";
17657
+ var isProduction2 = secureCookies2;
17658
+ var sessionCookieOptions2 = {
16639
17659
  httpOnly: true,
16640
- secure: secureCookies,
17660
+ secure: secureCookies2,
16641
17661
  path: "/",
16642
17662
  maxAge: 60 * 60 * 24 * 30,
16643
17663
  sameSite: "Strict"
@@ -16698,7 +17718,7 @@ var adminAuth = createMiddleware3(async (c, next) => {
16698
17718
  return next();
16699
17719
  });
16700
17720
  var sessionAuth = createMiddleware3(async (c, next) => {
16701
- const token = getCookie(c, "session");
17721
+ const token = getCookie2(c, "session");
16702
17722
  if (!token) return c.json({ error: "Not authenticated" }, 401);
16703
17723
  const userId = verifySession(token);
16704
17724
  if (!userId) return c.json({ error: "Session expired" }, 401);
@@ -16709,7 +17729,7 @@ var sessionAuth = createMiddleware3(async (c, next) => {
16709
17729
  c.set("sessionUser", { ...refreshed, balance_mc: balanceMc });
16710
17730
  return next();
16711
17731
  });
16712
- var app = new Hono13();
17732
+ var app = new Hono15();
16713
17733
  var STRIPE_API_VERSION = "2026-02-25.clover";
16714
17734
  function requireStripeSecret() {
16715
17735
  const secret2 = process.env.STRIPE_SECRET_KEY?.trim();
@@ -16737,6 +17757,33 @@ app.use("*", async (c, next) => {
16737
17757
  c.header("X-Content-Type-Options", "nosniff");
16738
17758
  c.header("Referrer-Policy", "same-origin");
16739
17759
  });
17760
+ function opForCostPath(p) {
17761
+ if (p === "/harvest" || p === "/harvest/sync") return "harvest";
17762
+ if (p === "/extract-url") return "page_scrape";
17763
+ if (p === "/extract-site") return "extract_site";
17764
+ if (p === "/map-urls") return "url_map";
17765
+ if (p === "/youtube/harvest") return "yt_channel";
17766
+ if (p === "/youtube/transcribe") return "yt_transcription";
17767
+ if (p === "/maps/search") return "maps_search";
17768
+ if (p === "/maps/place") return "maps_place";
17769
+ if (p.startsWith("/facebook/")) {
17770
+ if (p.endsWith("transcribe")) return "fb_transcribe";
17771
+ if (p.endsWith("/search")) return "fb_search";
17772
+ return "fb_ad";
17773
+ }
17774
+ if (p.startsWith("/instagram/")) return "instagram";
17775
+ if (p.startsWith("/api/internal/site-architecture-auditor")) return "audit_site";
17776
+ if (p.startsWith("/agent/")) return "browser_session";
17777
+ return null;
17778
+ }
17779
+ app.use("*", (c, next) => {
17780
+ const op = opForCostPath(new URL(c.req.url).pathname);
17781
+ if (!op) return next();
17782
+ const probeRunId = c.req.header("x-cost-probe-id") ?? null;
17783
+ const forceHeadless = c.req.header("x-kernel-headless") === "true";
17784
+ return runWithCostContext({ op, probeRunId, forceHeadless }, () => next());
17785
+ });
17786
+ void migrateCostTelemetry();
16740
17787
  app.post("/auth/register", requireAllowedOrigin, async (c) => {
16741
17788
  const { email, password } = await c.req.json();
16742
17789
  const normalizedEmail = email?.trim().toLowerCase();
@@ -16768,8 +17815,16 @@ app.post("/auth/register", requireAllowedOrigin, async (c) => {
16768
17815
  }
16769
17816
  const newKey = await rotateApiKey(user.id);
16770
17817
  await grantSignupCredit(user.id);
17818
+ void (async () => {
17819
+ try {
17820
+ const full = await getUserById(user.id);
17821
+ if (full) await provisionMemoryForUser(full);
17822
+ } catch (err) {
17823
+ console.warn("[auth/register] memory provision failed (will retry on first write):", err instanceof Error ? err.message : String(err));
17824
+ }
17825
+ })();
16771
17826
  const token = signSession(user.id);
16772
- setCookie(c, "session", token, sessionCookieOptions);
17827
+ setCookie2(c, "session", token, sessionCookieOptions2);
16773
17828
  return c.json({ email: user.email, api_key: newKey }, 201);
16774
17829
  } catch (err) {
16775
17830
  const msg = err instanceof Error ? err.message : "";
@@ -16788,11 +17843,11 @@ app.post("/auth/login", requireAllowedOrigin, async (c) => {
16788
17843
  return c.json({ error: "Invalid email or password" }, 401);
16789
17844
  }
16790
17845
  const token = signSession(user.id);
16791
- setCookie(c, "session", token, sessionCookieOptions);
17846
+ setCookie2(c, "session", token, sessionCookieOptions2);
16792
17847
  return c.json({ email: user.email, api_key: user.key_active ? user.api_key : null });
16793
17848
  });
16794
17849
  app.post("/auth/logout", requireAllowedOrigin, (c) => {
16795
- deleteCookie(c, "session", { path: "/", secure: secureCookies, sameSite: "Strict" });
17850
+ deleteCookie(c, "session", { path: "/", secure: secureCookies2, sameSite: "Strict" });
16796
17851
  return c.json({ ok: true });
16797
17852
  });
16798
17853
  app.post("/auth/forgot-password", requireAllowedOrigin, async (c) => {
@@ -16846,16 +17901,16 @@ app.get("/catalog", (c) => {
16846
17901
  return c.json(CATALOG);
16847
17902
  });
16848
17903
  app.get("/me", async (c) => {
16849
- const token = getCookie(c, "session");
17904
+ const token = getCookie2(c, "session");
16850
17905
  if (!token) return c.json({ authenticated: false });
16851
17906
  const userId = verifySession(token);
16852
17907
  if (!userId) {
16853
- deleteCookie(c, "session", { path: "/", secure: secureCookies, sameSite: "Strict" });
17908
+ deleteCookie(c, "session", { path: "/", secure: secureCookies2, sameSite: "Strict" });
16854
17909
  return c.json({ authenticated: false });
16855
17910
  }
16856
17911
  const foundUser = await getUserById(userId);
16857
17912
  if (!foundUser) {
16858
- deleteCookie(c, "session", { path: "/", secure: secureCookies, sameSite: "Strict" });
17913
+ deleteCookie(c, "session", { path: "/", secure: secureCookies2, sameSite: "Strict" });
16859
17914
  return c.json({ authenticated: false });
16860
17915
  }
16861
17916
  const refreshed = await applyMonthlyFreeRefresh(foundUser);
@@ -16874,6 +17929,8 @@ app.get("/me", async (c) => {
16874
17929
  has_concurrency_sub: !!user.concurrency_stripe_sub_id,
16875
17930
  subscription_tier: user.subscription_tier,
16876
17931
  subscription_concurrency: user.subscription_concurrency,
17932
+ memory_plan: user.memory_plan ?? "free",
17933
+ has_memory_sub: !!user.memory_subscription_id,
16877
17934
  ...stats
16878
17935
  });
16879
17936
  });
@@ -16887,6 +17944,227 @@ app.delete("/api-key", requireAllowedOrigin, sessionAuth, async (c) => {
16887
17944
  await revokeApiKey(user.id);
16888
17945
  return c.json({ ok: true });
16889
17946
  });
17947
+ function maybeProvisionInBackground(user, vaultCount) {
17948
+ if (vaultCount > 0 || user.memory_provisioned === 1) return;
17949
+ void (async () => {
17950
+ try {
17951
+ if (await getMemoryProvisioned(user.id)) return;
17952
+ await provisionMemoryForUser(user);
17953
+ } catch (err) {
17954
+ console.warn("[memory] background provision failed:", err instanceof Error ? err.message : String(err));
17955
+ }
17956
+ })();
17957
+ }
17958
+ app.get("/memory/overview", auth2, async (c) => {
17959
+ const user = c.get("user");
17960
+ const identity = resolveMemoryIdentity(user);
17961
+ const plan = memoryPlanForReads(user);
17962
+ const requestedVault = c.req.query("vault")?.trim() || null;
17963
+ try {
17964
+ const [vaults, storage, usage] = await Promise.all([
17965
+ dbListVaults(identity),
17966
+ dbStorage(identity, plan),
17967
+ dbUsage(identity, plan)
17968
+ ]);
17969
+ maybeProvisionInBackground(user, vaults.length);
17970
+ const activeVault = requestedVault && vaults.some((v) => v.vault === requestedVault) ? requestedVault : vaults[0]?.vault ?? null;
17971
+ const notes = activeVault ? await dbNotes(identity, activeVault) : [];
17972
+ const billing = {
17973
+ ok: true,
17974
+ memory_plan: user.memory_plan ?? "free",
17975
+ has_memory_sub: !!user.memory_subscription_id,
17976
+ free_quota_gb: MEMORY_PLAN_QUOTA.free / 1e9,
17977
+ plans: Object.values(MEMORY_PLANS).map((p) => ({ plan: p.plan, label: p.label, price_id: p.price_id, interval: p.interval, monthly_usd: p.monthly_usd, quota_gb: p.quota_bytes / 1e9 }))
17978
+ };
17979
+ return c.json({ ok: true, vaults, storage, activeVault, notes, usage, billing });
17980
+ } catch (err) {
17981
+ const message = err instanceof Error ? err.message : "memory overview failed";
17982
+ console.error("[memory/overview]", message);
17983
+ return c.json({ ok: false, error: message }, 502);
17984
+ }
17985
+ });
17986
+ app.get("/memory/vaults", auth2, async (c) => {
17987
+ const user = c.get("user");
17988
+ try {
17989
+ const vaults = await dbListVaults(resolveMemoryIdentity(user));
17990
+ maybeProvisionInBackground(user, vaults.length);
17991
+ return c.json({ ok: true, vaults });
17992
+ } catch (err) {
17993
+ return c.json({ ok: false, error: err instanceof Error ? err.message : "could not load vaults" }, 502);
17994
+ }
17995
+ });
17996
+ app.get("/memory/connect", auth2, async (c) => {
17997
+ const { key, error } = await getOrCreateUserMemoryKey(c.get("user"));
17998
+ if (!key) return c.json({ error: error ?? "memory unavailable" }, 502);
17999
+ const command = `MCP_MEMORY_KEY=${key} npx -y -p @vilovieta/mcp-memory@latest mcp-memory`;
18000
+ const config = {
18001
+ mcpServers: {
18002
+ "mcp-memory": {
18003
+ command: "npx",
18004
+ args: ["-y", "-p", "@vilovieta/mcp-memory@latest", "mcp-memory"],
18005
+ env: { MCP_MEMORY_KEY: key }
18006
+ }
18007
+ }
18008
+ };
18009
+ return c.json({ ok: true, command, config });
18010
+ });
18011
+ app.post("/memory/vault", auth2, async (c) => {
18012
+ const { key, error } = await getOrCreateUserMemoryKey(c.get("user"));
18013
+ if (!key) return c.json({ error: error ?? "memory unavailable" }, 502);
18014
+ const body = await c.req.json().catch(() => ({}));
18015
+ const vault = body.vault?.trim();
18016
+ if (!vault) return c.json({ error: "vault name required" }, 400);
18017
+ const result = await memoryCall("addVaultTool", { vault }, key);
18018
+ return c.json(result, result.ok ? 200 : 502);
18019
+ });
18020
+ app.get("/memory/notes", auth2, async (c) => {
18021
+ const user = c.get("user");
18022
+ const identity = resolveMemoryIdentity(user);
18023
+ try {
18024
+ let vault = c.req.query("vault")?.trim() || null;
18025
+ if (!vault) {
18026
+ const vaults = await dbListVaults(identity);
18027
+ vault = vaults[0]?.vault ?? null;
18028
+ }
18029
+ if (!vault) return c.json({ ok: true, notes: [], vault: null });
18030
+ const notes = await dbNotes(identity, vault);
18031
+ return c.json({ ok: true, vault, notes });
18032
+ } catch (err) {
18033
+ return c.json({ ok: false, error: err instanceof Error ? err.message : "could not load notes" }, 502);
18034
+ }
18035
+ });
18036
+ app.get("/memory/note", auth2, async (c) => {
18037
+ const user = c.get("user");
18038
+ const vault = c.req.query("vault")?.trim();
18039
+ const path5 = c.req.query("path");
18040
+ if (!path5) return c.json({ error: "path required" }, 400);
18041
+ if (!vault) return c.json({ ok: false, error: "vault required" }, 400);
18042
+ try {
18043
+ const note = await dbNote(resolveMemoryIdentity(user), vault, path5);
18044
+ if (!note) return c.json({ ok: false, error: "note not found" }, 404);
18045
+ return c.json({ ok: true, note });
18046
+ } catch (err) {
18047
+ return c.json({ ok: false, error: err instanceof Error ? err.message : "could not load note" }, 502);
18048
+ }
18049
+ });
18050
+ app.put("/memory/note", auth2, async (c) => {
18051
+ const { key, error } = await getOrCreateUserMemoryKey(c.get("user"));
18052
+ if (!key) return c.json({ error: error ?? "memory unavailable" }, 502);
18053
+ const body = await c.req.json().catch(() => ({}));
18054
+ if (!body.path) return c.json({ error: "path required" }, 400);
18055
+ const result = await memoryCall("putTool", { vault: body.vault, path: body.path, title: body.title, content: body.content }, key);
18056
+ return c.json(result, result.ok ? 200 : 502);
18057
+ });
18058
+ app.delete("/memory/note", auth2, async (c) => {
18059
+ const { key, error } = await getOrCreateUserMemoryKey(c.get("user"));
18060
+ if (!key) return c.json({ error: error ?? "memory unavailable" }, 502);
18061
+ const vault = c.req.query("vault");
18062
+ const path5 = c.req.query("path");
18063
+ if (!path5) return c.json({ error: "path required" }, 400);
18064
+ const result = await memoryCall("deleteNoteTool", { vault, path: path5 }, key);
18065
+ return c.json(result, result.ok ? 200 : 502);
18066
+ });
18067
+ app.get("/memory/note/download", auth2, async (c) => {
18068
+ const user = c.get("user");
18069
+ const vault = c.req.query("vault")?.trim();
18070
+ const path5 = c.req.query("path");
18071
+ if (!path5) return c.json({ error: "path required" }, 400);
18072
+ if (!vault) return c.json({ error: "vault required" }, 400);
18073
+ const note = await dbNote(resolveMemoryIdentity(user), vault, path5);
18074
+ if (!note) return c.json({ error: "note not found" }, 404);
18075
+ const filename = (path5.split("/").pop() || "note").replace(/"/g, "");
18076
+ return new Response(note.content ?? "", {
18077
+ status: 200,
18078
+ headers: {
18079
+ "Content-Type": "text/markdown; charset=utf-8",
18080
+ "Content-Disposition": `attachment; filename="${filename.endsWith(".md") ? filename : filename + ".md"}"`
18081
+ }
18082
+ });
18083
+ });
18084
+ app.get("/memory/storage", auth2, async (c) => {
18085
+ const user = c.get("user");
18086
+ try {
18087
+ const result = await dbStorage(resolveMemoryIdentity(user), memoryPlanForReads(user));
18088
+ return c.json(result);
18089
+ } catch (err) {
18090
+ return c.json({ ok: false, error: err instanceof Error ? err.message : "could not load storage" }, 502);
18091
+ }
18092
+ });
18093
+ app.get("/memory/billing", auth2, async (c) => {
18094
+ const user = c.get("user");
18095
+ const plans = Object.values(MEMORY_PLANS).map((p) => ({ plan: p.plan, label: p.label, price_id: p.price_id, interval: p.interval, monthly_usd: p.monthly_usd, quota_gb: p.quota_bytes / 1e9 }));
18096
+ return c.json({
18097
+ ok: true,
18098
+ memory_plan: user.memory_plan ?? "free",
18099
+ has_memory_sub: !!user.memory_subscription_id,
18100
+ free_quota_gb: MEMORY_PLAN_QUOTA.free / 1e9,
18101
+ plans
18102
+ });
18103
+ });
18104
+ app.post("/memory/checkout", auth2, async (c) => {
18105
+ try {
18106
+ const user = c.get("user");
18107
+ const body = await c.req.json().catch(() => ({}));
18108
+ const priceId = body.priceId?.trim();
18109
+ if (!priceId || !MEMORY_PLANS[priceId]) return c.json({ error: "Invalid priceId \u2014 must be a memory plan price." }, 400);
18110
+ const secret2 = process.env.STRIPE_SECRET_KEY?.trim();
18111
+ if (!secret2) return c.json({ error: "Stripe is not configured." }, 503);
18112
+ const stripeClient = new Stripe2(secret2, { apiVersion: STRIPE_API_VERSION });
18113
+ let customerId = user.stripe_customer_id;
18114
+ if (!customerId) {
18115
+ const customer = await stripeClient.customers.create({ email: user.email });
18116
+ customerId = customer.id;
18117
+ await setStripeCustomerId(user.id, customerId);
18118
+ }
18119
+ const session = await stripeClient.checkout.sessions.create({
18120
+ mode: "subscription",
18121
+ customer: customerId,
18122
+ line_items: [{ price: priceId, quantity: 1 }],
18123
+ client_reference_id: String(user.id),
18124
+ metadata: { product: "mcp-memory", userId: String(user.id) },
18125
+ subscription_data: { metadata: { product: "mcp-memory", userId: String(user.id) } },
18126
+ automatic_tax: { enabled: true },
18127
+ billing_address_collection: "required",
18128
+ customer_update: { address: "auto", name: "auto" },
18129
+ tax_id_collection: { enabled: true },
18130
+ success_url: `${appOrigin()}/memory?billing=success`,
18131
+ cancel_url: `${appOrigin()}/memory?billing=cancel`
18132
+ });
18133
+ if (!session.url) return c.json({ error: "Stripe did not return a checkout URL." }, 502);
18134
+ return c.json({ url: session.url });
18135
+ } catch (err) {
18136
+ const message = err instanceof Error ? err.message : "Unable to start memory checkout.";
18137
+ console.error("[memory/checkout]", message);
18138
+ return c.json({ error: message }, 500);
18139
+ }
18140
+ });
18141
+ app.post("/memory/portal", auth2, async (c) => {
18142
+ try {
18143
+ const user = c.get("user");
18144
+ if (!user.stripe_customer_id) return c.json({ error: "No billing account yet \u2014 subscribe first." }, 409);
18145
+ const secret2 = process.env.STRIPE_SECRET_KEY?.trim();
18146
+ if (!secret2) return c.json({ error: "Stripe is not configured." }, 503);
18147
+ const stripeClient = new Stripe2(secret2, { apiVersion: STRIPE_API_VERSION });
18148
+ const session = await stripeClient.billingPortal.sessions.create({
18149
+ customer: user.stripe_customer_id,
18150
+ return_url: `${appOrigin()}/memory`
18151
+ });
18152
+ return c.json({ url: session.url });
18153
+ } catch (err) {
18154
+ const message = err instanceof Error ? err.message : "Unable to open billing portal.";
18155
+ console.error("[memory/portal]", message);
18156
+ return c.json({ error: message }, 500);
18157
+ }
18158
+ });
18159
+ app.get("/memory/usage", auth2, async (c) => {
18160
+ const user = c.get("user");
18161
+ try {
18162
+ const result = await dbUsage(resolveMemoryIdentity(user), memoryPlanForReads(user));
18163
+ return c.json(result);
18164
+ } catch (err) {
18165
+ return c.json({ ok: false, error: err instanceof Error ? err.message : "could not load usage" }, 502);
18166
+ }
18167
+ });
16890
18168
  var SYNC_HARVEST_TIMEOUT_OVERRIDE_MS = (() => {
16891
18169
  const raw = process.env.SYNC_HARVEST_TIMEOUT_MS;
16892
18170
  const parsed = raw === void 0 ? NaN : Number(raw);
@@ -17013,6 +18291,7 @@ app.post("/harvest/sync", auth2, async (c) => {
17013
18291
  format: "json",
17014
18292
  outputDir: "/tmp/paa-output-api",
17015
18293
  signal: syncSignal,
18294
+ softDeadlineMs: Date.now() + Math.floor(syncTimeoutMs * 0.8),
17016
18295
  onAttemptEvent: recordAttempt
17017
18296
  });
17018
18297
  await completeJob(jobId, result);
@@ -17165,8 +18444,14 @@ app.post("/extract-url", auth2, async (c) => {
17165
18444
  screenshotMeta = { base64: screenshotBuf.toString("base64"), sizeBytes: screenshotBuf.length, device };
17166
18445
  }
17167
18446
  const mediaMeta = downloadMedia ? await harvestPageMedia(result.bodyHtml, canonicalUrl, { types: mediaTypes ?? ["image", "video", "audio"], outputDir: null }) : null;
18447
+ const memory = bodyResult.data.depositToVault ? await persistScrapeBody(user, {
18448
+ title: result.title ?? canonicalUrl,
18449
+ content: result.bodyMarkdown ?? "",
18450
+ source: canonicalUrl,
18451
+ vault: bodyResult.data.vaultName
18452
+ }) : void 0;
17168
18453
  await logRequestEvent({ userId: user.id, source: "extract_url", status: "done", query: canonicalUrl, resultCount: result.headings.length, result });
17169
- return c.json({ ...result, screenshot: screenshotMeta, branding: brandingData, media: mediaMeta });
18454
+ return c.json({ ...result, screenshot: screenshotMeta, branding: brandingData, media: mediaMeta, memory });
17170
18455
  } catch (err) {
17171
18456
  const msg = err instanceof Error ? err.message : String(err);
17172
18457
  if (debited) await creditMc(user.id, MC_COSTS.page_scrape, LedgerOperation.EXTRACT_URL, "failed call");
@@ -17584,16 +18869,17 @@ app.get("/cron/tick", async (c) => {
17584
18869
  if (!process.env.CRON_SECRET || secret2 !== `Bearer ${process.env.CRON_SECRET}`) {
17585
18870
  return c.json({ error: "Unauthorized" }, 401);
17586
18871
  }
17587
- const { drainQueue } = await import("./worker-UM5UM6O5.js");
18872
+ const { drainQueue } = await import("./worker-UC6D2756.js");
17588
18873
  const budget = { maxJobs: 10, deadlineMs: Date.now() + 28e4 };
17589
18874
  const workflowDispatchResult = await dispatchDueWorkflowSchedules(`${new URL(c.req.url).protocol}//${new URL(c.req.url).host}`);
17590
- const [results, sweepResult, reapResult, expiredResult] = await Promise.all([
18875
+ const [results, sweepResult, reapResult, expiredResult, blobCleanup] = await Promise.all([
17591
18876
  drainQueue(budget),
17592
18877
  runMonthlyRefreshSweep(),
17593
18878
  reapIdleBrowserSessions(120).catch(() => ({ reaped: 0 })),
17594
- expireOldLots().catch(() => ({ expired_lots: 0, expired_mc: 0 }))
18879
+ expireOldLots().catch(() => ({ expired_lots: 0, expired_mc: 0 })),
18880
+ cleanupExpiredScrapeBlobs().catch(() => ({ deleted: 0, store: "none" }))
17595
18881
  ]);
17596
- return c.json({ drained: results.length, results, sweepResult, reaped: reapResult, expired: expiredResult, workflowDispatch: workflowDispatchResult });
18882
+ return c.json({ drained: results.length, results, sweepResult, reaped: reapResult, expired: expiredResult, blobCleanup, workflowDispatch: workflowDispatchResult });
17597
18883
  });
17598
18884
  app.on(["GET", "POST", "PUT"], "/api/inngest", serveInngest({ client: inngest, functions: [siteAuditFn, siteExtractFn] }));
17599
18885
  app.route("/api/internal/site-architecture-auditor", siteAuditApp);
@@ -17601,6 +18887,7 @@ app.route("/youtube", youtubeApp);
17601
18887
  app.route("/screenshot", screenshotApp);
17602
18888
  app.route("/facebook", facebookAdApp);
17603
18889
  app.route("/instagram", instagramApp);
18890
+ app.route("/reddit", redditApp);
17604
18891
  app.route("/maps", mapsApp);
17605
18892
  app.route("/directory", directoryApp);
17606
18893
  app.route("/workflows", workflowApp);
@@ -17623,6 +18910,7 @@ app.get("/console/auth/:id/go", async (c) => {
17623
18910
  });
17624
18911
  app.get("/console/:id", (c) => c.html(renderConsoleHtml(c.req.param("id"))));
17625
18912
  app.route("/stripe", stripeApp);
18913
+ app.route("/", oauthApp);
17626
18914
  if (!process.env.INNGEST_EVENT_KEY) {
17627
18915
  startSiteAuditWorker();
17628
18916
  }
@@ -17729,4 +19017,4 @@ app.get("/blog/:slug/", (c) => {
17729
19017
  export {
17730
19018
  app
17731
19019
  };
17732
- //# sourceMappingURL=server-2SRARAO5.js.map
19020
+ //# sourceMappingURL=server-HIGHKTEV.js.map