mcp-scraper 0.3.24 → 0.3.25
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bin/api-server.cjs +8140 -5941
- package/dist/bin/api-server.cjs.map +1 -1
- package/dist/bin/api-server.js +3 -3
- package/dist/bin/mcp-scraper-cli.cjs +320 -58
- package/dist/bin/mcp-scraper-cli.cjs.map +1 -1
- package/dist/bin/mcp-scraper-cli.js +4 -3
- package/dist/bin/mcp-scraper-cli.js.map +1 -1
- package/dist/bin/mcp-scraper-install.cjs +3 -3
- package/dist/bin/mcp-scraper-install.cjs.map +1 -1
- package/dist/bin/mcp-scraper-install.js +2 -2
- package/dist/bin/mcp-stdio-server.cjs +121 -24
- package/dist/bin/mcp-stdio-server.cjs.map +1 -1
- package/dist/bin/mcp-stdio-server.js +5 -5
- package/dist/bin/paa-harvest.cjs +284 -10
- package/dist/bin/paa-harvest.cjs.map +1 -1
- package/dist/bin/paa-harvest.js +5 -3
- package/dist/bin/paa-harvest.js.map +1 -1
- package/dist/{chunk-ZAP7MYVO.js → chunk-3PRO376E.js} +2 -3
- package/dist/chunk-3PRO376E.js.map +1 -0
- package/dist/{chunk-LFATOGDF.js → chunk-BP27CZ5Q.js} +4 -3
- package/dist/chunk-BP27CZ5Q.js.map +1 -0
- package/dist/{chunk-DE7WP4UU.js → chunk-DOBQN3EY.js} +3 -3
- package/dist/chunk-DOBQN3EY.js.map +1 -0
- package/dist/{chunk-GZ4W72WF.js → chunk-FE2WC4JR.js} +119 -23
- package/dist/chunk-FE2WC4JR.js.map +1 -0
- package/dist/{chunk-IGOEAKC7.js → chunk-G7PQ64LM.js} +274 -9
- package/dist/chunk-G7PQ64LM.js.map +1 -0
- package/dist/chunk-K443GQY5.js +24 -0
- package/dist/chunk-K443GQY5.js.map +1 -0
- package/dist/{chunk-H2R232HK.js → chunk-NNEIXK5L.js} +307 -62
- package/dist/chunk-NNEIXK5L.js.map +1 -0
- package/dist/{chunk-3KYRG7O7.js → chunk-TRNSXR46.js} +185 -2
- package/dist/chunk-TRNSXR46.js.map +1 -0
- package/dist/chunk-WPHAMNT7.js +7 -0
- package/dist/chunk-WPHAMNT7.js.map +1 -0
- package/dist/{chunk-JTWQGPKF.js → chunk-WYCER2HW.js} +33 -10
- package/dist/chunk-WYCER2HW.js.map +1 -0
- package/dist/{db-EG5ETPTY.js → db-FP2ABUU4.js} +32 -2
- package/dist/index.cjs +284 -10
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +7 -3
- package/dist/index.d.ts +7 -3
- package/dist/index.js +5 -3
- package/dist/index.js.map +1 -1
- package/dist/{server-2SRARAO5.js → server-RRCKVBHG.js} +1563 -265
- package/dist/server-RRCKVBHG.js.map +1 -0
- package/dist/{worker-UM5UM6O5.js → worker-UC6D2756.js} +7 -6
- package/dist/{worker-UM5UM6O5.js.map → worker-UC6D2756.js.map} +1 -1
- package/docs/kernel-proxy-future-enhancements.md +80 -0
- package/docs/spec-browser-routing.md +108 -0
- package/docs/spec-mcp-tools.md +146 -0
- package/docs/specs/audit-visual-demo.md +309 -0
- package/package.json +5 -2
- package/dist/chunk-3KYRG7O7.js.map +0 -1
- package/dist/chunk-DE7WP4UU.js.map +0 -1
- package/dist/chunk-GZ4W72WF.js.map +0 -1
- package/dist/chunk-H2R232HK.js.map +0 -1
- package/dist/chunk-H7R4PPMG.js +0 -7
- package/dist/chunk-H7R4PPMG.js.map +0 -1
- package/dist/chunk-IGOEAKC7.js.map +0 -1
- package/dist/chunk-JTWQGPKF.js.map +0 -1
- package/dist/chunk-LFATOGDF.js.map +0 -1
- package/dist/chunk-ZAP7MYVO.js.map +0 -1
- package/dist/server-2SRARAO5.js.map +0 -1
- /package/dist/{db-EG5ETPTY.js.map → db-FP2ABUU4.js.map} +0 -0
|
@@ -21,26 +21,30 @@ import {
|
|
|
21
21
|
renderLinkReport,
|
|
22
22
|
sanitizeAttempts,
|
|
23
23
|
sanitizeHarvestResult
|
|
24
|
-
} from "./chunk-
|
|
24
|
+
} from "./chunk-FE2WC4JR.js";
|
|
25
25
|
import {
|
|
26
26
|
csvRecords,
|
|
27
27
|
listWorkflowDefinitions,
|
|
28
28
|
rowsToCsv,
|
|
29
29
|
runWorkflow,
|
|
30
30
|
runWorkflowStep,
|
|
31
|
+
slugify,
|
|
31
32
|
workflowDefinition,
|
|
32
33
|
workflowOutputBaseDir,
|
|
33
34
|
workflowStepCount,
|
|
34
35
|
workflowSupportsSteps
|
|
35
|
-
} from "./chunk-
|
|
36
|
-
import "./chunk-
|
|
36
|
+
} from "./chunk-NNEIXK5L.js";
|
|
37
|
+
import "./chunk-WPHAMNT7.js";
|
|
37
38
|
import {
|
|
38
39
|
BROWSER_OPEN_MIN_BALANCE_MC,
|
|
39
40
|
CONCURRENCY_PRICE_ID,
|
|
40
41
|
CREDIT_COST_CATALOG,
|
|
42
|
+
FREE_SIGNUP_MC,
|
|
41
43
|
LedgerOperation,
|
|
42
44
|
MC_COSTS,
|
|
43
45
|
MC_PER_CREDIT,
|
|
46
|
+
MEMORY_PLANS,
|
|
47
|
+
MEMORY_PLAN_QUOTA,
|
|
44
48
|
SUBSCRIPTION_TIERS,
|
|
45
49
|
SUBSCRIPTION_TIER_BY_KEY,
|
|
46
50
|
browserActiveCostMc,
|
|
@@ -50,15 +54,20 @@ import {
|
|
|
50
54
|
harvestProblemResponse,
|
|
51
55
|
insufficientBalanceResponse,
|
|
52
56
|
serializeHarvestProblem
|
|
53
|
-
} from "./chunk-
|
|
57
|
+
} from "./chunk-WYCER2HW.js";
|
|
54
58
|
import {
|
|
55
59
|
BrowserDriver,
|
|
56
60
|
MapsSelectors,
|
|
57
61
|
buildYouTubeChannelVideosUrl,
|
|
62
|
+
currentCostContext,
|
|
58
63
|
deleteKernelProxyId,
|
|
59
64
|
harvest,
|
|
60
|
-
|
|
61
|
-
|
|
65
|
+
migrateCostTelemetry,
|
|
66
|
+
recordKernelSession,
|
|
67
|
+
recordVendorUsage,
|
|
68
|
+
resolveKernelProxyId,
|
|
69
|
+
runWithCostContext
|
|
70
|
+
} from "./chunk-G7PQ64LM.js";
|
|
62
71
|
import {
|
|
63
72
|
CaptchaError,
|
|
64
73
|
RECAPTCHA_INSTRUCTIONS,
|
|
@@ -67,7 +76,8 @@ import {
|
|
|
67
76
|
browserServiceProfileSaveChanges,
|
|
68
77
|
browserServiceProxyId,
|
|
69
78
|
sanitizeVendorName
|
|
70
|
-
} from "./chunk-
|
|
79
|
+
} from "./chunk-3PRO376E.js";
|
|
80
|
+
import "./chunk-K443GQY5.js";
|
|
71
81
|
import {
|
|
72
82
|
DEFAULT_MAPS_PROXY_MODE,
|
|
73
83
|
DEFAULT_PROXY_MODE,
|
|
@@ -77,7 +87,7 @@ import {
|
|
|
77
87
|
RawMapsHoursRowSchema,
|
|
78
88
|
RawMapsOverviewSchema,
|
|
79
89
|
RawMapsReviewStatsSchema
|
|
80
|
-
} from "./chunk-
|
|
90
|
+
} from "./chunk-BP27CZ5Q.js";
|
|
81
91
|
import {
|
|
82
92
|
SiteAuditJobRowSchema,
|
|
83
93
|
advanceWorkflowRunStep,
|
|
@@ -86,6 +96,7 @@ import {
|
|
|
86
96
|
checkRateLimit,
|
|
87
97
|
completeJob,
|
|
88
98
|
completeWorkflowRunRecord,
|
|
99
|
+
consumeCode,
|
|
89
100
|
consumePasswordResetToken,
|
|
90
101
|
countActiveUsers,
|
|
91
102
|
createJob,
|
|
@@ -101,13 +112,18 @@ import {
|
|
|
101
112
|
expireOldLots,
|
|
102
113
|
failJob,
|
|
103
114
|
failWorkflowRunRecord,
|
|
115
|
+
getClient,
|
|
104
116
|
getDb,
|
|
105
117
|
getJob,
|
|
106
118
|
getLedger,
|
|
119
|
+
getMemoryPlan,
|
|
120
|
+
getMemoryProvisioned,
|
|
121
|
+
getRefresh,
|
|
107
122
|
getUserByApiKey,
|
|
108
123
|
getUserByEmail,
|
|
109
124
|
getUserById,
|
|
110
125
|
getUserByStripeCustomerId,
|
|
126
|
+
getUserMemoryCreds,
|
|
111
127
|
getUserStats,
|
|
112
128
|
getWorkflowArtifact,
|
|
113
129
|
getWorkflowRun,
|
|
@@ -127,19 +143,27 @@ import {
|
|
|
127
143
|
markWorkflowRunRunning,
|
|
128
144
|
markWorkflowScheduleRan,
|
|
129
145
|
patchWorkflowSchedule,
|
|
146
|
+
putCode,
|
|
147
|
+
putRefresh,
|
|
130
148
|
reconcileBalanceMc,
|
|
131
149
|
recordStripeEvent,
|
|
132
150
|
recordWorkflowWebhookDelivery,
|
|
151
|
+
registerClient,
|
|
133
152
|
replaceWorkflowArtifacts,
|
|
153
|
+
resolveUserByMemorySubscription,
|
|
134
154
|
revokeApiKey,
|
|
135
155
|
rotateApiKey,
|
|
156
|
+
rotateRefresh,
|
|
136
157
|
setConcurrencySubId,
|
|
137
158
|
setExtraConcurrencySlots,
|
|
159
|
+
setMemoryPlan,
|
|
160
|
+
setMemoryProvisioned,
|
|
138
161
|
setPassword,
|
|
139
162
|
setStripeCustomerId,
|
|
140
163
|
setSubscriptionTier,
|
|
164
|
+
setUserMemoryCreds,
|
|
141
165
|
verifyPassword
|
|
142
|
-
} from "./chunk-
|
|
166
|
+
} from "./chunk-TRNSXR46.js";
|
|
143
167
|
|
|
144
168
|
// src/blog/registry.ts
|
|
145
169
|
var posts = [
|
|
@@ -3511,7 +3535,9 @@ async function fetchWithKernel(url) {
|
|
|
3511
3535
|
const apiKey = browserServiceApiKey();
|
|
3512
3536
|
if (!apiKey) throw new Error("Browser backend API key not set");
|
|
3513
3537
|
const client2 = new Kernel({ apiKey });
|
|
3514
|
-
const
|
|
3538
|
+
const openedAtMs = Date.now();
|
|
3539
|
+
const useHeadless = currentCostContext()?.forceHeadful !== true;
|
|
3540
|
+
const kb = await client2.browsers.create({ stealth: true, ...useHeadless ? { headless: true } : {}, timeout_seconds: 60 });
|
|
3515
3541
|
const browser = await chromium.connectOverCDP(kb.cdp_ws_url);
|
|
3516
3542
|
try {
|
|
3517
3543
|
const context = browser.contexts()[0] ?? await browser.newContext({
|
|
@@ -3526,6 +3552,7 @@ async function fetchWithKernel(url) {
|
|
|
3526
3552
|
});
|
|
3527
3553
|
await client2.browsers.deleteByID(kb.session_id).catch(() => {
|
|
3528
3554
|
});
|
|
3555
|
+
void recordKernelSession({ kernelSessionId: kb.session_id, source: "kernel_fetch", stealth: typeof kb.stealth === "boolean" ? kb.stealth : null, headlessSent: useHeadless ? true : null, proxyUsed: typeof kb.proxy_id === "string" ? Boolean(kb.proxy_id) : null, fallback: true, openedAtMs, closedAtMs: Date.now() });
|
|
3529
3556
|
}
|
|
3530
3557
|
}
|
|
3531
3558
|
|
|
@@ -4127,6 +4154,7 @@ async function captureLocal(url, device, wantScreenshot, wantBranding) {
|
|
|
4127
4154
|
}
|
|
4128
4155
|
async function captureKernel(url, kernelApiKey, device, wantScreenshot, wantBranding) {
|
|
4129
4156
|
const kernel = new Kernel2({ apiKey: kernelApiKey });
|
|
4157
|
+
const openedAtMs = Date.now();
|
|
4130
4158
|
const session = await kernel.browsers.create({ stealth: true, timeout_seconds: 240 });
|
|
4131
4159
|
const browser = await chromium2.connectOverCDP(session.cdp_ws_url);
|
|
4132
4160
|
const vp = device === "mobile" ? MOBILE_VIEWPORT : DESKTOP_VIEWPORT;
|
|
@@ -4146,6 +4174,7 @@ async function captureKernel(url, kernelApiKey, device, wantScreenshot, wantBran
|
|
|
4146
4174
|
return result;
|
|
4147
4175
|
} finally {
|
|
4148
4176
|
await browser.close();
|
|
4177
|
+
void recordKernelSession({ kernelSessionId: session.session_id, source: "screenshot", stealth: typeof session.stealth === "boolean" ? session.stealth : null, headlessSent: null, proxyUsed: typeof session.proxy_id === "string" ? Boolean(session.proxy_id) : null, openedAtMs, closedAtMs: Date.now() });
|
|
4149
4178
|
}
|
|
4150
4179
|
}
|
|
4151
4180
|
async function captureScreenshot(url, kernelApiKey, device = "desktop") {
|
|
@@ -4800,14 +4829,19 @@ async function crawlWithRotation(urls, opts) {
|
|
|
4800
4829
|
let kb = null;
|
|
4801
4830
|
let browser = null;
|
|
4802
4831
|
let inBrowser = 0;
|
|
4832
|
+
let kbOpenedAtMs = 0;
|
|
4803
4833
|
const retire = () => {
|
|
4804
|
-
const b = browser, k = kb;
|
|
4834
|
+
const b = browser, k = kb, openedAtMs = kbOpenedAtMs;
|
|
4805
4835
|
browser = null;
|
|
4806
4836
|
kb = null;
|
|
4837
|
+
kbOpenedAtMs = 0;
|
|
4807
4838
|
if (b) void b.close().catch(() => {
|
|
4808
4839
|
});
|
|
4809
|
-
if (k)
|
|
4810
|
-
|
|
4840
|
+
if (k) {
|
|
4841
|
+
void client2.browsers.deleteByID(k.session_id).catch(() => {
|
|
4842
|
+
});
|
|
4843
|
+
void recordKernelSession({ kernelSessionId: k.session_id, source: "rotating_crawl", stealth: typeof k.stealth === "boolean" ? k.stealth : null, headlessSent: headless, proxyUsed: typeof k.proxy_id === "string" ? Boolean(k.proxy_id) : null, openedAtMs, closedAtMs: Date.now() });
|
|
4844
|
+
}
|
|
4811
4845
|
};
|
|
4812
4846
|
try {
|
|
4813
4847
|
while (true) {
|
|
@@ -4819,6 +4853,7 @@ async function crawlWithRotation(urls, opts) {
|
|
|
4819
4853
|
if (!browser || inBrowser >= urlsPerBrowser) {
|
|
4820
4854
|
retire();
|
|
4821
4855
|
kb = await createBrowserWithRetry(client2, headless);
|
|
4856
|
+
kbOpenedAtMs = Date.now();
|
|
4822
4857
|
browser = await withTimeout(chromium3.connectOverCDP(kb.cdp_ws_url), 3e4);
|
|
4823
4858
|
inBrowser = 0;
|
|
4824
4859
|
}
|
|
@@ -5226,7 +5261,7 @@ async function extractSite(opts) {
|
|
|
5226
5261
|
}
|
|
5227
5262
|
|
|
5228
5263
|
// src/api/server.ts
|
|
5229
|
-
import { Hono as
|
|
5264
|
+
import { Hono as Hono15 } from "hono";
|
|
5230
5265
|
import { serve as serveInngest } from "inngest/hono";
|
|
5231
5266
|
|
|
5232
5267
|
// src/inngest/client.ts
|
|
@@ -5301,6 +5336,7 @@ var DeepInfraLlmClient = class {
|
|
|
5301
5336
|
});
|
|
5302
5337
|
if (!resp.ok) throw new Error(`DeepInfra error ${resp.status}: ${await resp.text()}`);
|
|
5303
5338
|
const data = await resp.json();
|
|
5339
|
+
void recordVendorUsage({ vendor: "deepinfra_qwen", model: this.model, units: data.usage?.completion_tokens ?? 0, unitType: "tokens" });
|
|
5304
5340
|
return data.choices[0].message.content;
|
|
5305
5341
|
}
|
|
5306
5342
|
async completeJson(prompt) {
|
|
@@ -8924,6 +8960,8 @@ async function attemptKernelWhisper(videoId, kernelApiKey, kernelProxyId, falKey
|
|
|
8924
8960
|
timestamp: c.timestamp,
|
|
8925
8961
|
text: c.text.trim()
|
|
8926
8962
|
}));
|
|
8963
|
+
const audioSec = chunks.length ? chunks[chunks.length - 1]?.timestamp?.[1] ?? 0 : 0;
|
|
8964
|
+
void recordVendorUsage({ vendor: "fal_wizper", model: "fal-ai/wizper", units: audioSec / 60, unitType: "audio_min" });
|
|
8927
8965
|
const text = data.text ?? chunks.map((c) => c.text).join(" ");
|
|
8928
8966
|
if (!text) return null;
|
|
8929
8967
|
return { videoId, text, chunks, durationMs: Date.now() - start, method: "browser-whisper" };
|
|
@@ -9007,7 +9045,9 @@ var ExtractUrlBodySchema = z11.object({
|
|
|
9007
9045
|
extractBranding: z11.boolean().optional(),
|
|
9008
9046
|
downloadMedia: z11.boolean().optional(),
|
|
9009
9047
|
mediaTypes: z11.array(z11.enum(["image", "video", "audio"])).optional(),
|
|
9010
|
-
allowLocal: z11.boolean().optional()
|
|
9048
|
+
allowLocal: z11.boolean().optional(),
|
|
9049
|
+
depositToVault: z11.boolean().optional(),
|
|
9050
|
+
vaultName: z11.string().trim().min(1).max(120).optional()
|
|
9011
9051
|
});
|
|
9012
9052
|
var MapUrlsBodySchema = z11.object({
|
|
9013
9053
|
url: z11.string().min(1, "url is required"),
|
|
@@ -10351,6 +10391,8 @@ async function transcribeMediaUrl(mediaUrl, markdownTitle = "# Media Transcript"
|
|
|
10351
10391
|
const text = data.text ?? "";
|
|
10352
10392
|
const chunks = data.chunks ?? [];
|
|
10353
10393
|
const durationMs = Date.now() - startMs;
|
|
10394
|
+
const audioSec = chunks.length ? chunks[chunks.length - 1]?.timestamp?.[1] ?? 0 : 0;
|
|
10395
|
+
void recordVendorUsage({ vendor: "fal_wizper", model: "fal-ai/wizper", units: audioSec / 60, unitType: "audio_min" });
|
|
10354
10396
|
return {
|
|
10355
10397
|
text,
|
|
10356
10398
|
chunks,
|
|
@@ -10410,21 +10452,24 @@ function buildPageIntelUrl(body, country) {
|
|
|
10410
10452
|
if (body.pageId?.trim()) return `https://www.facebook.com/ads/library/?active_status=all&ad_type=all&country=${country}&is_targeted_country=false&media_type=all&search_type=page&view_all_page_id=${body.pageId.trim()}`;
|
|
10411
10453
|
return `https://www.facebook.com/ads/library/?active_status=all&ad_type=all&country=${country}&q=${encodeURIComponent(body.query.trim())}&search_type=keyword_unordered`;
|
|
10412
10454
|
}
|
|
10455
|
+
function configuredKernelProxy() {
|
|
10456
|
+
return (process.env.BROWSER_SERVICE_PROXY_ID ?? process.env.KERNEL_PROXY_ID)?.trim() || void 0;
|
|
10457
|
+
}
|
|
10413
10458
|
function kernelLaunchOpts() {
|
|
10414
|
-
return { headless: true, kernelApiKey: browserServiceApiKey(), kernelProxyId:
|
|
10459
|
+
return { headless: true, kernelApiKey: browserServiceApiKey(), kernelProxyId: configuredKernelProxy(), viewport: { width: 1280, height: 900 }, locale: "en-US" };
|
|
10415
10460
|
}
|
|
10416
10461
|
async function kernelLaunchOptsResidential() {
|
|
10417
|
-
let proxyId =
|
|
10462
|
+
let proxyId = configuredKernelProxy();
|
|
10418
10463
|
try {
|
|
10419
10464
|
const resolution = await resolveKernelProxyId({
|
|
10420
10465
|
kernelApiKey: browserServiceApiKey(),
|
|
10421
10466
|
proxyMode: DEFAULT_PROXY_MODE,
|
|
10422
|
-
configuredKernelProxyId:
|
|
10467
|
+
configuredKernelProxyId: configuredKernelProxy(),
|
|
10423
10468
|
gl: "us"
|
|
10424
10469
|
});
|
|
10425
10470
|
if (resolution.kernelProxyId) proxyId = resolution.kernelProxyId;
|
|
10426
10471
|
} catch {
|
|
10427
|
-
proxyId =
|
|
10472
|
+
proxyId = configuredKernelProxy();
|
|
10428
10473
|
}
|
|
10429
10474
|
return { headless: true, kernelApiKey: browserServiceApiKey(), kernelProxyId: proxyId, viewport: { width: 1280, height: 900 }, locale: "en-US" };
|
|
10430
10475
|
}
|
|
@@ -11162,10 +11207,14 @@ function resolveProfileName(body) {
|
|
|
11162
11207
|
const explicit = body.profile?.trim();
|
|
11163
11208
|
return explicit || browserServiceProfileName();
|
|
11164
11209
|
}
|
|
11210
|
+
function configuredKernelProxy2() {
|
|
11211
|
+
return (process.env.BROWSER_SERVICE_PROXY_ID ?? process.env.KERNEL_PROXY_ID)?.trim() || void 0;
|
|
11212
|
+
}
|
|
11165
11213
|
async function kernelLaunchOptsDirect(profileName, saveProfileChanges) {
|
|
11166
11214
|
return {
|
|
11167
11215
|
headless: true,
|
|
11168
11216
|
kernelApiKey: browserServiceApiKey(),
|
|
11217
|
+
kernelProxyId: configuredKernelProxy2(),
|
|
11169
11218
|
...profileName ? { kernelProfileName: profileName } : {},
|
|
11170
11219
|
...typeof saveProfileChanges === "boolean" ? { kernelProfileSaveChanges: saveProfileChanges } : {},
|
|
11171
11220
|
viewport: { width: 1280, height: 900 },
|
|
@@ -11420,8 +11469,136 @@ instagramApp.post("/media-download", createApiKeyAuth(), async (c) => {
|
|
|
11420
11469
|
}
|
|
11421
11470
|
});
|
|
11422
11471
|
|
|
11423
|
-
// src/api/
|
|
11472
|
+
// src/api/reddit-routes.ts
|
|
11424
11473
|
import { Hono as Hono6 } from "hono";
|
|
11474
|
+
import { z as z15 } from "zod";
|
|
11475
|
+
var RedditThreadBodySchema = z15.object({
|
|
11476
|
+
url: z15.string().trim().min(1, "url is required"),
|
|
11477
|
+
maxComments: z15.number().int().min(1).max(2e3).optional()
|
|
11478
|
+
});
|
|
11479
|
+
function invalidRequest3(message) {
|
|
11480
|
+
return { error_code: "invalid_request", message };
|
|
11481
|
+
}
|
|
11482
|
+
function toOldReddit(url) {
|
|
11483
|
+
try {
|
|
11484
|
+
const u = new URL(url.trim());
|
|
11485
|
+
if (!/(^|\.)reddit\.com$/.test(u.hostname.toLowerCase()) && u.hostname.toLowerCase() !== "redd.it") return null;
|
|
11486
|
+
u.hostname = "old.reddit.com";
|
|
11487
|
+
u.protocol = "https:";
|
|
11488
|
+
return u.toString();
|
|
11489
|
+
} catch {
|
|
11490
|
+
return null;
|
|
11491
|
+
}
|
|
11492
|
+
}
|
|
11493
|
+
async function residentialProxyId(attemptIndex) {
|
|
11494
|
+
try {
|
|
11495
|
+
const r = await resolveKernelProxyId({
|
|
11496
|
+
kernelApiKey: browserServiceApiKey(),
|
|
11497
|
+
proxyMode: "location",
|
|
11498
|
+
location: "Austin, TX",
|
|
11499
|
+
gl: "us",
|
|
11500
|
+
attemptIndex,
|
|
11501
|
+
fresh: true
|
|
11502
|
+
});
|
|
11503
|
+
return r.kernelProxyId;
|
|
11504
|
+
} catch {
|
|
11505
|
+
return void 0;
|
|
11506
|
+
}
|
|
11507
|
+
}
|
|
11508
|
+
var PARSE_REDDIT = `(() => {
|
|
11509
|
+
const txt = el => ((el && el.innerText) || '').trim();
|
|
11510
|
+
const attr = (el, a) => ((el && el.getAttribute(a)) || '').trim();
|
|
11511
|
+
const bodyText = (document.body && document.body.innerText) || '';
|
|
11512
|
+
const blocked = /whoa there|blocked by network|you've been blocked|network (policy|security)|log in to your reddit account/i.test(bodyText);
|
|
11513
|
+
const post = document.querySelector('#siteTable .thing.link') || document.querySelector('.thing.link');
|
|
11514
|
+
const title = txt(document.querySelector('a.title')) || (document.title || '');
|
|
11515
|
+
const author = attr(post, 'data-author');
|
|
11516
|
+
const scoreEl = post && post.querySelector('.score.unvoted');
|
|
11517
|
+
const score = scoreEl ? ((scoreEl.getAttribute('title') || scoreEl.textContent || '').trim()) : '';
|
|
11518
|
+
const postBody = txt(post && post.querySelector('.usertext-body .md'));
|
|
11519
|
+
const comments = [];
|
|
11520
|
+
document.querySelectorAll('.commentarea .comment').forEach(c => {
|
|
11521
|
+
const entry = c.querySelector(':scope > .entry');
|
|
11522
|
+
if (!entry) return;
|
|
11523
|
+
const a = attr(c, 'data-author');
|
|
11524
|
+
const b = txt(entry.querySelector('.usertext-body .md'));
|
|
11525
|
+
if (!b) return;
|
|
11526
|
+
const s = ((entry.querySelector('.score.unvoted') || {}).textContent || '').trim();
|
|
11527
|
+
let depth = 0, p = c.parentElement;
|
|
11528
|
+
while (p) { if (p.classList && p.classList.contains('comment')) depth++; p = p.parentElement; }
|
|
11529
|
+
comments.push({ author: a, score: s, depth: depth, body: b });
|
|
11530
|
+
});
|
|
11531
|
+
return { title: title, author: author, score: score, postBody: postBody, blocked: blocked, numComments: comments.length, comments: comments };
|
|
11532
|
+
})()`;
|
|
11533
|
+
var redditApp = new Hono6();
|
|
11534
|
+
redditApp.post("/thread", createApiKeyAuth(), async (c) => {
|
|
11535
|
+
const raw = await c.req.json().catch(() => ({}));
|
|
11536
|
+
const parsed = RedditThreadBodySchema.safeParse(raw);
|
|
11537
|
+
if (!parsed.success) return c.json(invalidRequest3(parsed.error.issues[0]?.message ?? "Invalid request"), 400);
|
|
11538
|
+
const body = parsed.data;
|
|
11539
|
+
const oldUrl = toOldReddit(body.url);
|
|
11540
|
+
if (!oldUrl) return c.json(invalidRequest3("url must be a reddit.com link"), 400);
|
|
11541
|
+
const user = c.get("user");
|
|
11542
|
+
const gate = await acquireConcurrencyGate(user, "reddit_thread", {
|
|
11543
|
+
reuseLockId: c.req.header("x-mcp-scraper-concurrency-lock"),
|
|
11544
|
+
metadata: { url: body.url }
|
|
11545
|
+
});
|
|
11546
|
+
if (!gate.ok) return c.json(concurrencyLimitExceededResponse(gate), 429, { "Retry-After": String(gate.retryAfterSeconds) });
|
|
11547
|
+
let debited = false;
|
|
11548
|
+
let refunded = false;
|
|
11549
|
+
try {
|
|
11550
|
+
const { ok, balance_mc } = await debitMc(user.id, MC_COSTS.reddit_thread, LedgerOperation.REDDIT_THREAD, body.url);
|
|
11551
|
+
if (!ok) return c.json(insufficientBalanceResponse(balance_mc, MC_COSTS.reddit_thread), 402);
|
|
11552
|
+
debited = true;
|
|
11553
|
+
let result = null;
|
|
11554
|
+
for (let attempt = 0; attempt < 3 && !result; attempt++) {
|
|
11555
|
+
const proxyId = await residentialProxyId(attempt);
|
|
11556
|
+
const driver = new BrowserDriver();
|
|
11557
|
+
try {
|
|
11558
|
+
await driver.launch({
|
|
11559
|
+
headless: true,
|
|
11560
|
+
headlessMode: "headful",
|
|
11561
|
+
kernelApiKey: browserServiceApiKey(),
|
|
11562
|
+
kernelProxyId: proxyId,
|
|
11563
|
+
viewport: { width: 1280, height: 900 },
|
|
11564
|
+
locale: "en-US"
|
|
11565
|
+
});
|
|
11566
|
+
const page = driver.getPage();
|
|
11567
|
+
await page.goto(oldUrl, { waitUntil: "domcontentloaded", timeout: 45e3 });
|
|
11568
|
+
await page.waitForTimeout(2500);
|
|
11569
|
+
const data = await page.evaluate(PARSE_REDDIT);
|
|
11570
|
+
if (!data.blocked && (data.postBody || data.comments.length > 0)) {
|
|
11571
|
+
result = { ...data, sourceUrl: body.url, oldRedditUrl: oldUrl, attempts: attempt + 1 };
|
|
11572
|
+
}
|
|
11573
|
+
} catch {
|
|
11574
|
+
} finally {
|
|
11575
|
+
await driver.close();
|
|
11576
|
+
}
|
|
11577
|
+
}
|
|
11578
|
+
if (!result) {
|
|
11579
|
+
await creditMc(user.id, MC_COSTS.reddit_thread, LedgerOperation.REDDIT_THREAD_REFUND, "reddit blocked/empty after retries");
|
|
11580
|
+
refunded = true;
|
|
11581
|
+
await logRequestEvent({ userId: user.id, source: "reddit_thread", status: "failed", query: body.url, error: "reddit blocked/empty after retries" });
|
|
11582
|
+
return c.json({ error: "Reddit blocked the request after retries (refunded)" }, 503);
|
|
11583
|
+
}
|
|
11584
|
+
if (body.maxComments && Array.isArray(result.comments)) {
|
|
11585
|
+
result.comments = result.comments.slice(0, body.maxComments);
|
|
11586
|
+
result.numComments = result.comments.length;
|
|
11587
|
+
}
|
|
11588
|
+
await logRequestEvent({ userId: user.id, source: "reddit_thread", status: "done", query: body.url, resultCount: Number(result.numComments ?? 0), result });
|
|
11589
|
+
return c.json(result);
|
|
11590
|
+
} catch (err) {
|
|
11591
|
+
if (debited && !refunded) await creditMc(user.id, MC_COSTS.reddit_thread, LedgerOperation.REDDIT_THREAD_REFUND, "failed call");
|
|
11592
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
11593
|
+
await logRequestEvent({ userId: user.id, source: "reddit_thread", status: "failed", query: body.url, error: msg });
|
|
11594
|
+
return c.json({ error: msg }, 500);
|
|
11595
|
+
} finally {
|
|
11596
|
+
await releaseConcurrencyGate(gate.lockId);
|
|
11597
|
+
}
|
|
11598
|
+
});
|
|
11599
|
+
|
|
11600
|
+
// src/api/maps-routes.ts
|
|
11601
|
+
import { Hono as Hono7 } from "hono";
|
|
11425
11602
|
|
|
11426
11603
|
// src/extractor/MapsNavigator.ts
|
|
11427
11604
|
var MapsNavigator = class {
|
|
@@ -11596,7 +11773,7 @@ var MapsReviewCollector = class {
|
|
|
11596
11773
|
};
|
|
11597
11774
|
|
|
11598
11775
|
// src/extractor/MapsExtractor.ts
|
|
11599
|
-
import { z as
|
|
11776
|
+
import { z as z16 } from "zod";
|
|
11600
11777
|
var MapsExtractor = class {
|
|
11601
11778
|
constructor(driver) {
|
|
11602
11779
|
this.driver = driver;
|
|
@@ -11782,7 +11959,7 @@ var MapsExtractor = class {
|
|
|
11782
11959
|
});
|
|
11783
11960
|
return rows;
|
|
11784
11961
|
}, { hoursTable: MapsSelectors.hoursTable, hoursTableAlt: MapsSelectors.hoursTableAlt });
|
|
11785
|
-
const result =
|
|
11962
|
+
const result = z16.array(RawMapsHoursRowSchema).safeParse(raw);
|
|
11786
11963
|
if (!result.success) {
|
|
11787
11964
|
console.warn("[MapsExtractor] hours parse failed", result.error.flatten());
|
|
11788
11965
|
return [];
|
|
@@ -11846,7 +12023,7 @@ var MapsExtractor = class {
|
|
|
11846
12023
|
});
|
|
11847
12024
|
return results;
|
|
11848
12025
|
});
|
|
11849
|
-
const result =
|
|
12026
|
+
const result = z16.array(RawMapsAboutAttributeSchema).safeParse(raw);
|
|
11850
12027
|
if (!result.success) {
|
|
11851
12028
|
console.warn("[MapsExtractor] about parse failed", result.error.flatten());
|
|
11852
12029
|
return [];
|
|
@@ -12107,7 +12284,7 @@ async function cleanupDisposableProxy(kernelApiKey, proxyId, eventName) {
|
|
|
12107
12284
|
}
|
|
12108
12285
|
async function runMapsSearchWithRotation(options) {
|
|
12109
12286
|
const attempts = [];
|
|
12110
|
-
const maxAttempts = mapsSearchMaxAttemptsForProxyMode(options.proxyMode);
|
|
12287
|
+
const maxAttempts = mapsSearchMaxAttemptsForProxyMode(options.proxyMode) + 1;
|
|
12111
12288
|
const started = Date.now();
|
|
12112
12289
|
let lastError = null;
|
|
12113
12290
|
for (let attemptIndex = 0; attemptIndex < maxAttempts; attemptIndex += 1) {
|
|
@@ -12129,12 +12306,15 @@ async function runMapsSearchWithRotation(options) {
|
|
|
12129
12306
|
fresh: options.proxyMode === "location"
|
|
12130
12307
|
});
|
|
12131
12308
|
disposableProxyId = resolution.disposableProxyId;
|
|
12132
|
-
const
|
|
12309
|
+
const extractArgs = {
|
|
12133
12310
|
...options,
|
|
12134
12311
|
proxyZip,
|
|
12135
12312
|
kernelProxyId: options.proxyMode === "none" ? void 0 : resolution.kernelProxyId,
|
|
12136
12313
|
kernelProxyResolution: resolution.resolution
|
|
12137
|
-
}
|
|
12314
|
+
};
|
|
12315
|
+
const isHeadfulEscalation = attemptIndex === maxAttempts - 1;
|
|
12316
|
+
const baseCtx = currentCostContext();
|
|
12317
|
+
const result = isHeadfulEscalation && baseCtx ? await runWithCostContext({ ...baseCtx, forceHeadful: true }, () => extractor.extract(extractArgs)) : await extractor.extract(extractArgs);
|
|
12138
12318
|
const attempt = attemptFromDebug({
|
|
12139
12319
|
attemptIndex,
|
|
12140
12320
|
maxAttempts,
|
|
@@ -12219,7 +12399,7 @@ function mapsErrorResponse(c, err, errorCode) {
|
|
|
12219
12399
|
attempts: rotationError?.attempts ?? void 0
|
|
12220
12400
|
}, retryable ? 503 : 500);
|
|
12221
12401
|
}
|
|
12222
|
-
var mapsApp = new
|
|
12402
|
+
var mapsApp = new Hono7();
|
|
12223
12403
|
mapsApp.post("/search", createApiKeyAuth(), async (c) => {
|
|
12224
12404
|
const user = c.get("user");
|
|
12225
12405
|
const body = await c.req.json().catch(() => ({}));
|
|
@@ -12357,12 +12537,12 @@ mapsApp.post("/place", createApiKeyAuth(), async (c) => {
|
|
|
12357
12537
|
});
|
|
12358
12538
|
|
|
12359
12539
|
// src/api/directory-routes.ts
|
|
12360
|
-
import { Hono as
|
|
12540
|
+
import { Hono as Hono8 } from "hono";
|
|
12361
12541
|
|
|
12362
12542
|
// src/directory/directory-workflow.ts
|
|
12363
12543
|
import { mkdir as mkdir2, writeFile } from "fs/promises";
|
|
12364
12544
|
import { join as join6 } from "path";
|
|
12365
|
-
import { z as
|
|
12545
|
+
import { z as z17 } from "zod";
|
|
12366
12546
|
|
|
12367
12547
|
// src/directory/location-db.ts
|
|
12368
12548
|
import { access, readFile } from "fs/promises";
|
|
@@ -12523,24 +12703,24 @@ async function resolveDirectoryMarkets(options) {
|
|
|
12523
12703
|
}
|
|
12524
12704
|
|
|
12525
12705
|
// src/directory/directory-workflow.ts
|
|
12526
|
-
var DirectoryWorkflowOptionsSchema =
|
|
12527
|
-
query:
|
|
12528
|
-
state:
|
|
12529
|
-
minPopulation:
|
|
12530
|
-
populationYear:
|
|
12531
|
-
maxCities:
|
|
12532
|
-
maxResultsPerCity:
|
|
12533
|
-
concurrency:
|
|
12534
|
-
includeZipGroups:
|
|
12535
|
-
usZipsCsvPath:
|
|
12536
|
-
saveCsv:
|
|
12537
|
-
gl:
|
|
12538
|
-
hl:
|
|
12539
|
-
proxyMode:
|
|
12540
|
-
proxyZip:
|
|
12541
|
-
debug:
|
|
12542
|
-
headless:
|
|
12543
|
-
kernelApiKey:
|
|
12706
|
+
var DirectoryWorkflowOptionsSchema = z17.object({
|
|
12707
|
+
query: z17.string().min(1),
|
|
12708
|
+
state: z17.string().min(2).default("TN"),
|
|
12709
|
+
minPopulation: z17.number().int().min(0).default(1e5),
|
|
12710
|
+
populationYear: z17.union(POPULATION_YEARS.map((year) => z17.literal(year))).default(2025),
|
|
12711
|
+
maxCities: z17.number().int().min(1).max(100).default(25),
|
|
12712
|
+
maxResultsPerCity: z17.number().int().min(1).max(50).default(50),
|
|
12713
|
+
concurrency: z17.number().int().min(1).max(5).default(5),
|
|
12714
|
+
includeZipGroups: z17.boolean().default(true),
|
|
12715
|
+
usZipsCsvPath: z17.string().optional(),
|
|
12716
|
+
saveCsv: z17.boolean().default(true),
|
|
12717
|
+
gl: z17.string().length(2).default("us"),
|
|
12718
|
+
hl: z17.string().length(2).default("en"),
|
|
12719
|
+
proxyMode: z17.enum(["location", "configured", "none"]).default(DEFAULT_MAPS_PROXY_MODE),
|
|
12720
|
+
proxyZip: z17.string().regex(/^\d{5}$/).optional(),
|
|
12721
|
+
debug: z17.boolean().default(false),
|
|
12722
|
+
headless: z17.boolean().default(true),
|
|
12723
|
+
kernelApiKey: z17.string().optional()
|
|
12544
12724
|
});
|
|
12545
12725
|
function errorMessage(err) {
|
|
12546
12726
|
return err instanceof Error ? err.message : String(err);
|
|
@@ -12761,7 +12941,7 @@ async function runDirectoryWorkflowFromPlan(options, plan) {
|
|
|
12761
12941
|
}
|
|
12762
12942
|
|
|
12763
12943
|
// src/api/directory-routes.ts
|
|
12764
|
-
var directoryApp = new
|
|
12944
|
+
var directoryApp = new Hono8();
|
|
12765
12945
|
directoryApp.post("/run", createApiKeyAuth(), async (c) => {
|
|
12766
12946
|
const user = c.get("user");
|
|
12767
12947
|
const body = await c.req.json().catch(() => ({}));
|
|
@@ -12834,35 +13014,35 @@ directoryApp.post("/run", createApiKeyAuth(), async (c) => {
|
|
|
12834
13014
|
// src/api/workflow-routes.ts
|
|
12835
13015
|
import { createHmac } from "crypto";
|
|
12836
13016
|
import { readFile as readFile2 } from "fs/promises";
|
|
12837
|
-
import { Hono as
|
|
12838
|
-
import { z as
|
|
12839
|
-
var workflowApp = new
|
|
12840
|
-
var WorkflowInputSchema =
|
|
12841
|
-
var WorkflowIdSchema =
|
|
12842
|
-
var CadenceSchema =
|
|
12843
|
-
var ScheduleStatusSchema =
|
|
12844
|
-
var RunBodySchema =
|
|
13017
|
+
import { Hono as Hono9 } from "hono";
|
|
13018
|
+
import { z as z18 } from "zod";
|
|
13019
|
+
var workflowApp = new Hono9();
|
|
13020
|
+
var WorkflowInputSchema = z18.record(z18.unknown()).default({});
|
|
13021
|
+
var WorkflowIdSchema = z18.string().min(1);
|
|
13022
|
+
var CadenceSchema = z18.enum(["daily", "weekly", "monthly"]);
|
|
13023
|
+
var ScheduleStatusSchema = z18.enum(["active", "paused"]);
|
|
13024
|
+
var RunBodySchema = z18.object({
|
|
12845
13025
|
workflowId: WorkflowIdSchema,
|
|
12846
13026
|
input: WorkflowInputSchema,
|
|
12847
|
-
webhookUrl:
|
|
13027
|
+
webhookUrl: z18.string().url().optional()
|
|
12848
13028
|
});
|
|
12849
|
-
var ScheduleCreateSchema =
|
|
13029
|
+
var ScheduleCreateSchema = z18.object({
|
|
12850
13030
|
workflowId: WorkflowIdSchema,
|
|
12851
|
-
name:
|
|
13031
|
+
name: z18.string().min(1).max(120).optional(),
|
|
12852
13032
|
input: WorkflowInputSchema,
|
|
12853
13033
|
cadence: CadenceSchema.default("weekly"),
|
|
12854
|
-
timezone:
|
|
12855
|
-
webhookUrl:
|
|
12856
|
-
nextRunAt:
|
|
13034
|
+
timezone: z18.string().min(1).max(64).default("UTC"),
|
|
13035
|
+
webhookUrl: z18.string().url().optional(),
|
|
13036
|
+
nextRunAt: z18.string().datetime().optional()
|
|
12857
13037
|
});
|
|
12858
|
-
var SchedulePatchSchema =
|
|
12859
|
-
name:
|
|
13038
|
+
var SchedulePatchSchema = z18.object({
|
|
13039
|
+
name: z18.string().min(1).max(120).optional(),
|
|
12860
13040
|
status: ScheduleStatusSchema.optional(),
|
|
12861
13041
|
input: WorkflowInputSchema.optional(),
|
|
12862
13042
|
cadence: CadenceSchema.optional(),
|
|
12863
|
-
timezone:
|
|
12864
|
-
webhookUrl:
|
|
12865
|
-
nextRunAt:
|
|
13043
|
+
timezone: z18.string().min(1).max(64).optional(),
|
|
13044
|
+
webhookUrl: z18.string().url().nullable().optional(),
|
|
13045
|
+
nextRunAt: z18.string().datetime().nullable().optional()
|
|
12866
13046
|
});
|
|
12867
13047
|
function hostedWorkflowOutputDir() {
|
|
12868
13048
|
return workflowOutputBaseDir(process.env.MCP_SCRAPER_WORKFLOW_OUTPUT_DIR?.trim() || "/tmp/mcp-scraper-workflows");
|
|
@@ -13366,7 +13546,7 @@ workflowApp.post("/cron/dispatch", async (c) => {
|
|
|
13366
13546
|
});
|
|
13367
13547
|
|
|
13368
13548
|
// src/api/serp-intelligence-routes.ts
|
|
13369
|
-
import { Hono as
|
|
13549
|
+
import { Hono as Hono10 } from "hono";
|
|
13370
13550
|
|
|
13371
13551
|
// src/serp-intelligence/page-snapshot-extractor.ts
|
|
13372
13552
|
import { createHash as createHash2 } from "crypto";
|
|
@@ -13678,7 +13858,7 @@ async function capturePageSnapshots(targets, options = {}) {
|
|
|
13678
13858
|
}
|
|
13679
13859
|
|
|
13680
13860
|
// src/serp-intelligence/schemas.ts
|
|
13681
|
-
import { z as
|
|
13861
|
+
import { z as z19 } from "zod";
|
|
13682
13862
|
var SerpIntelligenceDeviceValues = ["desktop", "mobile"];
|
|
13683
13863
|
var SerpIntelligenceProxyModeValues = ["location", "configured", "none"];
|
|
13684
13864
|
var SerpIntelligenceAttemptOutcomeValues = [
|
|
@@ -13740,171 +13920,171 @@ function isPublicHttpUrl(value) {
|
|
|
13740
13920
|
return false;
|
|
13741
13921
|
}
|
|
13742
13922
|
}
|
|
13743
|
-
var SerpIntelligencePublicHttpUrlSchema =
|
|
13744
|
-
var SerpIntelligenceCaptureBodySchema =
|
|
13745
|
-
query:
|
|
13746
|
-
location:
|
|
13747
|
-
gl:
|
|
13748
|
-
hl:
|
|
13749
|
-
device:
|
|
13750
|
-
proxyMode:
|
|
13751
|
-
proxyZip:
|
|
13752
|
-
pages:
|
|
13753
|
-
debug:
|
|
13754
|
-
includePageSnapshots:
|
|
13755
|
-
pageSnapshotLimit:
|
|
13923
|
+
var SerpIntelligencePublicHttpUrlSchema = z19.string().url().refine(isPublicHttpUrl, "url must be a public HTTP or HTTPS URL");
|
|
13924
|
+
var SerpIntelligenceCaptureBodySchema = z19.object({
|
|
13925
|
+
query: z19.string().trim().min(1, "query is required"),
|
|
13926
|
+
location: z19.string().trim().min(1).optional(),
|
|
13927
|
+
gl: z19.string().trim().length(2).default("us"),
|
|
13928
|
+
hl: z19.string().trim().length(2).default("en"),
|
|
13929
|
+
device: z19.enum(SerpIntelligenceDeviceValues).default("desktop"),
|
|
13930
|
+
proxyMode: z19.enum(SerpIntelligenceProxyModeValues).default(DEFAULT_PROXY_MODE),
|
|
13931
|
+
proxyZip: z19.string().regex(/^\d{5}$/).optional(),
|
|
13932
|
+
pages: z19.number().int().min(1).max(2).default(1),
|
|
13933
|
+
debug: z19.boolean().default(false),
|
|
13934
|
+
includePageSnapshots: z19.boolean().default(false),
|
|
13935
|
+
pageSnapshotLimit: z19.number().int().min(0).max(10).default(0)
|
|
13756
13936
|
}).strict();
|
|
13757
|
-
var SerpIntelligencePageSnapshotRequestSchema =
|
|
13937
|
+
var SerpIntelligencePageSnapshotRequestSchema = z19.object({
|
|
13758
13938
|
url: SerpIntelligencePublicHttpUrlSchema,
|
|
13759
|
-
sourceKind:
|
|
13760
|
-
sourcePosition:
|
|
13939
|
+
sourceKind: z19.enum(SerpPageSnapshotSourceKindValues).default("configured_target"),
|
|
13940
|
+
sourcePosition: z19.number().int().min(1).optional()
|
|
13761
13941
|
}).strict();
|
|
13762
|
-
var SerpIntelligencePageSnapshotsBodySchema =
|
|
13763
|
-
urls:
|
|
13764
|
-
targets:
|
|
13765
|
-
maxConcurrency:
|
|
13766
|
-
timeoutMs:
|
|
13767
|
-
debug:
|
|
13942
|
+
var SerpIntelligencePageSnapshotsBodySchema = z19.object({
|
|
13943
|
+
urls: z19.array(SerpIntelligencePublicHttpUrlSchema).min(1).max(25),
|
|
13944
|
+
targets: z19.array(SerpIntelligencePageSnapshotRequestSchema).min(1).max(25).optional(),
|
|
13945
|
+
maxConcurrency: z19.number().int().min(1).max(5).default(2),
|
|
13946
|
+
timeoutMs: z19.number().int().min(1e3).max(6e4).default(15e3),
|
|
13947
|
+
debug: z19.boolean().default(false)
|
|
13768
13948
|
}).strict();
|
|
13769
|
-
var SerpIntelligenceAICitationSchema =
|
|
13770
|
-
text:
|
|
13771
|
-
href:
|
|
13949
|
+
var SerpIntelligenceAICitationSchema = z19.object({
|
|
13950
|
+
text: z19.string(),
|
|
13951
|
+
href: z19.string()
|
|
13772
13952
|
}).strict();
|
|
13773
|
-
var SerpIntelligenceOrganicResultSchema =
|
|
13774
|
-
position:
|
|
13775
|
-
title:
|
|
13776
|
-
url:
|
|
13777
|
-
domain:
|
|
13778
|
-
cite:
|
|
13779
|
-
snippet:
|
|
13780
|
-
isRedditStyle:
|
|
13781
|
-
inlineRating:
|
|
13782
|
-
value:
|
|
13783
|
-
count:
|
|
13953
|
+
var SerpIntelligenceOrganicResultSchema = z19.object({
|
|
13954
|
+
position: z19.number().int().min(1),
|
|
13955
|
+
title: z19.string(),
|
|
13956
|
+
url: z19.string(),
|
|
13957
|
+
domain: z19.string(),
|
|
13958
|
+
cite: z19.string().nullable(),
|
|
13959
|
+
snippet: z19.string().nullable(),
|
|
13960
|
+
isRedditStyle: z19.boolean(),
|
|
13961
|
+
inlineRating: z19.object({
|
|
13962
|
+
value: z19.string(),
|
|
13963
|
+
count: z19.string()
|
|
13784
13964
|
}).strict().nullable()
|
|
13785
13965
|
}).strict();
|
|
13786
|
-
var SerpIntelligenceLocationEvidenceSchema =
|
|
13787
|
-
status:
|
|
13788
|
-
expected:
|
|
13789
|
-
city:
|
|
13790
|
-
regionCode:
|
|
13791
|
-
canonicalLocation:
|
|
13966
|
+
var SerpIntelligenceLocationEvidenceSchema = z19.object({
|
|
13967
|
+
status: z19.enum(SerpIntelligenceLocalizationStatusValues),
|
|
13968
|
+
expected: z19.object({
|
|
13969
|
+
city: z19.string(),
|
|
13970
|
+
regionCode: z19.string().nullable(),
|
|
13971
|
+
canonicalLocation: z19.string()
|
|
13792
13972
|
}).strict().nullable(),
|
|
13793
|
-
candidates:
|
|
13794
|
-
city:
|
|
13795
|
-
regionCode:
|
|
13796
|
-
count:
|
|
13797
|
-
examples:
|
|
13973
|
+
candidates: z19.array(z19.object({
|
|
13974
|
+
city: z19.string(),
|
|
13975
|
+
regionCode: z19.string(),
|
|
13976
|
+
count: z19.number().int().min(0),
|
|
13977
|
+
examples: z19.array(z19.string())
|
|
13798
13978
|
}).strict())
|
|
13799
13979
|
}).strict();
|
|
13800
|
-
var SerpIntelligenceHarvestResultSchema =
|
|
13801
|
-
seed:
|
|
13802
|
-
location:
|
|
13803
|
-
extractedAt:
|
|
13804
|
-
totalQuestions:
|
|
13805
|
-
surface:
|
|
13806
|
-
aiOverview:
|
|
13807
|
-
detected:
|
|
13808
|
-
text:
|
|
13809
|
-
citations:
|
|
13810
|
-
expanded:
|
|
13811
|
-
fullyExpanded:
|
|
13812
|
-
sections:
|
|
13980
|
+
var SerpIntelligenceHarvestResultSchema = z19.object({
|
|
13981
|
+
seed: z19.string(),
|
|
13982
|
+
location: z19.string().nullable(),
|
|
13983
|
+
extractedAt: z19.string(),
|
|
13984
|
+
totalQuestions: z19.number().int().min(0),
|
|
13985
|
+
surface: z19.enum(["web", "aim", "unknown"]),
|
|
13986
|
+
aiOverview: z19.object({
|
|
13987
|
+
detected: z19.boolean(),
|
|
13988
|
+
text: z19.string().nullable(),
|
|
13989
|
+
citations: z19.array(SerpIntelligenceAICitationSchema),
|
|
13990
|
+
expanded: z19.boolean().optional(),
|
|
13991
|
+
fullyExpanded: z19.boolean().optional(),
|
|
13992
|
+
sections: z19.array(z19.string()).optional()
|
|
13813
13993
|
}).strict(),
|
|
13814
|
-
aiMode:
|
|
13815
|
-
detected:
|
|
13816
|
-
text:
|
|
13817
|
-
citations:
|
|
13994
|
+
aiMode: z19.object({
|
|
13995
|
+
detected: z19.boolean(),
|
|
13996
|
+
text: z19.string().nullable(),
|
|
13997
|
+
citations: z19.array(SerpIntelligenceAICitationSchema)
|
|
13818
13998
|
}).strict(),
|
|
13819
|
-
tree:
|
|
13820
|
-
flat:
|
|
13821
|
-
videos:
|
|
13822
|
-
forums:
|
|
13823
|
-
organicResults:
|
|
13824
|
-
localPack:
|
|
13825
|
-
entityIds:
|
|
13826
|
-
entities:
|
|
13827
|
-
name:
|
|
13828
|
-
kgId:
|
|
13829
|
-
cid:
|
|
13830
|
-
gcid:
|
|
13999
|
+
tree: z19.array(z19.unknown()),
|
|
14000
|
+
flat: z19.array(z19.unknown()),
|
|
14001
|
+
videos: z19.array(z19.unknown()),
|
|
14002
|
+
forums: z19.array(z19.unknown()),
|
|
14003
|
+
organicResults: z19.array(SerpIntelligenceOrganicResultSchema),
|
|
14004
|
+
localPack: z19.array(z19.unknown()),
|
|
14005
|
+
entityIds: z19.object({
|
|
14006
|
+
entities: z19.array(z19.object({
|
|
14007
|
+
name: z19.string(),
|
|
14008
|
+
kgId: z19.string().nullable(),
|
|
14009
|
+
cid: z19.string().nullable(),
|
|
14010
|
+
gcid: z19.string().nullable()
|
|
13831
14011
|
}).strict()),
|
|
13832
|
-
kgIds:
|
|
13833
|
-
cids:
|
|
13834
|
-
gcids:
|
|
14012
|
+
kgIds: z19.array(z19.string()),
|
|
14013
|
+
cids: z19.array(z19.string()),
|
|
14014
|
+
gcids: z19.array(z19.string())
|
|
13835
14015
|
}).strict(),
|
|
13836
|
-
stats:
|
|
13837
|
-
seed:
|
|
13838
|
-
totalQuestions:
|
|
13839
|
-
maxDepthReached:
|
|
13840
|
-
durationMs:
|
|
13841
|
-
errorCount:
|
|
14016
|
+
stats: z19.object({
|
|
14017
|
+
seed: z19.string(),
|
|
14018
|
+
totalQuestions: z19.number().int().min(0),
|
|
14019
|
+
maxDepthReached: z19.number().int().min(0),
|
|
14020
|
+
durationMs: z19.number().min(0),
|
|
14021
|
+
errorCount: z19.number().int().min(0)
|
|
13842
14022
|
}).strict(),
|
|
13843
|
-
diagnostics:
|
|
13844
|
-
completionStatus:
|
|
13845
|
-
problem:
|
|
13846
|
-
warnings:
|
|
13847
|
-
debug:
|
|
14023
|
+
diagnostics: z19.object({
|
|
14024
|
+
completionStatus: z19.enum(["paa_found", "no_paa", "serp_only"]),
|
|
14025
|
+
problem: z19.null(),
|
|
14026
|
+
warnings: z19.array(z19.unknown()).optional(),
|
|
14027
|
+
debug: z19.object({
|
|
13848
14028
|
locationEvidence: SerpIntelligenceLocationEvidenceSchema.optional()
|
|
13849
14029
|
}).passthrough().optional()
|
|
13850
14030
|
}).passthrough(),
|
|
13851
|
-
whatPeopleSaying:
|
|
14031
|
+
whatPeopleSaying: z19.array(z19.unknown())
|
|
13852
14032
|
}).strict();
|
|
13853
|
-
var SerpIntelligenceCaptureAttemptSchema =
|
|
13854
|
-
attemptNumber:
|
|
13855
|
-
outcome:
|
|
13856
|
-
startedAt:
|
|
13857
|
-
completedAt:
|
|
13858
|
-
durationMs:
|
|
13859
|
-
problemCode:
|
|
13860
|
-
message:
|
|
13861
|
-
kernelSessionId:
|
|
13862
|
-
cleanupSucceeded:
|
|
14033
|
+
var SerpIntelligenceCaptureAttemptSchema = z19.object({
|
|
14034
|
+
attemptNumber: z19.number().int().min(1),
|
|
14035
|
+
outcome: z19.enum(SerpIntelligenceAttemptOutcomeValues),
|
|
14036
|
+
startedAt: z19.string().optional(),
|
|
14037
|
+
completedAt: z19.string().optional(),
|
|
14038
|
+
durationMs: z19.number().min(0).optional(),
|
|
14039
|
+
problemCode: z19.string().optional(),
|
|
14040
|
+
message: z19.string().optional(),
|
|
14041
|
+
kernelSessionId: z19.string().nullable().optional(),
|
|
14042
|
+
cleanupSucceeded: z19.boolean().nullable().optional()
|
|
13863
14043
|
}).strict();
|
|
13864
|
-
var SerpPageSnapshotCaptureSchema =
|
|
14044
|
+
var SerpPageSnapshotCaptureSchema = z19.object({
|
|
13865
14045
|
url: SerpIntelligencePublicHttpUrlSchema,
|
|
13866
14046
|
requestedUrl: SerpIntelligencePublicHttpUrlSchema,
|
|
13867
14047
|
finalUrl: SerpIntelligencePublicHttpUrlSchema.nullable(),
|
|
13868
|
-
sourceKind:
|
|
13869
|
-
sourcePosition:
|
|
13870
|
-
status:
|
|
13871
|
-
fetchedVia:
|
|
13872
|
-
httpStatus:
|
|
13873
|
-
contentType:
|
|
13874
|
-
title:
|
|
14048
|
+
sourceKind: z19.enum(SerpPageSnapshotSourceKindValues),
|
|
14049
|
+
sourcePosition: z19.number().int().min(1).nullable(),
|
|
14050
|
+
status: z19.enum(SerpPageFetchStatusValues),
|
|
14051
|
+
fetchedVia: z19.enum(SerpPageFetchedViaValues).nullable(),
|
|
14052
|
+
httpStatus: z19.number().int().min(100).max(599).nullable(),
|
|
14053
|
+
contentType: z19.string().nullable(),
|
|
14054
|
+
title: z19.string().nullable(),
|
|
13875
14055
|
canonicalUrl: SerpIntelligencePublicHttpUrlSchema.nullable(),
|
|
13876
|
-
metaDescription:
|
|
13877
|
-
headings:
|
|
13878
|
-
level:
|
|
13879
|
-
text:
|
|
14056
|
+
metaDescription: z19.string().nullable(),
|
|
14057
|
+
headings: z19.array(z19.object({
|
|
14058
|
+
level: z19.number().int().min(1).max(6),
|
|
14059
|
+
text: z19.string()
|
|
13880
14060
|
}).strict()).default([]),
|
|
13881
|
-
artifact:
|
|
13882
|
-
htmlBlobUrl:
|
|
13883
|
-
textBlobUrl:
|
|
13884
|
-
markdownBlobUrl:
|
|
13885
|
-
screenshotBlobUrl:
|
|
13886
|
-
contentSha256:
|
|
13887
|
-
capturedAt:
|
|
14061
|
+
artifact: z19.object({
|
|
14062
|
+
htmlBlobUrl: z19.string().url().nullable(),
|
|
14063
|
+
textBlobUrl: z19.string().url().nullable(),
|
|
14064
|
+
markdownBlobUrl: z19.string().url().nullable(),
|
|
14065
|
+
screenshotBlobUrl: z19.string().url().nullable(),
|
|
14066
|
+
contentSha256: z19.string().nullable(),
|
|
14067
|
+
capturedAt: z19.string().nullable()
|
|
13888
14068
|
}).strict(),
|
|
13889
|
-
error:
|
|
13890
|
-
code:
|
|
13891
|
-
message:
|
|
14069
|
+
error: z19.object({
|
|
14070
|
+
code: z19.string(),
|
|
14071
|
+
message: z19.string()
|
|
13892
14072
|
}).strict().nullable()
|
|
13893
14073
|
}).strict();
|
|
13894
|
-
var SerpIntelligenceCaptureResponseSchema =
|
|
14074
|
+
var SerpIntelligenceCaptureResponseSchema = z19.object({
|
|
13895
14075
|
harvestResult: SerpIntelligenceHarvestResultSchema,
|
|
13896
|
-
attempts:
|
|
14076
|
+
attempts: z19.array(SerpIntelligenceCaptureAttemptSchema),
|
|
13897
14077
|
locationEvidence: SerpIntelligenceLocationEvidenceSchema.nullable(),
|
|
13898
|
-
pageSnapshotArtifacts:
|
|
13899
|
-
billing:
|
|
13900
|
-
creditsUsed:
|
|
13901
|
-
requestId:
|
|
13902
|
-
jobId:
|
|
14078
|
+
pageSnapshotArtifacts: z19.array(SerpPageSnapshotCaptureSchema),
|
|
14079
|
+
billing: z19.object({
|
|
14080
|
+
creditsUsed: z19.number().min(0).optional(),
|
|
14081
|
+
requestId: z19.string().optional(),
|
|
14082
|
+
jobId: z19.string().optional()
|
|
13903
14083
|
}).strict().optional()
|
|
13904
14084
|
}).strict();
|
|
13905
|
-
var SerpIntelligencePageSnapshotsResponseSchema =
|
|
13906
|
-
pageSnapshotArtifacts:
|
|
13907
|
-
attempts:
|
|
14085
|
+
var SerpIntelligencePageSnapshotsResponseSchema = z19.object({
|
|
14086
|
+
pageSnapshotArtifacts: z19.array(SerpPageSnapshotCaptureSchema),
|
|
14087
|
+
attempts: z19.array(SerpIntelligenceCaptureAttemptSchema).default([])
|
|
13908
14088
|
}).strict();
|
|
13909
14089
|
|
|
13910
14090
|
// src/serp-intelligence/serp-capture-service.ts
|
|
@@ -14076,7 +14256,7 @@ var SERP_INTELLIGENCE_RATE_LIMIT = 60;
|
|
|
14076
14256
|
var SERP_INTELLIGENCE_RATE_WINDOW_SECONDS = 60;
|
|
14077
14257
|
var POST_CAPTURE_ROUTE_LABEL = "POST /capture";
|
|
14078
14258
|
var POST_PAGE_SNAPSHOTS_ROUTE_LABEL = "POST /page-snapshots";
|
|
14079
|
-
var serpIntelligenceApp = new
|
|
14259
|
+
var serpIntelligenceApp = new Hono10();
|
|
14080
14260
|
serpIntelligenceApp.use("*", createApiKeyAuth());
|
|
14081
14261
|
function structuredError(input) {
|
|
14082
14262
|
return {
|
|
@@ -14245,41 +14425,95 @@ serpIntelligenceApp.post("/page-snapshots", async (c) => {
|
|
|
14245
14425
|
});
|
|
14246
14426
|
|
|
14247
14427
|
// src/mcp/mcp-routes.ts
|
|
14248
|
-
import { Hono as
|
|
14428
|
+
import { Hono as Hono11 } from "hono";
|
|
14249
14429
|
import { WebStandardStreamableHTTPServerTransport } from "@modelcontextprotocol/sdk/server/webStandardStreamableHttp.js";
|
|
14430
|
+
|
|
14431
|
+
// src/mcp/mcp-oauth.ts
|
|
14432
|
+
import { createRemoteJWKSet, jwtVerify } from "jose";
|
|
14433
|
+
function oauthIssuer() {
|
|
14434
|
+
return (process.env.OAUTH_ISSUER ?? "https://mcpscraper.dev").replace(/\/$/, "");
|
|
14435
|
+
}
|
|
14436
|
+
function oauthResourceUrl() {
|
|
14437
|
+
return (process.env.MCP_SCRAPER_RESOURCE_URL ?? "https://mcpscraper.dev/mcp").replace(/\/$/, "");
|
|
14438
|
+
}
|
|
14439
|
+
function oauthResourceBaseUrl() {
|
|
14440
|
+
return (process.env.OAUTH_RS_BASE_URL ?? "https://mcpscraper.dev").replace(/\/$/, "");
|
|
14441
|
+
}
|
|
14442
|
+
function protectedResourceMetadata() {
|
|
14443
|
+
return {
|
|
14444
|
+
resource: oauthResourceUrl(),
|
|
14445
|
+
authorization_servers: [oauthIssuer()],
|
|
14446
|
+
scopes_supported: ["scraper.use"],
|
|
14447
|
+
bearer_methods_supported: ["header"]
|
|
14448
|
+
};
|
|
14449
|
+
}
|
|
14450
|
+
function wwwAuthenticateValue() {
|
|
14451
|
+
return `Bearer resource_metadata="${oauthResourceBaseUrl()}/.well-known/oauth-protected-resource"`;
|
|
14452
|
+
}
|
|
14453
|
+
function isBearerJwt(token) {
|
|
14454
|
+
return !token.startsWith("sk_") && token.split(".").length === 3;
|
|
14455
|
+
}
|
|
14456
|
+
var jwksSingleton = null;
|
|
14457
|
+
function getJwks() {
|
|
14458
|
+
if (jwksSingleton) return jwksSingleton;
|
|
14459
|
+
jwksSingleton = createRemoteJWKSet(new URL(`${oauthIssuer()}/.well-known/jwks.json`));
|
|
14460
|
+
return jwksSingleton;
|
|
14461
|
+
}
|
|
14462
|
+
async function verifyScraperOAuthJwt(token) {
|
|
14463
|
+
const { payload } = await jwtVerify(token, getJwks(), {
|
|
14464
|
+
issuer: oauthIssuer(),
|
|
14465
|
+
audience: oauthResourceUrl()
|
|
14466
|
+
});
|
|
14467
|
+
return payload;
|
|
14468
|
+
}
|
|
14469
|
+
|
|
14470
|
+
// src/mcp/mcp-routes.ts
|
|
14250
14471
|
configureReportSaving(false);
|
|
14251
|
-
function mcpAuthError() {
|
|
14472
|
+
function mcpAuthError(message = "authentication required") {
|
|
14252
14473
|
const body = JSON.stringify({
|
|
14253
14474
|
jsonrpc: "2.0",
|
|
14254
14475
|
id: null,
|
|
14255
14476
|
error: {
|
|
14256
14477
|
code: -32001,
|
|
14257
|
-
message
|
|
14478
|
+
message
|
|
14258
14479
|
}
|
|
14259
14480
|
});
|
|
14260
14481
|
return new Response(body, {
|
|
14261
14482
|
status: 401,
|
|
14262
14483
|
headers: {
|
|
14263
14484
|
"Content-Type": "application/json",
|
|
14264
|
-
"WWW-Authenticate":
|
|
14485
|
+
"WWW-Authenticate": wwwAuthenticateValue()
|
|
14265
14486
|
}
|
|
14266
14487
|
});
|
|
14267
14488
|
}
|
|
14268
14489
|
async function requireMcpCallerKey(c) {
|
|
14269
14490
|
const xApiKey = c.req.header("x-api-key")?.trim();
|
|
14270
14491
|
const authHeader = c.req.header("Authorization")?.trim();
|
|
14271
|
-
const
|
|
14272
|
-
|
|
14492
|
+
const bearerToken = authHeader?.startsWith("Bearer ") ? authHeader.slice(7).trim() : void 0;
|
|
14493
|
+
if (!xApiKey && bearerToken && isBearerJwt(bearerToken)) {
|
|
14494
|
+
let identity;
|
|
14495
|
+
try {
|
|
14496
|
+
const payload = await verifyScraperOAuthJwt(bearerToken);
|
|
14497
|
+
identity = typeof payload.sub === "string" ? payload.sub : "";
|
|
14498
|
+
} catch {
|
|
14499
|
+
return mcpAuthError("Bearer token is invalid or expired");
|
|
14500
|
+
}
|
|
14501
|
+
if (!identity) return mcpAuthError("Bearer token has no subject");
|
|
14502
|
+
const user2 = await getUserByEmail(identity);
|
|
14503
|
+
if (!user2 || !user2.api_key || user2.key_active !== 1) return mcpAuthError("No active MCP Scraper account for this identity");
|
|
14504
|
+
return user2.api_key;
|
|
14505
|
+
}
|
|
14506
|
+
const callerKey = xApiKey ?? bearerToken;
|
|
14273
14507
|
if (!callerKey) return mcpAuthError();
|
|
14274
14508
|
const user = await getUserByApiKey(callerKey);
|
|
14275
14509
|
if (!user) return mcpAuthError();
|
|
14276
14510
|
return callerKey;
|
|
14277
14511
|
}
|
|
14278
|
-
var mcpApp = new
|
|
14512
|
+
var mcpApp = new Hono11();
|
|
14279
14513
|
function registerSerpIntelligenceCaptureTools(server, executor) {
|
|
14280
14514
|
server.registerTool("capture_serp_snapshot", {
|
|
14281
14515
|
title: "SERP Intelligence Snapshot",
|
|
14282
|
-
description: "Capture a structured SERP Intelligence Google snapshot through POST /serp-intelligence/capture, the same product capture path used by Phoenix. Split query from location and infer gl/hl.
|
|
14516
|
+
description: "Capture a structured SERP Intelligence Google snapshot through POST /serp-intelligence/capture, the same product capture path used by Phoenix. Split query from location and infer gl/hl. Routing is automatic \u2014 leave proxyMode unset. Set debug true when investigating location evidence, proxy behavior, CAPTCHA, or capture reliability.",
|
|
14283
14517
|
inputSchema: CaptureSerpSnapshotInputSchema,
|
|
14284
14518
|
outputSchema: CaptureSerpSnapshotOutputSchema,
|
|
14285
14519
|
annotations: liveWebToolAnnotations("SERP Intelligence Snapshot")
|
|
@@ -14313,7 +14547,7 @@ mcpApp.all("/", async (c) => {
|
|
|
14313
14547
|
});
|
|
14314
14548
|
|
|
14315
14549
|
// src/api/browser-agent-routes.ts
|
|
14316
|
-
import { Hono as
|
|
14550
|
+
import { Hono as Hono12 } from "hono";
|
|
14317
14551
|
|
|
14318
14552
|
// src/api/browser-agent-db.ts
|
|
14319
14553
|
import { randomUUID as randomUUID2 } from "crypto";
|
|
@@ -15747,7 +15981,7 @@ async function loadOpenSession(id, userId) {
|
|
|
15747
15981
|
return row;
|
|
15748
15982
|
}
|
|
15749
15983
|
function buildBrowserAgentRoutes() {
|
|
15750
|
-
const app2 = new
|
|
15984
|
+
const app2 = new Hono12();
|
|
15751
15985
|
app2.use("*", async (c, next) => {
|
|
15752
15986
|
await migrateBrowserAgent();
|
|
15753
15987
|
return next();
|
|
@@ -16453,9 +16687,149 @@ if (state.key) { refreshSessions(); if (state.current) selectSession(state.curre
|
|
|
16453
16687
|
|
|
16454
16688
|
// src/api/stripe-routes.ts
|
|
16455
16689
|
import Stripe from "stripe";
|
|
16456
|
-
import { Hono as
|
|
16690
|
+
import { Hono as Hono13 } from "hono";
|
|
16691
|
+
|
|
16692
|
+
// src/api/memory.ts
|
|
16693
|
+
import { createCipheriv, createDecipheriv, randomBytes, scryptSync } from "crypto";
|
|
16694
|
+
|
|
16695
|
+
// src/api/session.ts
|
|
16696
|
+
import { createHmac as createHmac2, timingSafeEqual } from "crypto";
|
|
16697
|
+
var isProduction = () => process.env.NODE_ENV === "production" || process.env.VERCEL === "1";
|
|
16698
|
+
function getSessionSecret() {
|
|
16699
|
+
const configured = process.env.SESSION_SECRET?.trim();
|
|
16700
|
+
if (configured) return configured;
|
|
16701
|
+
if (isProduction()) throw new Error("SESSION_SECRET is not set \u2014 add it to your Vercel env vars (see src/api/env.ts)");
|
|
16702
|
+
return "dev-secret-change-me";
|
|
16703
|
+
}
|
|
16704
|
+
var secret = () => getSessionSecret();
|
|
16705
|
+
function safeEqualHex(a, b) {
|
|
16706
|
+
if (a.length !== b.length) return false;
|
|
16707
|
+
try {
|
|
16708
|
+
return timingSafeEqual(Buffer.from(a, "hex"), Buffer.from(b, "hex"));
|
|
16709
|
+
} catch {
|
|
16710
|
+
return false;
|
|
16711
|
+
}
|
|
16712
|
+
}
|
|
16713
|
+
function signSession(userId) {
|
|
16714
|
+
const payload = String(userId);
|
|
16715
|
+
const sig = createHmac2("sha256", secret()).update(payload).digest("hex");
|
|
16716
|
+
return `${payload}.${sig}`;
|
|
16717
|
+
}
|
|
16718
|
+
function verifySession(token) {
|
|
16719
|
+
const dot = token.lastIndexOf(".");
|
|
16720
|
+
if (dot === -1) return null;
|
|
16721
|
+
const payload = token.slice(0, dot);
|
|
16722
|
+
const sig = token.slice(dot + 1);
|
|
16723
|
+
const expected = createHmac2("sha256", secret()).update(payload).digest("hex");
|
|
16724
|
+
if (!safeEqualHex(sig, expected)) return null;
|
|
16725
|
+
const id = parseInt(payload);
|
|
16726
|
+
return isNaN(id) ? null : id;
|
|
16727
|
+
}
|
|
16728
|
+
|
|
16729
|
+
// src/api/memory.ts
|
|
16730
|
+
var MEMORY_BASE_URL = () => (process.env.MCP_MEMORY_BASE_URL ?? "https://mcp-memory-omega.vercel.app").replace(/\/$/, "");
|
|
16731
|
+
var ADMIN_KEY = () => process.env.MCP_MEMORY_ADMIN_KEY?.trim() ?? "";
|
|
16732
|
+
function isMemoryOperator(email) {
|
|
16733
|
+
if (!email) return false;
|
|
16734
|
+
const ops = (process.env.MEMORY_OPERATOR_IDENTITIES ?? "").split(",").map((s) => s.trim().toLowerCase()).filter(Boolean);
|
|
16735
|
+
return ops.includes(email.trim().toLowerCase());
|
|
16736
|
+
}
|
|
16737
|
+
function encKey() {
|
|
16738
|
+
return scryptSync(getSessionSecret(), "mcp-memory-key-v1", 32);
|
|
16739
|
+
}
|
|
16740
|
+
function encryptMemoryKey(secret2) {
|
|
16741
|
+
const iv = randomBytes(12);
|
|
16742
|
+
const cipher = createCipheriv("aes-256-gcm", encKey(), iv);
|
|
16743
|
+
const enc = Buffer.concat([cipher.update(secret2, "utf8"), cipher.final()]);
|
|
16744
|
+
const tag = cipher.getAuthTag();
|
|
16745
|
+
return `${iv.toString("base64")}:${tag.toString("base64")}:${enc.toString("base64")}`;
|
|
16746
|
+
}
|
|
16747
|
+
function decryptMemoryKey(stored) {
|
|
16748
|
+
try {
|
|
16749
|
+
const [ivB, tagB, dataB] = stored.split(":");
|
|
16750
|
+
const decipher = createDecipheriv("aes-256-gcm", encKey(), Buffer.from(ivB, "base64"));
|
|
16751
|
+
decipher.setAuthTag(Buffer.from(tagB, "base64"));
|
|
16752
|
+
return Buffer.concat([decipher.update(Buffer.from(dataB, "base64")), decipher.final()]).toString("utf8");
|
|
16753
|
+
} catch {
|
|
16754
|
+
return null;
|
|
16755
|
+
}
|
|
16756
|
+
}
|
|
16757
|
+
async function memoryCall(toolName, args, userMemoryKey) {
|
|
16758
|
+
try {
|
|
16759
|
+
const res = await fetch(`${MEMORY_BASE_URL()}/api/mcp/thorbitMemoryServer/tools/${toolName}/execute`, {
|
|
16760
|
+
method: "POST",
|
|
16761
|
+
headers: { "content-type": "application/json" },
|
|
16762
|
+
body: JSON.stringify({ data: { ...args, apiKey: userMemoryKey } })
|
|
16763
|
+
});
|
|
16764
|
+
const body = await res.json().catch(() => null);
|
|
16765
|
+
if (!res.ok) return { ok: false, error: body?.error ?? `memory ${toolName} failed (${res.status})` };
|
|
16766
|
+
if (body?.result) return body.result;
|
|
16767
|
+
return { ok: false, error: body?.error ?? `memory ${toolName} returned no result` };
|
|
16768
|
+
} catch (err) {
|
|
16769
|
+
return { ok: false, error: err?.message ?? "memory call failed" };
|
|
16770
|
+
}
|
|
16771
|
+
}
|
|
16772
|
+
function personalVault(user) {
|
|
16773
|
+
return `mcp-${user.id}`;
|
|
16774
|
+
}
|
|
16775
|
+
function memoryIdentity(user) {
|
|
16776
|
+
return user.email;
|
|
16777
|
+
}
|
|
16778
|
+
function resolveMemoryIdentity(user) {
|
|
16779
|
+
return memoryIdentity(user);
|
|
16780
|
+
}
|
|
16781
|
+
function memoryPlanForReads(user) {
|
|
16782
|
+
if (isMemoryOperator(user.email)) return "unlimited";
|
|
16783
|
+
if (user.memory_plan === "pro" || user.memory_plan === "team") return user.memory_plan;
|
|
16784
|
+
return "free";
|
|
16785
|
+
}
|
|
16786
|
+
async function provisionMemoryForUser(user) {
|
|
16787
|
+
try {
|
|
16788
|
+
const { key } = await getOrCreateUserMemoryKey(user);
|
|
16789
|
+
if (key) await setMemoryProvisioned(user.id, true);
|
|
16790
|
+
} catch {
|
|
16791
|
+
}
|
|
16792
|
+
}
|
|
16793
|
+
async function getOrCreateUserMemoryKey(user) {
|
|
16794
|
+
const creds = await getUserMemoryCreds(user.id);
|
|
16795
|
+
if (creds.memory_key) {
|
|
16796
|
+
const decrypted = decryptMemoryKey(creds.memory_key);
|
|
16797
|
+
if (decrypted) return { key: decrypted };
|
|
16798
|
+
}
|
|
16799
|
+
const admin = ADMIN_KEY();
|
|
16800
|
+
if (!admin) return { key: null, error: "memory not configured (MCP_MEMORY_ADMIN_KEY missing)" };
|
|
16801
|
+
const identity = memoryIdentity(user);
|
|
16802
|
+
const plan = isMemoryOperator(user.email) ? "unlimited" : (await getMemoryPlan(user.id)).plan;
|
|
16803
|
+
const provisioned = await memoryCall(
|
|
16804
|
+
"provisionDefaultsTool",
|
|
16805
|
+
{ granteeIdentity: identity, issueKey: true, plan },
|
|
16806
|
+
admin
|
|
16807
|
+
);
|
|
16808
|
+
if (!provisioned.ok || !provisioned.secret) {
|
|
16809
|
+
return { key: null, error: provisioned.error ?? "failed to provision default vaults" };
|
|
16810
|
+
}
|
|
16811
|
+
await setUserMemoryCreds(user.id, encryptMemoryKey(provisioned.secret), identity);
|
|
16812
|
+
return { key: provisioned.secret };
|
|
16813
|
+
}
|
|
16814
|
+
async function syncMemoryKeyPlan(user, plan) {
|
|
16815
|
+
const effectivePlan = isMemoryOperator(user.email) ? "unlimited" : plan;
|
|
16816
|
+
const { key, error } = await getOrCreateUserMemoryKey(user);
|
|
16817
|
+
if (!key) return { ok: false, error: error ?? "memory unavailable" };
|
|
16818
|
+
const listed = await memoryCall(
|
|
16819
|
+
"listKeysTool",
|
|
16820
|
+
{},
|
|
16821
|
+
key
|
|
16822
|
+
);
|
|
16823
|
+
if (!listed.ok || !listed.keys?.length) return { ok: false, error: listed.error ?? "no memory keys found" };
|
|
16824
|
+
const target = listed.keys[0];
|
|
16825
|
+
if (target.plan === effectivePlan) return { ok: true };
|
|
16826
|
+
const res = await memoryCall("setScopeTool", { keyId: target.keyId, plan: effectivePlan }, key);
|
|
16827
|
+
return { ok: res.ok, error: res.error };
|
|
16828
|
+
}
|
|
16829
|
+
|
|
16830
|
+
// src/api/stripe-routes.ts
|
|
16457
16831
|
var stripe = new Stripe(process.env.STRIPE_SECRET_KEY, { apiVersion: "2026-02-25.clover" });
|
|
16458
|
-
var stripeApp = new
|
|
16832
|
+
var stripeApp = new Hono13();
|
|
16459
16833
|
stripeApp.post("/webhooks", async (c) => {
|
|
16460
16834
|
const sig = c.req.header("stripe-signature");
|
|
16461
16835
|
const body = await c.req.text();
|
|
@@ -16469,6 +16843,8 @@ stripeApp.post("/webhooks", async (c) => {
|
|
|
16469
16843
|
if (!isNew) return c.json({ received: true });
|
|
16470
16844
|
if (event.type === "invoice.paid" || event.type === "invoice.payment_succeeded") {
|
|
16471
16845
|
const invoice = event.data.object;
|
|
16846
|
+
const memLineId = invoice.lines.data.map(linePriceId).find((id) => id && id in MEMORY_PLANS);
|
|
16847
|
+
if (memLineId) return c.json({ received: true });
|
|
16472
16848
|
const lineTierId = invoice.lines.data.map(linePriceId).find((id) => id && id in SUBSCRIPTION_TIERS);
|
|
16473
16849
|
if (lineTierId) {
|
|
16474
16850
|
const tier = SUBSCRIPTION_TIERS[lineTierId];
|
|
@@ -16482,6 +16858,16 @@ stripeApp.post("/webhooks", async (c) => {
|
|
|
16482
16858
|
if (event.type === "customer.subscription.created" || event.type === "customer.subscription.updated") {
|
|
16483
16859
|
const sub = event.data.object;
|
|
16484
16860
|
const priceId = sub.items.data[0]?.price?.id;
|
|
16861
|
+
if (priceId && priceId in MEMORY_PLANS) {
|
|
16862
|
+
const mem = MEMORY_PLANS[priceId];
|
|
16863
|
+
const user2 = await resolveUser(sub.customer, sub.customer_email);
|
|
16864
|
+
if (user2) {
|
|
16865
|
+
const live = sub.status === "active" || sub.status === "trialing";
|
|
16866
|
+
await setMemoryPlan(user2.id, live ? mem.plan : "free", live ? sub.id : null);
|
|
16867
|
+
await syncMemoryKeyPlan(user2, live ? mem.plan : "free");
|
|
16868
|
+
}
|
|
16869
|
+
return c.json({ received: true });
|
|
16870
|
+
}
|
|
16485
16871
|
const user = await getUserByStripeCustomerId(sub.customer);
|
|
16486
16872
|
if (!user) return c.json({ received: true });
|
|
16487
16873
|
if (priceId && priceId in SUBSCRIPTION_TIERS) {
|
|
@@ -16496,6 +16882,14 @@ stripeApp.post("/webhooks", async (c) => {
|
|
|
16496
16882
|
if (event.type === "customer.subscription.deleted") {
|
|
16497
16883
|
const sub = event.data.object;
|
|
16498
16884
|
const priceId = sub.items.data[0]?.price?.id;
|
|
16885
|
+
if (priceId && priceId in MEMORY_PLANS || await resolveUserByMemorySubscription(sub.id)) {
|
|
16886
|
+
const user2 = await resolveUserByMemorySubscription(sub.id) ?? await getUserByStripeCustomerId(sub.customer);
|
|
16887
|
+
if (user2) {
|
|
16888
|
+
await setMemoryPlan(user2.id, "free", null);
|
|
16889
|
+
await syncMemoryKeyPlan(user2, "free");
|
|
16890
|
+
}
|
|
16891
|
+
return c.json({ received: true });
|
|
16892
|
+
}
|
|
16499
16893
|
const user = await getUserByStripeCustomerId(sub.customer);
|
|
16500
16894
|
if (!user) return c.json({ received: true });
|
|
16501
16895
|
if (priceId && priceId in SUBSCRIPTION_TIERS) {
|
|
@@ -16520,6 +16914,417 @@ async function resolveUser(customerId, emailFallback) {
|
|
|
16520
16914
|
return user;
|
|
16521
16915
|
}
|
|
16522
16916
|
|
|
16917
|
+
// src/api/oauth-routes.ts
|
|
16918
|
+
import { Hono as Hono14 } from "hono";
|
|
16919
|
+
import { getCookie, setCookie } from "hono/cookie";
|
|
16920
|
+
import { createHash as createHash3, randomBytes as randomBytes2, randomUUID as randomUUID3 } from "crypto";
|
|
16921
|
+
import { importPKCS8, exportJWK, SignJWT } from "jose";
|
|
16922
|
+
var ISSUER = "https://mcpscraper.dev";
|
|
16923
|
+
var RESOURCE = () => process.env.MCP_MEMORY_RESOURCE_URL ?? "https://mcp-memory-omega.vercel.app/mcp/thorbitMemoryServer/mcp";
|
|
16924
|
+
var SCRAPER_RESOURCE = () => (process.env.MCP_SCRAPER_RESOURCE_URL ?? "https://mcpscraper.dev/mcp").replace(/\/$/, "");
|
|
16925
|
+
var MEMORY_SCOPES = ["memory.read", "memory.write", "memory.admin"];
|
|
16926
|
+
var SCRAPER_SCOPES = ["scraper.use"];
|
|
16927
|
+
var SUPPORTED_SCOPES = [...MEMORY_SCOPES, ...SCRAPER_SCOPES];
|
|
16928
|
+
function isScraperResource(resource) {
|
|
16929
|
+
if (!resource) return false;
|
|
16930
|
+
return resource.replace(/\/$/, "") === SCRAPER_RESOURCE();
|
|
16931
|
+
}
|
|
16932
|
+
var ACCESS_TTL_SECONDS = 3600;
|
|
16933
|
+
var REFRESH_TTL_SECONDS = 60 * 60 * 24 * 30;
|
|
16934
|
+
var CODE_TTL_SECONDS = 60;
|
|
16935
|
+
var secureCookies = process.env.NODE_ENV === "production" || process.env.VERCEL === "1";
|
|
16936
|
+
var sessionCookieOptions = {
|
|
16937
|
+
httpOnly: true,
|
|
16938
|
+
secure: secureCookies,
|
|
16939
|
+
path: "/",
|
|
16940
|
+
maxAge: 60 * 60 * 24 * 30,
|
|
16941
|
+
sameSite: "Lax"
|
|
16942
|
+
};
|
|
16943
|
+
var cachedKeys = null;
|
|
16944
|
+
async function getKeys() {
|
|
16945
|
+
if (cachedKeys) return cachedKeys;
|
|
16946
|
+
const raw = process.env.OAUTH_SIGNING_KEY;
|
|
16947
|
+
if (!raw?.trim()) throw new Error("OAUTH_SIGNING_KEY is not set");
|
|
16948
|
+
const pem = raw.includes("BEGIN") ? raw : Buffer.from(raw, "base64").toString("utf8");
|
|
16949
|
+
const privateKey = await importPKCS8(pem, "RS256", { extractable: true });
|
|
16950
|
+
const full = await exportJWK(privateKey);
|
|
16951
|
+
const publicJwk = { kty: full.kty, n: full.n, e: full.e };
|
|
16952
|
+
const kid = createHash3("sha256").update(JSON.stringify({ e: publicJwk.e, kty: publicJwk.kty, n: publicJwk.n })).digest("base64url").slice(0, 16);
|
|
16953
|
+
publicJwk.kid = kid;
|
|
16954
|
+
publicJwk.alg = "RS256";
|
|
16955
|
+
publicJwk.use = "sig";
|
|
16956
|
+
cachedKeys = { privateKey, publicJwk, kid };
|
|
16957
|
+
return cachedKeys;
|
|
16958
|
+
}
|
|
16959
|
+
function isAllowedRedirect(uri) {
|
|
16960
|
+
let u;
|
|
16961
|
+
try {
|
|
16962
|
+
u = new URL(uri);
|
|
16963
|
+
} catch {
|
|
16964
|
+
return false;
|
|
16965
|
+
}
|
|
16966
|
+
if (u.protocol !== "https:") return false;
|
|
16967
|
+
const h = u.hostname.toLowerCase();
|
|
16968
|
+
if (["claude.ai", "www.claude.ai", "claude.com", "www.claude.com"].includes(h)) return true;
|
|
16969
|
+
if (h.endsWith(".claude.ai") || h.endsWith(".claude.com")) return true;
|
|
16970
|
+
const extra = (process.env.OAUTH_EXTRA_REDIRECT_HOSTS ?? "").split(",").map((s) => s.trim().toLowerCase()).filter(Boolean);
|
|
16971
|
+
return extra.includes(h);
|
|
16972
|
+
}
|
|
16973
|
+
function resolvePlan(user) {
|
|
16974
|
+
if (isMemoryOperator(user.email)) return "unlimited";
|
|
16975
|
+
if (user.memory_plan === "pro" || user.memory_plan === "team") return user.memory_plan;
|
|
16976
|
+
return "free";
|
|
16977
|
+
}
|
|
16978
|
+
function negotiateScope(requested, user, resource) {
|
|
16979
|
+
if (isScraperResource(resource)) return "scraper.use";
|
|
16980
|
+
const operator = isMemoryOperator(user.email);
|
|
16981
|
+
const wanted = (requested ?? "").split(/\s+/).map((s) => s.trim()).filter(Boolean);
|
|
16982
|
+
const filtered = wanted.filter((s) => MEMORY_SCOPES.includes(s));
|
|
16983
|
+
const base = filtered.length > 0 ? filtered : ["memory.read", "memory.write"];
|
|
16984
|
+
const allowed = base.filter((s) => s === "memory.admin" ? operator : true);
|
|
16985
|
+
const unique = [...new Set(allowed)];
|
|
16986
|
+
if (operator && !unique.includes("memory.admin")) unique.push("memory.admin");
|
|
16987
|
+
return unique.join(" ");
|
|
16988
|
+
}
|
|
16989
|
+
async function sessionUser(token) {
|
|
16990
|
+
if (!token) return null;
|
|
16991
|
+
const userId = verifySession(token);
|
|
16992
|
+
if (!userId) return null;
|
|
16993
|
+
const user = await getUserById(userId);
|
|
16994
|
+
return user ?? null;
|
|
16995
|
+
}
|
|
16996
|
+
function escapeHtml(value) {
|
|
16997
|
+
return value.replace(/&/g, "&").replace(/</g, "<").replace(/>/g, ">").replace(/"/g, """).replace(/'/g, "'");
|
|
16998
|
+
}
|
|
16999
|
+
function hiddenFields(p) {
|
|
17000
|
+
const fields = [
|
|
17001
|
+
["response_type", p.response_type],
|
|
17002
|
+
["client_id", p.client_id],
|
|
17003
|
+
["redirect_uri", p.redirect_uri],
|
|
17004
|
+
["code_challenge", p.code_challenge],
|
|
17005
|
+
["code_challenge_method", p.code_challenge_method],
|
|
17006
|
+
["scope", p.scope],
|
|
17007
|
+
["state", p.state],
|
|
17008
|
+
["resource", p.resource]
|
|
17009
|
+
];
|
|
17010
|
+
return fields.map(([k, v]) => `<input type="hidden" name="${k}" value="${escapeHtml(v)}" />`).join("");
|
|
17011
|
+
}
|
|
17012
|
+
function scopeLabel(scope) {
|
|
17013
|
+
const map = {
|
|
17014
|
+
"memory.read": "Read your vaults, notes and search",
|
|
17015
|
+
"memory.write": "Create and edit notes in your vaults",
|
|
17016
|
+
"memory.admin": "Full administrative access",
|
|
17017
|
+
"scraper.use": "Run web scraping, search, maps and social tools (billed to your account)"
|
|
17018
|
+
};
|
|
17019
|
+
return scope.split(/\s+/).filter(Boolean).map((s) => map[s] ?? s).map((l) => `<li>${escapeHtml(l)}</li>`).join("");
|
|
17020
|
+
}
|
|
17021
|
+
function resourceLabel(resource) {
|
|
17022
|
+
return isScraperResource(resource) ? "MCP Scraper" : "mcp-memory";
|
|
17023
|
+
}
|
|
17024
|
+
function pageShell(inner) {
|
|
17025
|
+
return `<!doctype html><html lang="en"><head><meta charset="utf-8" /><meta name="viewport" content="width=device-width,initial-scale=1" /><title>Authorize Claude \xB7 mcp-memory</title><style>
|
|
17026
|
+
:root{color-scheme:dark}
|
|
17027
|
+
*{box-sizing:border-box}
|
|
17028
|
+
body{margin:0;min-height:100vh;display:flex;align-items:center;justify-content:center;background:#0b0d10;color:#e8eaed;font-family:-apple-system,BlinkMacSystemFont,"Segoe UI",Inter,system-ui,sans-serif;padding:24px}
|
|
17029
|
+
.card{width:100%;max-width:420px;background:#14171c;border:1px solid #232830;border-radius:16px;padding:32px}
|
|
17030
|
+
h1{font-size:20px;margin:0 0 4px}
|
|
17031
|
+
p.sub{color:#9aa3ad;font-size:14px;margin:0 0 24px}
|
|
17032
|
+
label{display:block;font-size:13px;color:#9aa3ad;margin:14px 0 6px}
|
|
17033
|
+
input[type=email],input[type=password]{width:100%;padding:11px 13px;background:#0b0d10;border:1px solid #2b313a;border-radius:9px;color:#e8eaed;font-size:14px}
|
|
17034
|
+
input:focus{outline:none;border-color:#5b8cff}
|
|
17035
|
+
button{width:100%;margin-top:20px;padding:12px;border:0;border-radius:9px;font-size:14px;font-weight:600;cursor:pointer}
|
|
17036
|
+
.primary{background:#5b8cff;color:#fff}
|
|
17037
|
+
.ghost{background:transparent;color:#9aa3ad;border:1px solid #2b313a;margin-top:10px}
|
|
17038
|
+
ul{margin:14px 0;padding-left:18px;color:#cbd2da;font-size:13px;line-height:1.7}
|
|
17039
|
+
.err{background:#3a1b1f;border:1px solid #6b2b30;color:#ffb4b4;padding:10px 12px;border-radius:9px;font-size:13px;margin-bottom:16px}
|
|
17040
|
+
.who{font-size:13px;color:#9aa3ad;margin-bottom:8px}
|
|
17041
|
+
</style></head><body><div class="card">${inner}</div></body></html>`;
|
|
17042
|
+
}
|
|
17043
|
+
function renderLogin(p, error) {
|
|
17044
|
+
return pageShell(`
|
|
17045
|
+
<h1>Sign in to ${escapeHtml(resourceLabel(p.resource))}</h1>
|
|
17046
|
+
<p class="sub">Authorize Claude to connect to your ${escapeHtml(resourceLabel(p.resource))} on mcpscraper.dev.</p>
|
|
17047
|
+
${error ? `<div class="err">${escapeHtml(error)}</div>` : ""}
|
|
17048
|
+
<form method="post" action="/oauth/authorize">
|
|
17049
|
+
${hiddenFields(p)}
|
|
17050
|
+
<input type="hidden" name="action" value="login" />
|
|
17051
|
+
<label for="email">Email</label>
|
|
17052
|
+
<input id="email" name="email" type="email" autocomplete="username" required />
|
|
17053
|
+
<label for="password">Password</label>
|
|
17054
|
+
<input id="password" name="password" type="password" autocomplete="current-password" required />
|
|
17055
|
+
<button class="primary" type="submit">Sign in</button>
|
|
17056
|
+
</form>`);
|
|
17057
|
+
}
|
|
17058
|
+
function renderConsent(p, email, error) {
|
|
17059
|
+
return pageShell(`
|
|
17060
|
+
<h1>Connect Claude to ${escapeHtml(resourceLabel(p.resource))}</h1>
|
|
17061
|
+
<p class="sub">Allow Claude to access your ${escapeHtml(resourceLabel(p.resource))}?</p>
|
|
17062
|
+
${error ? `<div class="err">${escapeHtml(error)}</div>` : ""}
|
|
17063
|
+
<div class="who">Signed in as <strong>${escapeHtml(email)}</strong></div>
|
|
17064
|
+
<ul>${scopeLabel(p.scope)}</ul>
|
|
17065
|
+
<form method="post" action="/oauth/authorize">
|
|
17066
|
+
${hiddenFields(p)}
|
|
17067
|
+
<input type="hidden" name="action" value="approve" />
|
|
17068
|
+
<button class="primary" type="submit">Allow</button>
|
|
17069
|
+
</form>
|
|
17070
|
+
<form method="post" action="/oauth/authorize">
|
|
17071
|
+
${hiddenFields(p)}
|
|
17072
|
+
<input type="hidden" name="action" value="deny" />
|
|
17073
|
+
<button class="ghost" type="submit">Deny</button>
|
|
17074
|
+
</form>`);
|
|
17075
|
+
}
|
|
17076
|
+
function redirectWithError(redirectUri, state, error) {
|
|
17077
|
+
const u = new URL(redirectUri);
|
|
17078
|
+
u.searchParams.set("error", error);
|
|
17079
|
+
if (state) u.searchParams.set("state", state);
|
|
17080
|
+
return new Response(null, { status: 302, headers: { Location: u.toString() } });
|
|
17081
|
+
}
|
|
17082
|
+
var oauthApp = new Hono14();
|
|
17083
|
+
oauthApp.use("*", async (c, next) => {
|
|
17084
|
+
c.header("Access-Control-Allow-Origin", "*");
|
|
17085
|
+
c.header("Access-Control-Allow-Methods", "GET, POST, OPTIONS");
|
|
17086
|
+
c.header("Access-Control-Allow-Headers", "Content-Type, Authorization, mcp-protocol-version");
|
|
17087
|
+
if (c.req.method === "OPTIONS") return c.body(null, 204);
|
|
17088
|
+
return next();
|
|
17089
|
+
});
|
|
17090
|
+
function authServerMetadata() {
|
|
17091
|
+
return {
|
|
17092
|
+
issuer: ISSUER,
|
|
17093
|
+
authorization_endpoint: `${ISSUER}/oauth/authorize`,
|
|
17094
|
+
token_endpoint: `${ISSUER}/oauth/token`,
|
|
17095
|
+
registration_endpoint: `${ISSUER}/oauth/register`,
|
|
17096
|
+
jwks_uri: `${ISSUER}/.well-known/jwks.json`,
|
|
17097
|
+
scopes_supported: [...SUPPORTED_SCOPES],
|
|
17098
|
+
response_types_supported: ["code"],
|
|
17099
|
+
grant_types_supported: ["authorization_code", "refresh_token"],
|
|
17100
|
+
code_challenge_methods_supported: ["S256"],
|
|
17101
|
+
token_endpoint_auth_methods_supported: ["none"]
|
|
17102
|
+
};
|
|
17103
|
+
}
|
|
17104
|
+
function openidConfiguration() {
|
|
17105
|
+
return {
|
|
17106
|
+
...authServerMetadata(),
|
|
17107
|
+
subject_types_supported: ["public"],
|
|
17108
|
+
id_token_signing_alg_values_supported: ["RS256"]
|
|
17109
|
+
};
|
|
17110
|
+
}
|
|
17111
|
+
oauthApp.get("/.well-known/oauth-authorization-server", (c) => {
|
|
17112
|
+
console.log("[oauth-dcr] GET /.well-known/oauth-authorization-server ua=%s", c.req.header("user-agent") ?? "");
|
|
17113
|
+
return c.json(authServerMetadata());
|
|
17114
|
+
});
|
|
17115
|
+
oauthApp.get("/.well-known/oauth-authorization-server/*", (c) => {
|
|
17116
|
+
console.log("[oauth-dcr] GET %s (path-aware AS) ua=%s", c.req.path, c.req.header("user-agent") ?? "");
|
|
17117
|
+
return c.json(authServerMetadata());
|
|
17118
|
+
});
|
|
17119
|
+
oauthApp.get("/.well-known/openid-configuration", (c) => {
|
|
17120
|
+
console.log("[oauth-dcr] GET /.well-known/openid-configuration ua=%s", c.req.header("user-agent") ?? "");
|
|
17121
|
+
return c.json(openidConfiguration());
|
|
17122
|
+
});
|
|
17123
|
+
oauthApp.get("/.well-known/openid-configuration/*", (c) => {
|
|
17124
|
+
console.log("[oauth-dcr] GET %s (path-aware OIDC) ua=%s", c.req.path, c.req.header("user-agent") ?? "");
|
|
17125
|
+
return c.json(openidConfiguration());
|
|
17126
|
+
});
|
|
17127
|
+
oauthApp.get("/.well-known/jwks.json", async (c) => {
|
|
17128
|
+
const { publicJwk } = await getKeys();
|
|
17129
|
+
c.header("Cache-Control", "public, max-age=3600");
|
|
17130
|
+
return c.json({ keys: [publicJwk] });
|
|
17131
|
+
});
|
|
17132
|
+
oauthApp.get("/.well-known/oauth-protected-resource", (c) => {
|
|
17133
|
+
return c.json(protectedResourceMetadata());
|
|
17134
|
+
});
|
|
17135
|
+
oauthApp.get("/.well-known/oauth-protected-resource/*", (c) => {
|
|
17136
|
+
return c.json(protectedResourceMetadata());
|
|
17137
|
+
});
|
|
17138
|
+
oauthApp.post("/oauth/register", async (c) => {
|
|
17139
|
+
const rawBody = await c.req.text();
|
|
17140
|
+
console.log("[oauth-dcr] POST /oauth/register ua=%s ct=%s body=%s", c.req.header("user-agent") ?? "", c.req.header("content-type") ?? "", rawBody.slice(0, 2e3));
|
|
17141
|
+
let body;
|
|
17142
|
+
try {
|
|
17143
|
+
body = rawBody ? JSON.parse(rawBody) : null;
|
|
17144
|
+
} catch {
|
|
17145
|
+
body = null;
|
|
17146
|
+
}
|
|
17147
|
+
if (!body) {
|
|
17148
|
+
console.log("[oauth-dcr] register REJECT reason=body_not_json");
|
|
17149
|
+
return c.json({ error: "invalid_client_metadata", error_description: "Body must be JSON" }, 400);
|
|
17150
|
+
}
|
|
17151
|
+
console.log("[oauth-dcr] register body keys=%s redirect_uris=%s", Object.keys(body).join(","), JSON.stringify(body.redirect_uris));
|
|
17152
|
+
const redirectUris = Array.isArray(body.redirect_uris) ? body.redirect_uris.filter((u) => typeof u === "string") : [];
|
|
17153
|
+
if (redirectUris.length === 0) {
|
|
17154
|
+
console.log("[oauth-dcr] register REJECT reason=no_redirect_uris");
|
|
17155
|
+
return c.json({ error: "invalid_redirect_uri", error_description: "redirect_uris is required" }, 400);
|
|
17156
|
+
}
|
|
17157
|
+
for (const uri of redirectUris) {
|
|
17158
|
+
if (!isAllowedRedirect(uri)) {
|
|
17159
|
+
console.log("[oauth-dcr] register REJECT reason=redirect_uri_not_allowed uri=%s", uri);
|
|
17160
|
+
return c.json({ error: "invalid_redirect_uri", error_description: `redirect_uri not allowed: ${uri}` }, 400);
|
|
17161
|
+
}
|
|
17162
|
+
}
|
|
17163
|
+
const clientName = typeof body.client_name === "string" ? body.client_name : null;
|
|
17164
|
+
const clientId = `client_${randomBytes2(16).toString("hex")}`;
|
|
17165
|
+
await registerClient(clientId, redirectUris, clientName);
|
|
17166
|
+
console.log("[oauth-dcr] register OK client_id=%s redirect_uris=%s", clientId, JSON.stringify(redirectUris));
|
|
17167
|
+
return c.json({
|
|
17168
|
+
client_id: clientId,
|
|
17169
|
+
redirect_uris: redirectUris,
|
|
17170
|
+
client_name: clientName ?? void 0,
|
|
17171
|
+
token_endpoint_auth_method: "none",
|
|
17172
|
+
grant_types: ["authorization_code", "refresh_token"],
|
|
17173
|
+
response_types: ["code"],
|
|
17174
|
+
client_id_issued_at: Math.floor(Date.now() / 1e3)
|
|
17175
|
+
}, 201);
|
|
17176
|
+
});
|
|
17177
|
+
function parseAuthParams(get) {
|
|
17178
|
+
return {
|
|
17179
|
+
response_type: get("response_type") ?? "",
|
|
17180
|
+
client_id: get("client_id") ?? "",
|
|
17181
|
+
redirect_uri: get("redirect_uri") ?? "",
|
|
17182
|
+
code_challenge: get("code_challenge") ?? "",
|
|
17183
|
+
code_challenge_method: get("code_challenge_method") ?? "",
|
|
17184
|
+
scope: get("scope") ?? "",
|
|
17185
|
+
state: get("state") ?? "",
|
|
17186
|
+
resource: get("resource") ?? ""
|
|
17187
|
+
};
|
|
17188
|
+
}
|
|
17189
|
+
async function validateAuthRequest(p) {
|
|
17190
|
+
if (p.response_type !== "code") return { ok: false, status: 400, message: "response_type must be code" };
|
|
17191
|
+
if (!p.client_id) return { ok: false, status: 400, message: "client_id is required" };
|
|
17192
|
+
const client2 = await getClient(p.client_id);
|
|
17193
|
+
if (!client2) return { ok: false, status: 400, message: "unknown client_id" };
|
|
17194
|
+
if (!p.redirect_uri || !client2.redirect_uris.includes(p.redirect_uri)) return { ok: false, status: 400, message: "redirect_uri does not match the registered set" };
|
|
17195
|
+
if (!isAllowedRedirect(p.redirect_uri)) return { ok: false, status: 400, message: "redirect_uri host not allowed" };
|
|
17196
|
+
if (!p.code_challenge || p.code_challenge_method !== "S256") return { ok: false, status: 400, message: "PKCE S256 code_challenge is required" };
|
|
17197
|
+
return { ok: true };
|
|
17198
|
+
}
|
|
17199
|
+
oauthApp.get("/oauth/authorize", async (c) => {
|
|
17200
|
+
const p = parseAuthParams((k) => c.req.query(k));
|
|
17201
|
+
console.log("[oauth-dcr] GET /oauth/authorize client_id=%s redirect_uri=%s scope=%s resource=%s", p.client_id, p.redirect_uri, p.scope, p.resource);
|
|
17202
|
+
const valid = await validateAuthRequest(p);
|
|
17203
|
+
if (!valid.ok) {
|
|
17204
|
+
console.log("[oauth-dcr] authorize REJECT reason=%s", valid.message);
|
|
17205
|
+
return c.text(valid.message, valid.status);
|
|
17206
|
+
}
|
|
17207
|
+
const user = await sessionUser(getCookie(c, "session"));
|
|
17208
|
+
if (!user) return c.html(renderLogin(p));
|
|
17209
|
+
const negotiated = { ...p, scope: negotiateScope(p.scope, user, p.resource) };
|
|
17210
|
+
return c.html(renderConsent(negotiated, user.email));
|
|
17211
|
+
});
|
|
17212
|
+
oauthApp.post("/oauth/authorize", async (c) => {
|
|
17213
|
+
const form = await c.req.parseBody();
|
|
17214
|
+
const get = (k) => {
|
|
17215
|
+
const v = form[k];
|
|
17216
|
+
return typeof v === "string" ? v : void 0;
|
|
17217
|
+
};
|
|
17218
|
+
const p = parseAuthParams(get);
|
|
17219
|
+
const action = get("action") ?? "";
|
|
17220
|
+
const valid = await validateAuthRequest(p);
|
|
17221
|
+
if (!valid.ok) return c.text(valid.message, valid.status);
|
|
17222
|
+
if (action === "login") {
|
|
17223
|
+
const email = (get("email") ?? "").trim().toLowerCase();
|
|
17224
|
+
const password = get("password") ?? "";
|
|
17225
|
+
const user2 = email ? await getUserByEmail(email) : void 0;
|
|
17226
|
+
const okPw = !!(user2 && user2.password_hash && verifyPassword(password, user2.password_hash));
|
|
17227
|
+
console.log("[oauth-dcr] LOGIN email=%s found=%s hasHash=%s pwLen=%d ok=%s", email, !!user2, !!(user2 && user2.password_hash), password.length, okPw);
|
|
17228
|
+
if (!okPw) {
|
|
17229
|
+
return c.html(renderLogin(p, "Invalid email or password."));
|
|
17230
|
+
}
|
|
17231
|
+
setCookie(c, "session", signSession(user2.id), sessionCookieOptions);
|
|
17232
|
+
return c.html(renderConsent({ ...p, scope: negotiateScope(p.scope, user2, p.resource) }, user2.email));
|
|
17233
|
+
}
|
|
17234
|
+
const user = await sessionUser(getCookie(c, "session"));
|
|
17235
|
+
if (!user) return c.html(renderLogin(p, "Your session expired. Please sign in again."));
|
|
17236
|
+
if (action === "deny") return redirectWithError(p.redirect_uri, p.state, "access_denied");
|
|
17237
|
+
if (action !== "approve") return c.text("unsupported action", 400);
|
|
17238
|
+
const scope = negotiateScope(p.scope, user, p.resource);
|
|
17239
|
+
const code = `code_${randomBytes2(32).toString("base64url")}`;
|
|
17240
|
+
const expiresAt = new Date(Date.now() + CODE_TTL_SECONDS * 1e3).toISOString();
|
|
17241
|
+
await putCode({
|
|
17242
|
+
code,
|
|
17243
|
+
client_id: p.client_id,
|
|
17244
|
+
redirect_uri: p.redirect_uri,
|
|
17245
|
+
code_challenge: p.code_challenge,
|
|
17246
|
+
identity: user.email,
|
|
17247
|
+
scope,
|
|
17248
|
+
resource: p.resource || RESOURCE(),
|
|
17249
|
+
expires_at: expiresAt
|
|
17250
|
+
});
|
|
17251
|
+
const u = new URL(p.redirect_uri);
|
|
17252
|
+
u.searchParams.set("code", code);
|
|
17253
|
+
if (p.state) u.searchParams.set("state", p.state);
|
|
17254
|
+
return new Response(null, { status: 302, headers: { Location: u.toString() } });
|
|
17255
|
+
});
|
|
17256
|
+
function pkceMatches(verifier, challenge) {
|
|
17257
|
+
if (!verifier) return false;
|
|
17258
|
+
const computed = createHash3("sha256").update(verifier).digest("base64url");
|
|
17259
|
+
return computed === challenge;
|
|
17260
|
+
}
|
|
17261
|
+
async function mintAccessToken(identity, scope, plan, audience) {
|
|
17262
|
+
const { privateKey, kid } = await getKeys();
|
|
17263
|
+
return new SignJWT({ scope, plan }).setProtectedHeader({ alg: "RS256", kid }).setIssuer(ISSUER).setSubject(identity).setAudience(audience).setIssuedAt().setJti(randomUUID3()).setExpirationTime(`${ACCESS_TTL_SECONDS}s`).sign(privateKey);
|
|
17264
|
+
}
|
|
17265
|
+
function tokenErrorResponse(c, error, description, status) {
|
|
17266
|
+
return c.json({ error, error_description: description }, status);
|
|
17267
|
+
}
|
|
17268
|
+
oauthApp.post("/oauth/token", async (c) => {
|
|
17269
|
+
const form = await c.req.parseBody();
|
|
17270
|
+
const get = (k) => {
|
|
17271
|
+
const v = form[k];
|
|
17272
|
+
return typeof v === "string" ? v : void 0;
|
|
17273
|
+
};
|
|
17274
|
+
const grantType = get("grant_type");
|
|
17275
|
+
if (grantType === "authorization_code") {
|
|
17276
|
+
const code = get("code") ?? "";
|
|
17277
|
+
const clientId = get("client_id") ?? "";
|
|
17278
|
+
const redirectUri = get("redirect_uri") ?? "";
|
|
17279
|
+
const verifier = get("code_verifier") ?? "";
|
|
17280
|
+
const record = await consumeCode(code);
|
|
17281
|
+
if (!record) return tokenErrorResponse(c, "invalid_grant", "Authorization code is invalid or already used", 400);
|
|
17282
|
+
if (new Date(record.expires_at).getTime() < Date.now()) return tokenErrorResponse(c, "invalid_grant", "Authorization code has expired", 400);
|
|
17283
|
+
if (record.client_id !== clientId) return tokenErrorResponse(c, "invalid_grant", "client_id mismatch", 400);
|
|
17284
|
+
if (record.redirect_uri !== redirectUri) return tokenErrorResponse(c, "invalid_grant", "redirect_uri mismatch", 400);
|
|
17285
|
+
if (!pkceMatches(verifier, record.code_challenge)) return tokenErrorResponse(c, "invalid_grant", "PKCE verification failed", 400);
|
|
17286
|
+
const user = await getUserByEmail(record.identity);
|
|
17287
|
+
const plan = user ? resolvePlan(user) : "free";
|
|
17288
|
+
const audience = record.resource ?? RESOURCE();
|
|
17289
|
+
const accessToken = await mintAccessToken(record.identity, record.scope, plan, audience);
|
|
17290
|
+
const refreshToken = `rt_${randomBytes2(40).toString("base64url")}`;
|
|
17291
|
+
await putRefresh({
|
|
17292
|
+
refresh_token: refreshToken,
|
|
17293
|
+
client_id: clientId,
|
|
17294
|
+
identity: record.identity,
|
|
17295
|
+
scope: record.scope,
|
|
17296
|
+
resource: audience,
|
|
17297
|
+
expires_at: new Date(Date.now() + REFRESH_TTL_SECONDS * 1e3).toISOString(),
|
|
17298
|
+
revoked: 0
|
|
17299
|
+
});
|
|
17300
|
+
return c.json({ access_token: accessToken, token_type: "Bearer", expires_in: ACCESS_TTL_SECONDS, refresh_token: refreshToken, scope: record.scope });
|
|
17301
|
+
}
|
|
17302
|
+
if (grantType === "refresh_token") {
|
|
17303
|
+
const refreshToken = get("refresh_token") ?? "";
|
|
17304
|
+
const clientId = get("client_id") ?? "";
|
|
17305
|
+
const record = await getRefresh(refreshToken);
|
|
17306
|
+
if (!record || record.revoked) return tokenErrorResponse(c, "invalid_grant", "Refresh token is invalid or revoked", 400);
|
|
17307
|
+
if (new Date(record.expires_at).getTime() < Date.now()) return tokenErrorResponse(c, "invalid_grant", "Refresh token has expired", 400);
|
|
17308
|
+
if (clientId && record.client_id !== clientId) return tokenErrorResponse(c, "invalid_grant", "client_id mismatch", 400);
|
|
17309
|
+
const user = await getUserByEmail(record.identity);
|
|
17310
|
+
const plan = user ? resolvePlan(user) : "free";
|
|
17311
|
+
const audience = record.resource ?? RESOURCE();
|
|
17312
|
+
const accessToken = await mintAccessToken(record.identity, record.scope, plan, audience);
|
|
17313
|
+
const nextRefresh = `rt_${randomBytes2(40).toString("base64url")}`;
|
|
17314
|
+
await rotateRefresh(refreshToken, {
|
|
17315
|
+
refresh_token: nextRefresh,
|
|
17316
|
+
client_id: record.client_id,
|
|
17317
|
+
identity: record.identity,
|
|
17318
|
+
scope: record.scope,
|
|
17319
|
+
resource: audience,
|
|
17320
|
+
expires_at: new Date(Date.now() + REFRESH_TTL_SECONDS * 1e3).toISOString(),
|
|
17321
|
+
revoked: 0
|
|
17322
|
+
});
|
|
17323
|
+
return c.json({ access_token: accessToken, token_type: "Bearer", expires_in: ACCESS_TTL_SECONDS, refresh_token: nextRefresh, scope: record.scope });
|
|
17324
|
+
}
|
|
17325
|
+
return tokenErrorResponse(c, "unsupported_grant_type", "grant_type must be authorization_code or refresh_token", 400);
|
|
17326
|
+
});
|
|
17327
|
+
|
|
16523
17328
|
// src/api/site-audit-worker.ts
|
|
16524
17329
|
var MAX_CONCURRENT_SITE_AUDIT = 1;
|
|
16525
17330
|
async function drainSiteAuditQueue(budget) {
|
|
@@ -16555,12 +17360,115 @@ function startSiteAuditWorker() {
|
|
|
16555
17360
|
|
|
16556
17361
|
// src/api/server.ts
|
|
16557
17362
|
import { createMiddleware as createMiddleware3 } from "hono/factory";
|
|
16558
|
-
import { getCookie, setCookie, deleteCookie } from "hono/cookie";
|
|
17363
|
+
import { getCookie as getCookie2, setCookie as setCookie2, deleteCookie } from "hono/cookie";
|
|
16559
17364
|
import Stripe2 from "stripe";
|
|
16560
17365
|
|
|
17366
|
+
// src/api/scrape-vault-sink.ts
|
|
17367
|
+
var MAX_DEPOSIT_CHARS = 5e6;
|
|
17368
|
+
var SCRAPE_FALLBACK_PREFIX = "scrape-fallback/";
|
|
17369
|
+
var SCRAPE_BLOB_TTL_MS = 24 * 60 * 60 * 1e3;
|
|
17370
|
+
async function depositScrapeToVault(user, opts) {
|
|
17371
|
+
try {
|
|
17372
|
+
const content = (opts.content ?? "").trim();
|
|
17373
|
+
if (!content) return { deposited: false, error: "empty content" };
|
|
17374
|
+
const clipped = content.length > MAX_DEPOSIT_CHARS ? content.slice(0, MAX_DEPOSIT_CHARS) : content;
|
|
17375
|
+
const { key, error } = await getOrCreateUserMemoryKey(user);
|
|
17376
|
+
if (!key) return { deposited: false, error: error ?? "memory unavailable" };
|
|
17377
|
+
const vault = opts.vault?.trim() || personalVault(user);
|
|
17378
|
+
const title = (opts.title?.trim() || opts.source).slice(0, 200);
|
|
17379
|
+
const res = await memoryCall(
|
|
17380
|
+
"libraryIngestTool",
|
|
17381
|
+
{ title, content: clipped, source: opts.source, vault },
|
|
17382
|
+
key
|
|
17383
|
+
);
|
|
17384
|
+
if (!res.ok) return { deposited: false, vault, error: res.error ?? "ingest failed" };
|
|
17385
|
+
return { deposited: true, vault, noteId: res.noteId, path: res.path, chunks: res.chunks };
|
|
17386
|
+
} catch (err) {
|
|
17387
|
+
return { deposited: false, error: err?.message ?? "deposit failed" };
|
|
17388
|
+
}
|
|
17389
|
+
}
|
|
17390
|
+
async function uploadFallbackBlob(opts) {
|
|
17391
|
+
try {
|
|
17392
|
+
const content = (opts.content ?? "").trim();
|
|
17393
|
+
if (!content) return null;
|
|
17394
|
+
const slug = (slugify(opts.title || opts.source) || "scrape").slice(0, 60);
|
|
17395
|
+
const key = `${SCRAPE_FALLBACK_PREFIX}${slug}.md`;
|
|
17396
|
+
const doc = `# ${opts.title}
|
|
17397
|
+
Source: ${opts.source}
|
|
17398
|
+
|
|
17399
|
+
${content}`;
|
|
17400
|
+
const stored = await getBlobStore().put(key, doc, "text/markdown");
|
|
17401
|
+
return { fileUrl: stored.url, fileExpiresAt: new Date(Date.now() + SCRAPE_BLOB_TTL_MS).toISOString() };
|
|
17402
|
+
} catch {
|
|
17403
|
+
return null;
|
|
17404
|
+
}
|
|
17405
|
+
}
|
|
17406
|
+
async function persistScrapeBody(user, opts) {
|
|
17407
|
+
const deposit = await depositScrapeToVault(user, opts);
|
|
17408
|
+
if (deposit.deposited) return deposit;
|
|
17409
|
+
const fallback = await uploadFallbackBlob(opts);
|
|
17410
|
+
if (!fallback) return deposit;
|
|
17411
|
+
return { ...deposit, ...fallback };
|
|
17412
|
+
}
|
|
17413
|
+
|
|
17414
|
+
// src/api/scrape-blob-cleanup.ts
|
|
17415
|
+
import { readdir, stat, unlink } from "fs/promises";
|
|
17416
|
+
import { homedir as homedir4 } from "os";
|
|
17417
|
+
import { join as join7 } from "path";
|
|
17418
|
+
async function cleanupVercel(token, cutoff) {
|
|
17419
|
+
const { list, del } = await import("@vercel/blob");
|
|
17420
|
+
let cursor;
|
|
17421
|
+
let deleted = 0;
|
|
17422
|
+
do {
|
|
17423
|
+
const res = await list({ prefix: SCRAPE_FALLBACK_PREFIX, cursor, token });
|
|
17424
|
+
const expired = res.blobs.filter((b) => new Date(b.uploadedAt).getTime() < cutoff);
|
|
17425
|
+
if (expired.length) {
|
|
17426
|
+
await del(expired.map((b) => b.url), { token });
|
|
17427
|
+
deleted += expired.length;
|
|
17428
|
+
}
|
|
17429
|
+
cursor = res.hasMore ? res.cursor : void 0;
|
|
17430
|
+
} while (cursor);
|
|
17431
|
+
return { deleted, store: "vercel-blob" };
|
|
17432
|
+
}
|
|
17433
|
+
async function cleanupLocal(cutoff) {
|
|
17434
|
+
const baseDir = process.env.MCP_SCRAPER_OUTPUT_DIR?.trim() || join7(homedir4(), "Downloads", "mcp-scraper");
|
|
17435
|
+
const dir = join7(baseDir, "blobs", SCRAPE_FALLBACK_PREFIX.replace(/\/$/, ""));
|
|
17436
|
+
let deleted = 0;
|
|
17437
|
+
let entries;
|
|
17438
|
+
try {
|
|
17439
|
+
entries = await readdir(dir);
|
|
17440
|
+
} catch {
|
|
17441
|
+
return { deleted: 0, store: "local" };
|
|
17442
|
+
}
|
|
17443
|
+
for (const name of entries) {
|
|
17444
|
+
const path5 = join7(dir, name);
|
|
17445
|
+
try {
|
|
17446
|
+
const s = await stat(path5);
|
|
17447
|
+
if (s.isFile() && s.mtimeMs < cutoff) {
|
|
17448
|
+
await unlink(path5);
|
|
17449
|
+
deleted++;
|
|
17450
|
+
}
|
|
17451
|
+
} catch {
|
|
17452
|
+
}
|
|
17453
|
+
}
|
|
17454
|
+
return { deleted, store: "local" };
|
|
17455
|
+
}
|
|
17456
|
+
async function cleanupExpiredScrapeBlobs(maxAgeMs = SCRAPE_BLOB_TTL_MS) {
|
|
17457
|
+
const cutoff = Date.now() - maxAgeMs;
|
|
17458
|
+
try {
|
|
17459
|
+
const token = process.env.BLOB_READ_WRITE_TOKEN;
|
|
17460
|
+
return token ? await cleanupVercel(token, cutoff) : await cleanupLocal(cutoff);
|
|
17461
|
+
} catch {
|
|
17462
|
+
return { deleted: 0, store: "none" };
|
|
17463
|
+
}
|
|
17464
|
+
}
|
|
17465
|
+
|
|
16561
17466
|
// src/api/credit-operations.ts
|
|
16562
|
-
async function grantSignupCredit(
|
|
16563
|
-
return;
|
|
17467
|
+
async function grantSignupCredit(userId) {
|
|
17468
|
+
if (!FREE_SIGNUP_MC || FREE_SIGNUP_MC <= 0) return;
|
|
17469
|
+
const already = await ledgerExistsForOperation(userId, LedgerOperation.SIGNUP_GRANT);
|
|
17470
|
+
if (already) return;
|
|
17471
|
+
await creditMc(userId, FREE_SIGNUP_MC, LedgerOperation.SIGNUP_GRANT, "Welcome credits");
|
|
16564
17472
|
}
|
|
16565
17473
|
async function applyMonthlyFreeRefresh(user) {
|
|
16566
17474
|
return user;
|
|
@@ -16598,46 +17506,168 @@ async function getFreeCreditBreakdown(userId) {
|
|
|
16598
17506
|
};
|
|
16599
17507
|
}
|
|
16600
17508
|
|
|
16601
|
-
// src/api/
|
|
16602
|
-
import {
|
|
16603
|
-
var
|
|
16604
|
-
function
|
|
16605
|
-
|
|
16606
|
-
|
|
16607
|
-
if (
|
|
16608
|
-
|
|
17509
|
+
// src/api/memory-db.ts
|
|
17510
|
+
import { neon } from "@neondatabase/serverless";
|
|
17511
|
+
var _sql = null;
|
|
17512
|
+
function sql() {
|
|
17513
|
+
if (_sql) return _sql;
|
|
17514
|
+
const url = process.env.MEMORY_DATABASE_URL?.trim();
|
|
17515
|
+
if (!url) throw new Error("MEMORY_DATABASE_URL is not set (paa-crawler needs mcp-memory Neon URL for reads)");
|
|
17516
|
+
_sql = neon(url);
|
|
17517
|
+
return _sql;
|
|
17518
|
+
}
|
|
17519
|
+
function physicalVault(owner, vault) {
|
|
17520
|
+
return `${owner} ${vault}`;
|
|
17521
|
+
}
|
|
17522
|
+
function isUnlimited(plan) {
|
|
17523
|
+
return plan === "unlimited";
|
|
17524
|
+
}
|
|
17525
|
+
function quotaBytesForPlan(plan) {
|
|
17526
|
+
return MEMORY_PLAN_QUOTA[plan] ?? MEMORY_PLAN_QUOTA.free;
|
|
17527
|
+
}
|
|
17528
|
+
function toGb(bytes) {
|
|
17529
|
+
return Math.round(bytes / 1e9 * 100) / 100;
|
|
17530
|
+
}
|
|
17531
|
+
async function entitledVaultRows(identity) {
|
|
17532
|
+
const rows = await sql().query(
|
|
17533
|
+
`SELECT vault, owner_identity FROM mem_vault_registry WHERE grantee = $1 ORDER BY created_at ASC`,
|
|
17534
|
+
[identity]
|
|
17535
|
+
);
|
|
17536
|
+
const seen = /* @__PURE__ */ new Map();
|
|
17537
|
+
for (const r of rows) {
|
|
17538
|
+
if (seen.has(r.vault)) continue;
|
|
17539
|
+
seen.set(r.vault, { vault: r.vault, owner: r.owner_identity, physical: physicalVault(r.owner_identity, r.vault) });
|
|
17540
|
+
}
|
|
17541
|
+
return [...seen.values()];
|
|
17542
|
+
}
|
|
17543
|
+
async function storageByPhysical(physicals) {
|
|
17544
|
+
if (physicals.length === 0) return /* @__PURE__ */ new Map();
|
|
17545
|
+
const rows = await sql().query(
|
|
17546
|
+
`SELECT vault, COUNT(*)::int AS notes, COALESCE(SUM(octet_length(content)), 0)::bigint AS bytes
|
|
17547
|
+
FROM mem_notes WHERE vault = ANY($1) GROUP BY vault`,
|
|
17548
|
+
[physicals]
|
|
17549
|
+
);
|
|
17550
|
+
const m = /* @__PURE__ */ new Map();
|
|
17551
|
+
for (const r of rows) m.set(r.vault, { notes: Number(r.notes), bytes: Number(r.bytes) });
|
|
17552
|
+
return m;
|
|
17553
|
+
}
|
|
17554
|
+
async function dbListVaults(identity) {
|
|
17555
|
+
const ents = await entitledVaultRows(identity);
|
|
17556
|
+
const storage = await storageByPhysical(ents.map((e) => e.physical));
|
|
17557
|
+
return ents.map((e) => {
|
|
17558
|
+
const st = storage.get(e.physical);
|
|
17559
|
+
const isOwner = e.owner === identity;
|
|
17560
|
+
return {
|
|
17561
|
+
vault: e.vault,
|
|
17562
|
+
role: isOwner ? "owner" : "shared",
|
|
17563
|
+
...isOwner ? {} : { sharedBy: e.owner },
|
|
17564
|
+
notes: st?.notes ?? 0,
|
|
17565
|
+
bytes: st?.bytes ?? 0
|
|
17566
|
+
};
|
|
17567
|
+
});
|
|
16609
17568
|
}
|
|
16610
|
-
|
|
16611
|
-
|
|
16612
|
-
|
|
16613
|
-
|
|
16614
|
-
|
|
16615
|
-
|
|
16616
|
-
|
|
16617
|
-
|
|
17569
|
+
async function dbStorage(identity, plan) {
|
|
17570
|
+
const ents = await entitledVaultRows(identity);
|
|
17571
|
+
const storage = await storageByPhysical(ents.map((e) => e.physical));
|
|
17572
|
+
let totalBytes = 0;
|
|
17573
|
+
const perVault = ents.map((e) => {
|
|
17574
|
+
const st = storage.get(e.physical);
|
|
17575
|
+
const bytes = st?.bytes ?? 0;
|
|
17576
|
+
totalBytes += bytes;
|
|
17577
|
+
return { vault: e.vault, bytes, gb: toGb(bytes), notes: st?.notes ?? 0 };
|
|
17578
|
+
});
|
|
17579
|
+
const unlimited = isUnlimited(plan);
|
|
17580
|
+
const quotaBytes = unlimited ? null : quotaBytesForPlan(plan);
|
|
17581
|
+
return {
|
|
17582
|
+
ok: true,
|
|
17583
|
+
totalBytes,
|
|
17584
|
+
totalGb: toGb(totalBytes),
|
|
17585
|
+
quotaBytes,
|
|
17586
|
+
quotaGb: quotaBytes == null ? null : toGb(quotaBytes),
|
|
17587
|
+
unlimited,
|
|
17588
|
+
perVault
|
|
17589
|
+
};
|
|
16618
17590
|
}
|
|
16619
|
-
function
|
|
16620
|
-
const
|
|
16621
|
-
|
|
16622
|
-
|
|
17591
|
+
async function resolvePhysical(identity, vault) {
|
|
17592
|
+
const rows = await sql().query(
|
|
17593
|
+
`SELECT owner_identity FROM mem_vault_registry WHERE grantee = $1 AND vault = $2 LIMIT 1`,
|
|
17594
|
+
[identity, vault]
|
|
17595
|
+
);
|
|
17596
|
+
if (!rows[0]) return null;
|
|
17597
|
+
return physicalVault(rows[0].owner_identity, vault);
|
|
17598
|
+
}
|
|
17599
|
+
async function dbNotes(identity, vault) {
|
|
17600
|
+
const physical = await resolvePhysical(identity, vault);
|
|
17601
|
+
if (!physical) return [];
|
|
17602
|
+
const rows = await sql().query(
|
|
17603
|
+
`SELECT path, title, kind, source, updated_at, octet_length(content) AS bytes, props
|
|
17604
|
+
FROM mem_notes WHERE vault = $1 ORDER BY updated_at DESC`,
|
|
17605
|
+
[physical]
|
|
17606
|
+
);
|
|
17607
|
+
return rows.map((r) => ({
|
|
17608
|
+
path: r.path,
|
|
17609
|
+
title: r.title,
|
|
17610
|
+
kind: r.kind,
|
|
17611
|
+
source: r.source,
|
|
17612
|
+
updatedAt: r.updated_at,
|
|
17613
|
+
bytes: Number(r.bytes),
|
|
17614
|
+
props: r.props && typeof r.props === "object" ? r.props : {}
|
|
17615
|
+
}));
|
|
16623
17616
|
}
|
|
16624
|
-
function
|
|
16625
|
-
const
|
|
16626
|
-
if (
|
|
16627
|
-
const
|
|
16628
|
-
|
|
16629
|
-
|
|
16630
|
-
|
|
16631
|
-
|
|
16632
|
-
|
|
17617
|
+
async function dbNote(identity, vault, path5) {
|
|
17618
|
+
const physical = await resolvePhysical(identity, vault);
|
|
17619
|
+
if (!physical) return null;
|
|
17620
|
+
const rows = await sql().query(
|
|
17621
|
+
`SELECT path, title, content, kind, source, updated_at, captured_at
|
|
17622
|
+
FROM mem_notes WHERE vault = $1 AND path = $2 LIMIT 1`,
|
|
17623
|
+
[physical, path5]
|
|
17624
|
+
);
|
|
17625
|
+
const r = rows[0];
|
|
17626
|
+
if (!r) return null;
|
|
17627
|
+
return {
|
|
17628
|
+
path: r.path,
|
|
17629
|
+
title: r.title,
|
|
17630
|
+
content: r.content,
|
|
17631
|
+
kind: r.kind,
|
|
17632
|
+
source: r.source,
|
|
17633
|
+
updatedAt: r.updated_at,
|
|
17634
|
+
capturedAt: r.captured_at
|
|
17635
|
+
};
|
|
17636
|
+
}
|
|
17637
|
+
var FREE_COST_CAP_USD = 1;
|
|
17638
|
+
async function dbUsage(identity, plan) {
|
|
17639
|
+
const now = /* @__PURE__ */ new Date();
|
|
17640
|
+
const period = `${now.getUTCFullYear()}-${String(now.getUTCMonth() + 1).padStart(2, "0")}`;
|
|
17641
|
+
const rows = await sql().query(
|
|
17642
|
+
`SELECT source, COALESCE(SUM(cost_usd), 0)::float8 AS total
|
|
17643
|
+
FROM mem_usage_ledger WHERE identity = $1 AND period = $2 GROUP BY source`,
|
|
17644
|
+
[identity, period]
|
|
17645
|
+
);
|
|
17646
|
+
const bySource = {};
|
|
17647
|
+
let costUsd = 0;
|
|
17648
|
+
for (const r of rows) {
|
|
17649
|
+
const v = Number(r.total);
|
|
17650
|
+
bySource[r.source] = v;
|
|
17651
|
+
costUsd += v;
|
|
17652
|
+
}
|
|
17653
|
+
return {
|
|
17654
|
+
ok: true,
|
|
17655
|
+
period,
|
|
17656
|
+
plan,
|
|
17657
|
+
costUsd,
|
|
17658
|
+
bySource,
|
|
17659
|
+
freeCapUsd: FREE_COST_CAP_USD,
|
|
17660
|
+
freeCapReached: plan === "free" && costUsd >= FREE_COST_CAP_USD,
|
|
17661
|
+
unlimited: isUnlimited(plan)
|
|
17662
|
+
};
|
|
16633
17663
|
}
|
|
16634
17664
|
|
|
16635
17665
|
// src/api/server.ts
|
|
16636
|
-
var
|
|
16637
|
-
var isProduction2 =
|
|
16638
|
-
var
|
|
17666
|
+
var secureCookies2 = process.env.NODE_ENV === "production" || process.env.VERCEL === "1";
|
|
17667
|
+
var isProduction2 = secureCookies2;
|
|
17668
|
+
var sessionCookieOptions2 = {
|
|
16639
17669
|
httpOnly: true,
|
|
16640
|
-
secure:
|
|
17670
|
+
secure: secureCookies2,
|
|
16641
17671
|
path: "/",
|
|
16642
17672
|
maxAge: 60 * 60 * 24 * 30,
|
|
16643
17673
|
sameSite: "Strict"
|
|
@@ -16698,7 +17728,7 @@ var adminAuth = createMiddleware3(async (c, next) => {
|
|
|
16698
17728
|
return next();
|
|
16699
17729
|
});
|
|
16700
17730
|
var sessionAuth = createMiddleware3(async (c, next) => {
|
|
16701
|
-
const token =
|
|
17731
|
+
const token = getCookie2(c, "session");
|
|
16702
17732
|
if (!token) return c.json({ error: "Not authenticated" }, 401);
|
|
16703
17733
|
const userId = verifySession(token);
|
|
16704
17734
|
if (!userId) return c.json({ error: "Session expired" }, 401);
|
|
@@ -16709,7 +17739,7 @@ var sessionAuth = createMiddleware3(async (c, next) => {
|
|
|
16709
17739
|
c.set("sessionUser", { ...refreshed, balance_mc: balanceMc });
|
|
16710
17740
|
return next();
|
|
16711
17741
|
});
|
|
16712
|
-
var app = new
|
|
17742
|
+
var app = new Hono15();
|
|
16713
17743
|
var STRIPE_API_VERSION = "2026-02-25.clover";
|
|
16714
17744
|
function requireStripeSecret() {
|
|
16715
17745
|
const secret2 = process.env.STRIPE_SECRET_KEY?.trim();
|
|
@@ -16737,6 +17767,33 @@ app.use("*", async (c, next) => {
|
|
|
16737
17767
|
c.header("X-Content-Type-Options", "nosniff");
|
|
16738
17768
|
c.header("Referrer-Policy", "same-origin");
|
|
16739
17769
|
});
|
|
17770
|
+
function opForCostPath(p) {
|
|
17771
|
+
if (p === "/harvest" || p === "/harvest/sync") return "harvest";
|
|
17772
|
+
if (p === "/extract-url") return "page_scrape";
|
|
17773
|
+
if (p === "/extract-site") return "extract_site";
|
|
17774
|
+
if (p === "/map-urls") return "url_map";
|
|
17775
|
+
if (p === "/youtube/harvest") return "yt_channel";
|
|
17776
|
+
if (p === "/youtube/transcribe") return "yt_transcription";
|
|
17777
|
+
if (p === "/maps/search") return "maps_search";
|
|
17778
|
+
if (p === "/maps/place") return "maps_place";
|
|
17779
|
+
if (p.startsWith("/facebook/")) {
|
|
17780
|
+
if (p.endsWith("transcribe")) return "fb_transcribe";
|
|
17781
|
+
if (p.endsWith("/search")) return "fb_search";
|
|
17782
|
+
return "fb_ad";
|
|
17783
|
+
}
|
|
17784
|
+
if (p.startsWith("/instagram/")) return "instagram";
|
|
17785
|
+
if (p.startsWith("/api/internal/site-architecture-auditor")) return "audit_site";
|
|
17786
|
+
if (p.startsWith("/agent/")) return "browser_session";
|
|
17787
|
+
return null;
|
|
17788
|
+
}
|
|
17789
|
+
app.use("*", (c, next) => {
|
|
17790
|
+
const op = opForCostPath(new URL(c.req.url).pathname);
|
|
17791
|
+
if (!op) return next();
|
|
17792
|
+
const probeRunId = c.req.header("x-cost-probe-id") ?? null;
|
|
17793
|
+
const forceHeadless = c.req.header("x-kernel-headless") === "true";
|
|
17794
|
+
return runWithCostContext({ op, probeRunId, forceHeadless }, () => next());
|
|
17795
|
+
});
|
|
17796
|
+
void migrateCostTelemetry();
|
|
16740
17797
|
app.post("/auth/register", requireAllowedOrigin, async (c) => {
|
|
16741
17798
|
const { email, password } = await c.req.json();
|
|
16742
17799
|
const normalizedEmail = email?.trim().toLowerCase();
|
|
@@ -16768,8 +17825,16 @@ app.post("/auth/register", requireAllowedOrigin, async (c) => {
|
|
|
16768
17825
|
}
|
|
16769
17826
|
const newKey = await rotateApiKey(user.id);
|
|
16770
17827
|
await grantSignupCredit(user.id);
|
|
17828
|
+
void (async () => {
|
|
17829
|
+
try {
|
|
17830
|
+
const full = await getUserById(user.id);
|
|
17831
|
+
if (full) await provisionMemoryForUser(full);
|
|
17832
|
+
} catch (err) {
|
|
17833
|
+
console.warn("[auth/register] memory provision failed (will retry on first write):", err instanceof Error ? err.message : String(err));
|
|
17834
|
+
}
|
|
17835
|
+
})();
|
|
16771
17836
|
const token = signSession(user.id);
|
|
16772
|
-
|
|
17837
|
+
setCookie2(c, "session", token, sessionCookieOptions2);
|
|
16773
17838
|
return c.json({ email: user.email, api_key: newKey }, 201);
|
|
16774
17839
|
} catch (err) {
|
|
16775
17840
|
const msg = err instanceof Error ? err.message : "";
|
|
@@ -16788,11 +17853,11 @@ app.post("/auth/login", requireAllowedOrigin, async (c) => {
|
|
|
16788
17853
|
return c.json({ error: "Invalid email or password" }, 401);
|
|
16789
17854
|
}
|
|
16790
17855
|
const token = signSession(user.id);
|
|
16791
|
-
|
|
17856
|
+
setCookie2(c, "session", token, sessionCookieOptions2);
|
|
16792
17857
|
return c.json({ email: user.email, api_key: user.key_active ? user.api_key : null });
|
|
16793
17858
|
});
|
|
16794
17859
|
app.post("/auth/logout", requireAllowedOrigin, (c) => {
|
|
16795
|
-
deleteCookie(c, "session", { path: "/", secure:
|
|
17860
|
+
deleteCookie(c, "session", { path: "/", secure: secureCookies2, sameSite: "Strict" });
|
|
16796
17861
|
return c.json({ ok: true });
|
|
16797
17862
|
});
|
|
16798
17863
|
app.post("/auth/forgot-password", requireAllowedOrigin, async (c) => {
|
|
@@ -16846,16 +17911,16 @@ app.get("/catalog", (c) => {
|
|
|
16846
17911
|
return c.json(CATALOG);
|
|
16847
17912
|
});
|
|
16848
17913
|
app.get("/me", async (c) => {
|
|
16849
|
-
const token =
|
|
17914
|
+
const token = getCookie2(c, "session");
|
|
16850
17915
|
if (!token) return c.json({ authenticated: false });
|
|
16851
17916
|
const userId = verifySession(token);
|
|
16852
17917
|
if (!userId) {
|
|
16853
|
-
deleteCookie(c, "session", { path: "/", secure:
|
|
17918
|
+
deleteCookie(c, "session", { path: "/", secure: secureCookies2, sameSite: "Strict" });
|
|
16854
17919
|
return c.json({ authenticated: false });
|
|
16855
17920
|
}
|
|
16856
17921
|
const foundUser = await getUserById(userId);
|
|
16857
17922
|
if (!foundUser) {
|
|
16858
|
-
deleteCookie(c, "session", { path: "/", secure:
|
|
17923
|
+
deleteCookie(c, "session", { path: "/", secure: secureCookies2, sameSite: "Strict" });
|
|
16859
17924
|
return c.json({ authenticated: false });
|
|
16860
17925
|
}
|
|
16861
17926
|
const refreshed = await applyMonthlyFreeRefresh(foundUser);
|
|
@@ -16874,6 +17939,8 @@ app.get("/me", async (c) => {
|
|
|
16874
17939
|
has_concurrency_sub: !!user.concurrency_stripe_sub_id,
|
|
16875
17940
|
subscription_tier: user.subscription_tier,
|
|
16876
17941
|
subscription_concurrency: user.subscription_concurrency,
|
|
17942
|
+
memory_plan: user.memory_plan ?? "free",
|
|
17943
|
+
has_memory_sub: !!user.memory_subscription_id,
|
|
16877
17944
|
...stats
|
|
16878
17945
|
});
|
|
16879
17946
|
});
|
|
@@ -16887,6 +17954,227 @@ app.delete("/api-key", requireAllowedOrigin, sessionAuth, async (c) => {
|
|
|
16887
17954
|
await revokeApiKey(user.id);
|
|
16888
17955
|
return c.json({ ok: true });
|
|
16889
17956
|
});
|
|
17957
|
+
function maybeProvisionInBackground(user, vaultCount) {
|
|
17958
|
+
if (vaultCount > 0 || user.memory_provisioned === 1) return;
|
|
17959
|
+
void (async () => {
|
|
17960
|
+
try {
|
|
17961
|
+
if (await getMemoryProvisioned(user.id)) return;
|
|
17962
|
+
await provisionMemoryForUser(user);
|
|
17963
|
+
} catch (err) {
|
|
17964
|
+
console.warn("[memory] background provision failed:", err instanceof Error ? err.message : String(err));
|
|
17965
|
+
}
|
|
17966
|
+
})();
|
|
17967
|
+
}
|
|
17968
|
+
app.get("/memory/overview", auth2, async (c) => {
|
|
17969
|
+
const user = c.get("user");
|
|
17970
|
+
const identity = resolveMemoryIdentity(user);
|
|
17971
|
+
const plan = memoryPlanForReads(user);
|
|
17972
|
+
const requestedVault = c.req.query("vault")?.trim() || null;
|
|
17973
|
+
try {
|
|
17974
|
+
const [vaults, storage, usage] = await Promise.all([
|
|
17975
|
+
dbListVaults(identity),
|
|
17976
|
+
dbStorage(identity, plan),
|
|
17977
|
+
dbUsage(identity, plan)
|
|
17978
|
+
]);
|
|
17979
|
+
maybeProvisionInBackground(user, vaults.length);
|
|
17980
|
+
const activeVault = requestedVault && vaults.some((v) => v.vault === requestedVault) ? requestedVault : vaults[0]?.vault ?? null;
|
|
17981
|
+
const notes = activeVault ? await dbNotes(identity, activeVault) : [];
|
|
17982
|
+
const billing = {
|
|
17983
|
+
ok: true,
|
|
17984
|
+
memory_plan: user.memory_plan ?? "free",
|
|
17985
|
+
has_memory_sub: !!user.memory_subscription_id,
|
|
17986
|
+
free_quota_gb: MEMORY_PLAN_QUOTA.free / 1e9,
|
|
17987
|
+
plans: Object.values(MEMORY_PLANS).map((p) => ({ plan: p.plan, label: p.label, price_id: p.price_id, interval: p.interval, monthly_usd: p.monthly_usd, quota_gb: p.quota_bytes / 1e9 }))
|
|
17988
|
+
};
|
|
17989
|
+
return c.json({ ok: true, vaults, storage, activeVault, notes, usage, billing });
|
|
17990
|
+
} catch (err) {
|
|
17991
|
+
const message = err instanceof Error ? err.message : "memory overview failed";
|
|
17992
|
+
console.error("[memory/overview]", message);
|
|
17993
|
+
return c.json({ ok: false, error: message }, 502);
|
|
17994
|
+
}
|
|
17995
|
+
});
|
|
17996
|
+
app.get("/memory/vaults", auth2, async (c) => {
|
|
17997
|
+
const user = c.get("user");
|
|
17998
|
+
try {
|
|
17999
|
+
const vaults = await dbListVaults(resolveMemoryIdentity(user));
|
|
18000
|
+
maybeProvisionInBackground(user, vaults.length);
|
|
18001
|
+
return c.json({ ok: true, vaults });
|
|
18002
|
+
} catch (err) {
|
|
18003
|
+
return c.json({ ok: false, error: err instanceof Error ? err.message : "could not load vaults" }, 502);
|
|
18004
|
+
}
|
|
18005
|
+
});
|
|
18006
|
+
app.get("/memory/connect", auth2, async (c) => {
|
|
18007
|
+
const { key, error } = await getOrCreateUserMemoryKey(c.get("user"));
|
|
18008
|
+
if (!key) return c.json({ error: error ?? "memory unavailable" }, 502);
|
|
18009
|
+
const command = `MCP_MEMORY_KEY=${key} npx -y -p @vilovieta/mcp-memory@latest mcp-memory`;
|
|
18010
|
+
const config = {
|
|
18011
|
+
mcpServers: {
|
|
18012
|
+
"mcp-memory": {
|
|
18013
|
+
command: "npx",
|
|
18014
|
+
args: ["-y", "-p", "@vilovieta/mcp-memory@latest", "mcp-memory"],
|
|
18015
|
+
env: { MCP_MEMORY_KEY: key }
|
|
18016
|
+
}
|
|
18017
|
+
}
|
|
18018
|
+
};
|
|
18019
|
+
return c.json({ ok: true, command, config });
|
|
18020
|
+
});
|
|
18021
|
+
app.post("/memory/vault", auth2, async (c) => {
|
|
18022
|
+
const { key, error } = await getOrCreateUserMemoryKey(c.get("user"));
|
|
18023
|
+
if (!key) return c.json({ error: error ?? "memory unavailable" }, 502);
|
|
18024
|
+
const body = await c.req.json().catch(() => ({}));
|
|
18025
|
+
const vault = body.vault?.trim();
|
|
18026
|
+
if (!vault) return c.json({ error: "vault name required" }, 400);
|
|
18027
|
+
const result = await memoryCall("addVaultTool", { vault }, key);
|
|
18028
|
+
return c.json(result, result.ok ? 200 : 502);
|
|
18029
|
+
});
|
|
18030
|
+
app.get("/memory/notes", auth2, async (c) => {
|
|
18031
|
+
const user = c.get("user");
|
|
18032
|
+
const identity = resolveMemoryIdentity(user);
|
|
18033
|
+
try {
|
|
18034
|
+
let vault = c.req.query("vault")?.trim() || null;
|
|
18035
|
+
if (!vault) {
|
|
18036
|
+
const vaults = await dbListVaults(identity);
|
|
18037
|
+
vault = vaults[0]?.vault ?? null;
|
|
18038
|
+
}
|
|
18039
|
+
if (!vault) return c.json({ ok: true, notes: [], vault: null });
|
|
18040
|
+
const notes = await dbNotes(identity, vault);
|
|
18041
|
+
return c.json({ ok: true, vault, notes });
|
|
18042
|
+
} catch (err) {
|
|
18043
|
+
return c.json({ ok: false, error: err instanceof Error ? err.message : "could not load notes" }, 502);
|
|
18044
|
+
}
|
|
18045
|
+
});
|
|
18046
|
+
app.get("/memory/note", auth2, async (c) => {
|
|
18047
|
+
const user = c.get("user");
|
|
18048
|
+
const vault = c.req.query("vault")?.trim();
|
|
18049
|
+
const path5 = c.req.query("path");
|
|
18050
|
+
if (!path5) return c.json({ error: "path required" }, 400);
|
|
18051
|
+
if (!vault) return c.json({ ok: false, error: "vault required" }, 400);
|
|
18052
|
+
try {
|
|
18053
|
+
const note = await dbNote(resolveMemoryIdentity(user), vault, path5);
|
|
18054
|
+
if (!note) return c.json({ ok: false, error: "note not found" }, 404);
|
|
18055
|
+
return c.json({ ok: true, note });
|
|
18056
|
+
} catch (err) {
|
|
18057
|
+
return c.json({ ok: false, error: err instanceof Error ? err.message : "could not load note" }, 502);
|
|
18058
|
+
}
|
|
18059
|
+
});
|
|
18060
|
+
app.put("/memory/note", auth2, async (c) => {
|
|
18061
|
+
const { key, error } = await getOrCreateUserMemoryKey(c.get("user"));
|
|
18062
|
+
if (!key) return c.json({ error: error ?? "memory unavailable" }, 502);
|
|
18063
|
+
const body = await c.req.json().catch(() => ({}));
|
|
18064
|
+
if (!body.path) return c.json({ error: "path required" }, 400);
|
|
18065
|
+
const result = await memoryCall("putTool", { vault: body.vault, path: body.path, title: body.title, content: body.content }, key);
|
|
18066
|
+
return c.json(result, result.ok ? 200 : 502);
|
|
18067
|
+
});
|
|
18068
|
+
app.delete("/memory/note", auth2, async (c) => {
|
|
18069
|
+
const { key, error } = await getOrCreateUserMemoryKey(c.get("user"));
|
|
18070
|
+
if (!key) return c.json({ error: error ?? "memory unavailable" }, 502);
|
|
18071
|
+
const vault = c.req.query("vault");
|
|
18072
|
+
const path5 = c.req.query("path");
|
|
18073
|
+
if (!path5) return c.json({ error: "path required" }, 400);
|
|
18074
|
+
const result = await memoryCall("deleteNoteTool", { vault, path: path5 }, key);
|
|
18075
|
+
return c.json(result, result.ok ? 200 : 502);
|
|
18076
|
+
});
|
|
18077
|
+
app.get("/memory/note/download", auth2, async (c) => {
|
|
18078
|
+
const user = c.get("user");
|
|
18079
|
+
const vault = c.req.query("vault")?.trim();
|
|
18080
|
+
const path5 = c.req.query("path");
|
|
18081
|
+
if (!path5) return c.json({ error: "path required" }, 400);
|
|
18082
|
+
if (!vault) return c.json({ error: "vault required" }, 400);
|
|
18083
|
+
const note = await dbNote(resolveMemoryIdentity(user), vault, path5);
|
|
18084
|
+
if (!note) return c.json({ error: "note not found" }, 404);
|
|
18085
|
+
const filename = (path5.split("/").pop() || "note").replace(/"/g, "");
|
|
18086
|
+
return new Response(note.content ?? "", {
|
|
18087
|
+
status: 200,
|
|
18088
|
+
headers: {
|
|
18089
|
+
"Content-Type": "text/markdown; charset=utf-8",
|
|
18090
|
+
"Content-Disposition": `attachment; filename="${filename.endsWith(".md") ? filename : filename + ".md"}"`
|
|
18091
|
+
}
|
|
18092
|
+
});
|
|
18093
|
+
});
|
|
18094
|
+
app.get("/memory/storage", auth2, async (c) => {
|
|
18095
|
+
const user = c.get("user");
|
|
18096
|
+
try {
|
|
18097
|
+
const result = await dbStorage(resolveMemoryIdentity(user), memoryPlanForReads(user));
|
|
18098
|
+
return c.json(result);
|
|
18099
|
+
} catch (err) {
|
|
18100
|
+
return c.json({ ok: false, error: err instanceof Error ? err.message : "could not load storage" }, 502);
|
|
18101
|
+
}
|
|
18102
|
+
});
|
|
18103
|
+
app.get("/memory/billing", auth2, async (c) => {
|
|
18104
|
+
const user = c.get("user");
|
|
18105
|
+
const plans = Object.values(MEMORY_PLANS).map((p) => ({ plan: p.plan, label: p.label, price_id: p.price_id, interval: p.interval, monthly_usd: p.monthly_usd, quota_gb: p.quota_bytes / 1e9 }));
|
|
18106
|
+
return c.json({
|
|
18107
|
+
ok: true,
|
|
18108
|
+
memory_plan: user.memory_plan ?? "free",
|
|
18109
|
+
has_memory_sub: !!user.memory_subscription_id,
|
|
18110
|
+
free_quota_gb: MEMORY_PLAN_QUOTA.free / 1e9,
|
|
18111
|
+
plans
|
|
18112
|
+
});
|
|
18113
|
+
});
|
|
18114
|
+
app.post("/memory/checkout", auth2, async (c) => {
|
|
18115
|
+
try {
|
|
18116
|
+
const user = c.get("user");
|
|
18117
|
+
const body = await c.req.json().catch(() => ({}));
|
|
18118
|
+
const priceId = body.priceId?.trim();
|
|
18119
|
+
if (!priceId || !MEMORY_PLANS[priceId]) return c.json({ error: "Invalid priceId \u2014 must be a memory plan price." }, 400);
|
|
18120
|
+
const secret2 = process.env.STRIPE_SECRET_KEY?.trim();
|
|
18121
|
+
if (!secret2) return c.json({ error: "Stripe is not configured." }, 503);
|
|
18122
|
+
const stripeClient = new Stripe2(secret2, { apiVersion: STRIPE_API_VERSION });
|
|
18123
|
+
let customerId = user.stripe_customer_id;
|
|
18124
|
+
if (!customerId) {
|
|
18125
|
+
const customer = await stripeClient.customers.create({ email: user.email });
|
|
18126
|
+
customerId = customer.id;
|
|
18127
|
+
await setStripeCustomerId(user.id, customerId);
|
|
18128
|
+
}
|
|
18129
|
+
const session = await stripeClient.checkout.sessions.create({
|
|
18130
|
+
mode: "subscription",
|
|
18131
|
+
customer: customerId,
|
|
18132
|
+
line_items: [{ price: priceId, quantity: 1 }],
|
|
18133
|
+
client_reference_id: String(user.id),
|
|
18134
|
+
metadata: { product: "mcp-memory", userId: String(user.id) },
|
|
18135
|
+
subscription_data: { metadata: { product: "mcp-memory", userId: String(user.id) } },
|
|
18136
|
+
automatic_tax: { enabled: true },
|
|
18137
|
+
billing_address_collection: "required",
|
|
18138
|
+
customer_update: { address: "auto", name: "auto" },
|
|
18139
|
+
tax_id_collection: { enabled: true },
|
|
18140
|
+
success_url: `${appOrigin()}/memory?billing=success`,
|
|
18141
|
+
cancel_url: `${appOrigin()}/memory?billing=cancel`
|
|
18142
|
+
});
|
|
18143
|
+
if (!session.url) return c.json({ error: "Stripe did not return a checkout URL." }, 502);
|
|
18144
|
+
return c.json({ url: session.url });
|
|
18145
|
+
} catch (err) {
|
|
18146
|
+
const message = err instanceof Error ? err.message : "Unable to start memory checkout.";
|
|
18147
|
+
console.error("[memory/checkout]", message);
|
|
18148
|
+
return c.json({ error: message }, 500);
|
|
18149
|
+
}
|
|
18150
|
+
});
|
|
18151
|
+
app.post("/memory/portal", auth2, async (c) => {
|
|
18152
|
+
try {
|
|
18153
|
+
const user = c.get("user");
|
|
18154
|
+
if (!user.stripe_customer_id) return c.json({ error: "No billing account yet \u2014 subscribe first." }, 409);
|
|
18155
|
+
const secret2 = process.env.STRIPE_SECRET_KEY?.trim();
|
|
18156
|
+
if (!secret2) return c.json({ error: "Stripe is not configured." }, 503);
|
|
18157
|
+
const stripeClient = new Stripe2(secret2, { apiVersion: STRIPE_API_VERSION });
|
|
18158
|
+
const session = await stripeClient.billingPortal.sessions.create({
|
|
18159
|
+
customer: user.stripe_customer_id,
|
|
18160
|
+
return_url: `${appOrigin()}/memory`
|
|
18161
|
+
});
|
|
18162
|
+
return c.json({ url: session.url });
|
|
18163
|
+
} catch (err) {
|
|
18164
|
+
const message = err instanceof Error ? err.message : "Unable to open billing portal.";
|
|
18165
|
+
console.error("[memory/portal]", message);
|
|
18166
|
+
return c.json({ error: message }, 500);
|
|
18167
|
+
}
|
|
18168
|
+
});
|
|
18169
|
+
app.get("/memory/usage", auth2, async (c) => {
|
|
18170
|
+
const user = c.get("user");
|
|
18171
|
+
try {
|
|
18172
|
+
const result = await dbUsage(resolveMemoryIdentity(user), memoryPlanForReads(user));
|
|
18173
|
+
return c.json(result);
|
|
18174
|
+
} catch (err) {
|
|
18175
|
+
return c.json({ ok: false, error: err instanceof Error ? err.message : "could not load usage" }, 502);
|
|
18176
|
+
}
|
|
18177
|
+
});
|
|
16890
18178
|
var SYNC_HARVEST_TIMEOUT_OVERRIDE_MS = (() => {
|
|
16891
18179
|
const raw = process.env.SYNC_HARVEST_TIMEOUT_MS;
|
|
16892
18180
|
const parsed = raw === void 0 ? NaN : Number(raw);
|
|
@@ -17013,6 +18301,7 @@ app.post("/harvest/sync", auth2, async (c) => {
|
|
|
17013
18301
|
format: "json",
|
|
17014
18302
|
outputDir: "/tmp/paa-output-api",
|
|
17015
18303
|
signal: syncSignal,
|
|
18304
|
+
softDeadlineMs: Date.now() + Math.floor(syncTimeoutMs * 0.8),
|
|
17016
18305
|
onAttemptEvent: recordAttempt
|
|
17017
18306
|
});
|
|
17018
18307
|
await completeJob(jobId, result);
|
|
@@ -17165,8 +18454,14 @@ app.post("/extract-url", auth2, async (c) => {
|
|
|
17165
18454
|
screenshotMeta = { base64: screenshotBuf.toString("base64"), sizeBytes: screenshotBuf.length, device };
|
|
17166
18455
|
}
|
|
17167
18456
|
const mediaMeta = downloadMedia ? await harvestPageMedia(result.bodyHtml, canonicalUrl, { types: mediaTypes ?? ["image", "video", "audio"], outputDir: null }) : null;
|
|
18457
|
+
const memory = bodyResult.data.depositToVault ? await persistScrapeBody(user, {
|
|
18458
|
+
title: result.title ?? canonicalUrl,
|
|
18459
|
+
content: result.bodyMarkdown ?? "",
|
|
18460
|
+
source: canonicalUrl,
|
|
18461
|
+
vault: bodyResult.data.vaultName
|
|
18462
|
+
}) : void 0;
|
|
17168
18463
|
await logRequestEvent({ userId: user.id, source: "extract_url", status: "done", query: canonicalUrl, resultCount: result.headings.length, result });
|
|
17169
|
-
return c.json({ ...result, screenshot: screenshotMeta, branding: brandingData, media: mediaMeta });
|
|
18464
|
+
return c.json({ ...result, screenshot: screenshotMeta, branding: brandingData, media: mediaMeta, memory });
|
|
17170
18465
|
} catch (err) {
|
|
17171
18466
|
const msg = err instanceof Error ? err.message : String(err);
|
|
17172
18467
|
if (debited) await creditMc(user.id, MC_COSTS.page_scrape, LedgerOperation.EXTRACT_URL, "failed call");
|
|
@@ -17584,16 +18879,17 @@ app.get("/cron/tick", async (c) => {
|
|
|
17584
18879
|
if (!process.env.CRON_SECRET || secret2 !== `Bearer ${process.env.CRON_SECRET}`) {
|
|
17585
18880
|
return c.json({ error: "Unauthorized" }, 401);
|
|
17586
18881
|
}
|
|
17587
|
-
const { drainQueue } = await import("./worker-
|
|
18882
|
+
const { drainQueue } = await import("./worker-UC6D2756.js");
|
|
17588
18883
|
const budget = { maxJobs: 10, deadlineMs: Date.now() + 28e4 };
|
|
17589
18884
|
const workflowDispatchResult = await dispatchDueWorkflowSchedules(`${new URL(c.req.url).protocol}//${new URL(c.req.url).host}`);
|
|
17590
|
-
const [results, sweepResult, reapResult, expiredResult] = await Promise.all([
|
|
18885
|
+
const [results, sweepResult, reapResult, expiredResult, blobCleanup] = await Promise.all([
|
|
17591
18886
|
drainQueue(budget),
|
|
17592
18887
|
runMonthlyRefreshSweep(),
|
|
17593
18888
|
reapIdleBrowserSessions(120).catch(() => ({ reaped: 0 })),
|
|
17594
|
-
expireOldLots().catch(() => ({ expired_lots: 0, expired_mc: 0 }))
|
|
18889
|
+
expireOldLots().catch(() => ({ expired_lots: 0, expired_mc: 0 })),
|
|
18890
|
+
cleanupExpiredScrapeBlobs().catch(() => ({ deleted: 0, store: "none" }))
|
|
17595
18891
|
]);
|
|
17596
|
-
return c.json({ drained: results.length, results, sweepResult, reaped: reapResult, expired: expiredResult, workflowDispatch: workflowDispatchResult });
|
|
18892
|
+
return c.json({ drained: results.length, results, sweepResult, reaped: reapResult, expired: expiredResult, blobCleanup, workflowDispatch: workflowDispatchResult });
|
|
17597
18893
|
});
|
|
17598
18894
|
app.on(["GET", "POST", "PUT"], "/api/inngest", serveInngest({ client: inngest, functions: [siteAuditFn, siteExtractFn] }));
|
|
17599
18895
|
app.route("/api/internal/site-architecture-auditor", siteAuditApp);
|
|
@@ -17601,6 +18897,7 @@ app.route("/youtube", youtubeApp);
|
|
|
17601
18897
|
app.route("/screenshot", screenshotApp);
|
|
17602
18898
|
app.route("/facebook", facebookAdApp);
|
|
17603
18899
|
app.route("/instagram", instagramApp);
|
|
18900
|
+
app.route("/reddit", redditApp);
|
|
17604
18901
|
app.route("/maps", mapsApp);
|
|
17605
18902
|
app.route("/directory", directoryApp);
|
|
17606
18903
|
app.route("/workflows", workflowApp);
|
|
@@ -17623,6 +18920,7 @@ app.get("/console/auth/:id/go", async (c) => {
|
|
|
17623
18920
|
});
|
|
17624
18921
|
app.get("/console/:id", (c) => c.html(renderConsoleHtml(c.req.param("id"))));
|
|
17625
18922
|
app.route("/stripe", stripeApp);
|
|
18923
|
+
app.route("/", oauthApp);
|
|
17626
18924
|
if (!process.env.INNGEST_EVENT_KEY) {
|
|
17627
18925
|
startSiteAuditWorker();
|
|
17628
18926
|
}
|
|
@@ -17729,4 +19027,4 @@ app.get("/blog/:slug/", (c) => {
|
|
|
17729
19027
|
export {
|
|
17730
19028
|
app
|
|
17731
19029
|
};
|
|
17732
|
-
//# sourceMappingURL=server-
|
|
19030
|
+
//# sourceMappingURL=server-RRCKVBHG.js.map
|