mcp-scraper 0.3.24 → 0.3.26
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bin/api-server.cjs +8220 -5991
- package/dist/bin/api-server.cjs.map +1 -1
- package/dist/bin/api-server.js +3 -3
- package/dist/bin/mcp-scraper-cli.cjs +320 -58
- package/dist/bin/mcp-scraper-cli.cjs.map +1 -1
- package/dist/bin/mcp-scraper-cli.js +4 -3
- package/dist/bin/mcp-scraper-cli.js.map +1 -1
- package/dist/bin/mcp-scraper-install.cjs +3 -3
- package/dist/bin/mcp-scraper-install.cjs.map +1 -1
- package/dist/bin/mcp-scraper-install.js +2 -2
- package/dist/bin/mcp-stdio-server.cjs +186 -49
- package/dist/bin/mcp-stdio-server.cjs.map +1 -1
- package/dist/bin/mcp-stdio-server.js +5 -5
- package/dist/bin/paa-harvest.cjs +284 -10
- package/dist/bin/paa-harvest.cjs.map +1 -1
- package/dist/bin/paa-harvest.js +5 -3
- package/dist/bin/paa-harvest.js.map +1 -1
- package/dist/{chunk-GZ4W72WF.js → chunk-2GHI2D6Q.js} +184 -48
- package/dist/chunk-2GHI2D6Q.js.map +1 -0
- package/dist/{chunk-ZAP7MYVO.js → chunk-3PRO376E.js} +2 -3
- package/dist/chunk-3PRO376E.js.map +1 -0
- package/dist/{chunk-LFATOGDF.js → chunk-BP27CZ5Q.js} +4 -3
- package/dist/chunk-BP27CZ5Q.js.map +1 -0
- package/dist/{chunk-DE7WP4UU.js → chunk-DOBQN3EY.js} +3 -3
- package/dist/chunk-DOBQN3EY.js.map +1 -0
- package/dist/{chunk-IGOEAKC7.js → chunk-G7PQ64LM.js} +274 -9
- package/dist/chunk-G7PQ64LM.js.map +1 -0
- package/dist/chunk-K443GQY5.js +24 -0
- package/dist/chunk-K443GQY5.js.map +1 -0
- package/dist/chunk-MS2UZ6HR.js +7 -0
- package/dist/chunk-MS2UZ6HR.js.map +1 -0
- package/dist/{chunk-H2R232HK.js → chunk-NNEIXK5L.js} +307 -62
- package/dist/chunk-NNEIXK5L.js.map +1 -0
- package/dist/{chunk-3KYRG7O7.js → chunk-TRNSXR46.js} +185 -2
- package/dist/chunk-TRNSXR46.js.map +1 -0
- package/dist/{chunk-JTWQGPKF.js → chunk-WYCER2HW.js} +33 -10
- package/dist/chunk-WYCER2HW.js.map +1 -0
- package/dist/{db-EG5ETPTY.js → db-FP2ABUU4.js} +32 -2
- package/dist/index.cjs +284 -10
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +7 -3
- package/dist/index.d.ts +7 -3
- package/dist/index.js +5 -3
- package/dist/index.js.map +1 -1
- package/dist/{server-2SRARAO5.js → server-HIGHKTEV.js} +1563 -275
- package/dist/server-HIGHKTEV.js.map +1 -0
- package/dist/{worker-UM5UM6O5.js → worker-UC6D2756.js} +7 -6
- package/dist/{worker-UM5UM6O5.js.map → worker-UC6D2756.js.map} +1 -1
- package/docs/kernel-proxy-future-enhancements.md +80 -0
- package/docs/spec-browser-routing.md +108 -0
- package/docs/spec-mcp-tools.md +146 -0
- package/docs/spec-server-instructions.md +94 -0
- package/docs/specs/audit-visual-demo.md +309 -0
- package/package.json +5 -2
- package/dist/chunk-3KYRG7O7.js.map +0 -1
- package/dist/chunk-DE7WP4UU.js.map +0 -1
- package/dist/chunk-GZ4W72WF.js.map +0 -1
- package/dist/chunk-H2R232HK.js.map +0 -1
- package/dist/chunk-H7R4PPMG.js +0 -7
- package/dist/chunk-H7R4PPMG.js.map +0 -1
- package/dist/chunk-IGOEAKC7.js.map +0 -1
- package/dist/chunk-JTWQGPKF.js.map +0 -1
- package/dist/chunk-LFATOGDF.js.map +0 -1
- package/dist/chunk-ZAP7MYVO.js.map +0 -1
- package/dist/server-2SRARAO5.js.map +0 -1
- /package/dist/{db-EG5ETPTY.js.map → db-FP2ABUU4.js.map} +0 -0
|
@@ -21,26 +21,30 @@ import {
|
|
|
21
21
|
renderLinkReport,
|
|
22
22
|
sanitizeAttempts,
|
|
23
23
|
sanitizeHarvestResult
|
|
24
|
-
} from "./chunk-
|
|
24
|
+
} from "./chunk-2GHI2D6Q.js";
|
|
25
25
|
import {
|
|
26
26
|
csvRecords,
|
|
27
27
|
listWorkflowDefinitions,
|
|
28
28
|
rowsToCsv,
|
|
29
29
|
runWorkflow,
|
|
30
30
|
runWorkflowStep,
|
|
31
|
+
slugify,
|
|
31
32
|
workflowDefinition,
|
|
32
33
|
workflowOutputBaseDir,
|
|
33
34
|
workflowStepCount,
|
|
34
35
|
workflowSupportsSteps
|
|
35
|
-
} from "./chunk-
|
|
36
|
-
import "./chunk-
|
|
36
|
+
} from "./chunk-NNEIXK5L.js";
|
|
37
|
+
import "./chunk-MS2UZ6HR.js";
|
|
37
38
|
import {
|
|
38
39
|
BROWSER_OPEN_MIN_BALANCE_MC,
|
|
39
40
|
CONCURRENCY_PRICE_ID,
|
|
40
41
|
CREDIT_COST_CATALOG,
|
|
42
|
+
FREE_SIGNUP_MC,
|
|
41
43
|
LedgerOperation,
|
|
42
44
|
MC_COSTS,
|
|
43
45
|
MC_PER_CREDIT,
|
|
46
|
+
MEMORY_PLANS,
|
|
47
|
+
MEMORY_PLAN_QUOTA,
|
|
44
48
|
SUBSCRIPTION_TIERS,
|
|
45
49
|
SUBSCRIPTION_TIER_BY_KEY,
|
|
46
50
|
browserActiveCostMc,
|
|
@@ -50,15 +54,20 @@ import {
|
|
|
50
54
|
harvestProblemResponse,
|
|
51
55
|
insufficientBalanceResponse,
|
|
52
56
|
serializeHarvestProblem
|
|
53
|
-
} from "./chunk-
|
|
57
|
+
} from "./chunk-WYCER2HW.js";
|
|
54
58
|
import {
|
|
55
59
|
BrowserDriver,
|
|
56
60
|
MapsSelectors,
|
|
57
61
|
buildYouTubeChannelVideosUrl,
|
|
62
|
+
currentCostContext,
|
|
58
63
|
deleteKernelProxyId,
|
|
59
64
|
harvest,
|
|
60
|
-
|
|
61
|
-
|
|
65
|
+
migrateCostTelemetry,
|
|
66
|
+
recordKernelSession,
|
|
67
|
+
recordVendorUsage,
|
|
68
|
+
resolveKernelProxyId,
|
|
69
|
+
runWithCostContext
|
|
70
|
+
} from "./chunk-G7PQ64LM.js";
|
|
62
71
|
import {
|
|
63
72
|
CaptchaError,
|
|
64
73
|
RECAPTCHA_INSTRUCTIONS,
|
|
@@ -67,7 +76,8 @@ import {
|
|
|
67
76
|
browserServiceProfileSaveChanges,
|
|
68
77
|
browserServiceProxyId,
|
|
69
78
|
sanitizeVendorName
|
|
70
|
-
} from "./chunk-
|
|
79
|
+
} from "./chunk-3PRO376E.js";
|
|
80
|
+
import "./chunk-K443GQY5.js";
|
|
71
81
|
import {
|
|
72
82
|
DEFAULT_MAPS_PROXY_MODE,
|
|
73
83
|
DEFAULT_PROXY_MODE,
|
|
@@ -77,7 +87,7 @@ import {
|
|
|
77
87
|
RawMapsHoursRowSchema,
|
|
78
88
|
RawMapsOverviewSchema,
|
|
79
89
|
RawMapsReviewStatsSchema
|
|
80
|
-
} from "./chunk-
|
|
90
|
+
} from "./chunk-BP27CZ5Q.js";
|
|
81
91
|
import {
|
|
82
92
|
SiteAuditJobRowSchema,
|
|
83
93
|
advanceWorkflowRunStep,
|
|
@@ -86,6 +96,7 @@ import {
|
|
|
86
96
|
checkRateLimit,
|
|
87
97
|
completeJob,
|
|
88
98
|
completeWorkflowRunRecord,
|
|
99
|
+
consumeCode,
|
|
89
100
|
consumePasswordResetToken,
|
|
90
101
|
countActiveUsers,
|
|
91
102
|
createJob,
|
|
@@ -101,13 +112,18 @@ import {
|
|
|
101
112
|
expireOldLots,
|
|
102
113
|
failJob,
|
|
103
114
|
failWorkflowRunRecord,
|
|
115
|
+
getClient,
|
|
104
116
|
getDb,
|
|
105
117
|
getJob,
|
|
106
118
|
getLedger,
|
|
119
|
+
getMemoryPlan,
|
|
120
|
+
getMemoryProvisioned,
|
|
121
|
+
getRefresh,
|
|
107
122
|
getUserByApiKey,
|
|
108
123
|
getUserByEmail,
|
|
109
124
|
getUserById,
|
|
110
125
|
getUserByStripeCustomerId,
|
|
126
|
+
getUserMemoryCreds,
|
|
111
127
|
getUserStats,
|
|
112
128
|
getWorkflowArtifact,
|
|
113
129
|
getWorkflowRun,
|
|
@@ -127,19 +143,27 @@ import {
|
|
|
127
143
|
markWorkflowRunRunning,
|
|
128
144
|
markWorkflowScheduleRan,
|
|
129
145
|
patchWorkflowSchedule,
|
|
146
|
+
putCode,
|
|
147
|
+
putRefresh,
|
|
130
148
|
reconcileBalanceMc,
|
|
131
149
|
recordStripeEvent,
|
|
132
150
|
recordWorkflowWebhookDelivery,
|
|
151
|
+
registerClient,
|
|
133
152
|
replaceWorkflowArtifacts,
|
|
153
|
+
resolveUserByMemorySubscription,
|
|
134
154
|
revokeApiKey,
|
|
135
155
|
rotateApiKey,
|
|
156
|
+
rotateRefresh,
|
|
136
157
|
setConcurrencySubId,
|
|
137
158
|
setExtraConcurrencySlots,
|
|
159
|
+
setMemoryPlan,
|
|
160
|
+
setMemoryProvisioned,
|
|
138
161
|
setPassword,
|
|
139
162
|
setStripeCustomerId,
|
|
140
163
|
setSubscriptionTier,
|
|
164
|
+
setUserMemoryCreds,
|
|
141
165
|
verifyPassword
|
|
142
|
-
} from "./chunk-
|
|
166
|
+
} from "./chunk-TRNSXR46.js";
|
|
143
167
|
|
|
144
168
|
// src/blog/registry.ts
|
|
145
169
|
var posts = [
|
|
@@ -3511,7 +3535,9 @@ async function fetchWithKernel(url) {
|
|
|
3511
3535
|
const apiKey = browserServiceApiKey();
|
|
3512
3536
|
if (!apiKey) throw new Error("Browser backend API key not set");
|
|
3513
3537
|
const client2 = new Kernel({ apiKey });
|
|
3514
|
-
const
|
|
3538
|
+
const openedAtMs = Date.now();
|
|
3539
|
+
const useHeadless = currentCostContext()?.forceHeadful !== true;
|
|
3540
|
+
const kb = await client2.browsers.create({ stealth: true, ...useHeadless ? { headless: true } : {}, timeout_seconds: 60 });
|
|
3515
3541
|
const browser = await chromium.connectOverCDP(kb.cdp_ws_url);
|
|
3516
3542
|
try {
|
|
3517
3543
|
const context = browser.contexts()[0] ?? await browser.newContext({
|
|
@@ -3526,6 +3552,7 @@ async function fetchWithKernel(url) {
|
|
|
3526
3552
|
});
|
|
3527
3553
|
await client2.browsers.deleteByID(kb.session_id).catch(() => {
|
|
3528
3554
|
});
|
|
3555
|
+
void recordKernelSession({ kernelSessionId: kb.session_id, source: "kernel_fetch", stealth: typeof kb.stealth === "boolean" ? kb.stealth : null, headlessSent: useHeadless ? true : null, proxyUsed: typeof kb.proxy_id === "string" ? Boolean(kb.proxy_id) : null, fallback: true, openedAtMs, closedAtMs: Date.now() });
|
|
3529
3556
|
}
|
|
3530
3557
|
}
|
|
3531
3558
|
|
|
@@ -4127,6 +4154,7 @@ async function captureLocal(url, device, wantScreenshot, wantBranding) {
|
|
|
4127
4154
|
}
|
|
4128
4155
|
async function captureKernel(url, kernelApiKey, device, wantScreenshot, wantBranding) {
|
|
4129
4156
|
const kernel = new Kernel2({ apiKey: kernelApiKey });
|
|
4157
|
+
const openedAtMs = Date.now();
|
|
4130
4158
|
const session = await kernel.browsers.create({ stealth: true, timeout_seconds: 240 });
|
|
4131
4159
|
const browser = await chromium2.connectOverCDP(session.cdp_ws_url);
|
|
4132
4160
|
const vp = device === "mobile" ? MOBILE_VIEWPORT : DESKTOP_VIEWPORT;
|
|
@@ -4146,6 +4174,7 @@ async function captureKernel(url, kernelApiKey, device, wantScreenshot, wantBran
|
|
|
4146
4174
|
return result;
|
|
4147
4175
|
} finally {
|
|
4148
4176
|
await browser.close();
|
|
4177
|
+
void recordKernelSession({ kernelSessionId: session.session_id, source: "screenshot", stealth: typeof session.stealth === "boolean" ? session.stealth : null, headlessSent: null, proxyUsed: typeof session.proxy_id === "string" ? Boolean(session.proxy_id) : null, openedAtMs, closedAtMs: Date.now() });
|
|
4149
4178
|
}
|
|
4150
4179
|
}
|
|
4151
4180
|
async function captureScreenshot(url, kernelApiKey, device = "desktop") {
|
|
@@ -4800,14 +4829,19 @@ async function crawlWithRotation(urls, opts) {
|
|
|
4800
4829
|
let kb = null;
|
|
4801
4830
|
let browser = null;
|
|
4802
4831
|
let inBrowser = 0;
|
|
4832
|
+
let kbOpenedAtMs = 0;
|
|
4803
4833
|
const retire = () => {
|
|
4804
|
-
const b = browser, k = kb;
|
|
4834
|
+
const b = browser, k = kb, openedAtMs = kbOpenedAtMs;
|
|
4805
4835
|
browser = null;
|
|
4806
4836
|
kb = null;
|
|
4837
|
+
kbOpenedAtMs = 0;
|
|
4807
4838
|
if (b) void b.close().catch(() => {
|
|
4808
4839
|
});
|
|
4809
|
-
if (k)
|
|
4810
|
-
|
|
4840
|
+
if (k) {
|
|
4841
|
+
void client2.browsers.deleteByID(k.session_id).catch(() => {
|
|
4842
|
+
});
|
|
4843
|
+
void recordKernelSession({ kernelSessionId: k.session_id, source: "rotating_crawl", stealth: typeof k.stealth === "boolean" ? k.stealth : null, headlessSent: headless, proxyUsed: typeof k.proxy_id === "string" ? Boolean(k.proxy_id) : null, openedAtMs, closedAtMs: Date.now() });
|
|
4844
|
+
}
|
|
4811
4845
|
};
|
|
4812
4846
|
try {
|
|
4813
4847
|
while (true) {
|
|
@@ -4819,6 +4853,7 @@ async function crawlWithRotation(urls, opts) {
|
|
|
4819
4853
|
if (!browser || inBrowser >= urlsPerBrowser) {
|
|
4820
4854
|
retire();
|
|
4821
4855
|
kb = await createBrowserWithRetry(client2, headless);
|
|
4856
|
+
kbOpenedAtMs = Date.now();
|
|
4822
4857
|
browser = await withTimeout(chromium3.connectOverCDP(kb.cdp_ws_url), 3e4);
|
|
4823
4858
|
inBrowser = 0;
|
|
4824
4859
|
}
|
|
@@ -5226,7 +5261,7 @@ async function extractSite(opts) {
|
|
|
5226
5261
|
}
|
|
5227
5262
|
|
|
5228
5263
|
// src/api/server.ts
|
|
5229
|
-
import { Hono as
|
|
5264
|
+
import { Hono as Hono15 } from "hono";
|
|
5230
5265
|
import { serve as serveInngest } from "inngest/hono";
|
|
5231
5266
|
|
|
5232
5267
|
// src/inngest/client.ts
|
|
@@ -5301,6 +5336,7 @@ var DeepInfraLlmClient = class {
|
|
|
5301
5336
|
});
|
|
5302
5337
|
if (!resp.ok) throw new Error(`DeepInfra error ${resp.status}: ${await resp.text()}`);
|
|
5303
5338
|
const data = await resp.json();
|
|
5339
|
+
void recordVendorUsage({ vendor: "deepinfra_qwen", model: this.model, units: data.usage?.completion_tokens ?? 0, unitType: "tokens" });
|
|
5304
5340
|
return data.choices[0].message.content;
|
|
5305
5341
|
}
|
|
5306
5342
|
async completeJson(prompt) {
|
|
@@ -8012,16 +8048,6 @@ var CATALOG = {
|
|
|
8012
8048
|
isSpecial: true,
|
|
8013
8049
|
tagline: "Generate a rank-tracker blueprint",
|
|
8014
8050
|
params: [{ key: "domain", label: "Domain", type: "string", required: true }]
|
|
8015
|
-
},
|
|
8016
|
-
{
|
|
8017
|
-
key: "deep_research_workflow",
|
|
8018
|
-
label: "Deep Research",
|
|
8019
|
-
kind: "simple",
|
|
8020
|
-
endpoint: "/workflows/run",
|
|
8021
|
-
mcpName: "workflow_run",
|
|
8022
|
-
isSpecial: true,
|
|
8023
|
-
tagline: "Multi-source, fact-checked research report",
|
|
8024
|
-
params: [{ key: "question", label: "Question", type: "string", required: true }]
|
|
8025
8051
|
}
|
|
8026
8052
|
]
|
|
8027
8053
|
}
|
|
@@ -8924,6 +8950,8 @@ async function attemptKernelWhisper(videoId, kernelApiKey, kernelProxyId, falKey
|
|
|
8924
8950
|
timestamp: c.timestamp,
|
|
8925
8951
|
text: c.text.trim()
|
|
8926
8952
|
}));
|
|
8953
|
+
const audioSec = chunks.length ? chunks[chunks.length - 1]?.timestamp?.[1] ?? 0 : 0;
|
|
8954
|
+
void recordVendorUsage({ vendor: "fal_wizper", model: "fal-ai/wizper", units: audioSec / 60, unitType: "audio_min" });
|
|
8927
8955
|
const text = data.text ?? chunks.map((c) => c.text).join(" ");
|
|
8928
8956
|
if (!text) return null;
|
|
8929
8957
|
return { videoId, text, chunks, durationMs: Date.now() - start, method: "browser-whisper" };
|
|
@@ -9007,7 +9035,9 @@ var ExtractUrlBodySchema = z11.object({
|
|
|
9007
9035
|
extractBranding: z11.boolean().optional(),
|
|
9008
9036
|
downloadMedia: z11.boolean().optional(),
|
|
9009
9037
|
mediaTypes: z11.array(z11.enum(["image", "video", "audio"])).optional(),
|
|
9010
|
-
allowLocal: z11.boolean().optional()
|
|
9038
|
+
allowLocal: z11.boolean().optional(),
|
|
9039
|
+
depositToVault: z11.boolean().optional(),
|
|
9040
|
+
vaultName: z11.string().trim().min(1).max(120).optional()
|
|
9011
9041
|
});
|
|
9012
9042
|
var MapUrlsBodySchema = z11.object({
|
|
9013
9043
|
url: z11.string().min(1, "url is required"),
|
|
@@ -10351,6 +10381,8 @@ async function transcribeMediaUrl(mediaUrl, markdownTitle = "# Media Transcript"
|
|
|
10351
10381
|
const text = data.text ?? "";
|
|
10352
10382
|
const chunks = data.chunks ?? [];
|
|
10353
10383
|
const durationMs = Date.now() - startMs;
|
|
10384
|
+
const audioSec = chunks.length ? chunks[chunks.length - 1]?.timestamp?.[1] ?? 0 : 0;
|
|
10385
|
+
void recordVendorUsage({ vendor: "fal_wizper", model: "fal-ai/wizper", units: audioSec / 60, unitType: "audio_min" });
|
|
10354
10386
|
return {
|
|
10355
10387
|
text,
|
|
10356
10388
|
chunks,
|
|
@@ -10410,21 +10442,24 @@ function buildPageIntelUrl(body, country) {
|
|
|
10410
10442
|
if (body.pageId?.trim()) return `https://www.facebook.com/ads/library/?active_status=all&ad_type=all&country=${country}&is_targeted_country=false&media_type=all&search_type=page&view_all_page_id=${body.pageId.trim()}`;
|
|
10411
10443
|
return `https://www.facebook.com/ads/library/?active_status=all&ad_type=all&country=${country}&q=${encodeURIComponent(body.query.trim())}&search_type=keyword_unordered`;
|
|
10412
10444
|
}
|
|
10445
|
+
function configuredKernelProxy() {
|
|
10446
|
+
return (process.env.BROWSER_SERVICE_PROXY_ID ?? process.env.KERNEL_PROXY_ID)?.trim() || void 0;
|
|
10447
|
+
}
|
|
10413
10448
|
function kernelLaunchOpts() {
|
|
10414
|
-
return { headless: true, kernelApiKey: browserServiceApiKey(), kernelProxyId:
|
|
10449
|
+
return { headless: true, kernelApiKey: browserServiceApiKey(), kernelProxyId: configuredKernelProxy(), viewport: { width: 1280, height: 900 }, locale: "en-US" };
|
|
10415
10450
|
}
|
|
10416
10451
|
async function kernelLaunchOptsResidential() {
|
|
10417
|
-
let proxyId =
|
|
10452
|
+
let proxyId = configuredKernelProxy();
|
|
10418
10453
|
try {
|
|
10419
10454
|
const resolution = await resolveKernelProxyId({
|
|
10420
10455
|
kernelApiKey: browserServiceApiKey(),
|
|
10421
10456
|
proxyMode: DEFAULT_PROXY_MODE,
|
|
10422
|
-
configuredKernelProxyId:
|
|
10457
|
+
configuredKernelProxyId: configuredKernelProxy(),
|
|
10423
10458
|
gl: "us"
|
|
10424
10459
|
});
|
|
10425
10460
|
if (resolution.kernelProxyId) proxyId = resolution.kernelProxyId;
|
|
10426
10461
|
} catch {
|
|
10427
|
-
proxyId =
|
|
10462
|
+
proxyId = configuredKernelProxy();
|
|
10428
10463
|
}
|
|
10429
10464
|
return { headless: true, kernelApiKey: browserServiceApiKey(), kernelProxyId: proxyId, viewport: { width: 1280, height: 900 }, locale: "en-US" };
|
|
10430
10465
|
}
|
|
@@ -11162,10 +11197,14 @@ function resolveProfileName(body) {
|
|
|
11162
11197
|
const explicit = body.profile?.trim();
|
|
11163
11198
|
return explicit || browserServiceProfileName();
|
|
11164
11199
|
}
|
|
11200
|
+
function configuredKernelProxy2() {
|
|
11201
|
+
return (process.env.BROWSER_SERVICE_PROXY_ID ?? process.env.KERNEL_PROXY_ID)?.trim() || void 0;
|
|
11202
|
+
}
|
|
11165
11203
|
async function kernelLaunchOptsDirect(profileName, saveProfileChanges) {
|
|
11166
11204
|
return {
|
|
11167
11205
|
headless: true,
|
|
11168
11206
|
kernelApiKey: browserServiceApiKey(),
|
|
11207
|
+
kernelProxyId: configuredKernelProxy2(),
|
|
11169
11208
|
...profileName ? { kernelProfileName: profileName } : {},
|
|
11170
11209
|
...typeof saveProfileChanges === "boolean" ? { kernelProfileSaveChanges: saveProfileChanges } : {},
|
|
11171
11210
|
viewport: { width: 1280, height: 900 },
|
|
@@ -11420,8 +11459,136 @@ instagramApp.post("/media-download", createApiKeyAuth(), async (c) => {
|
|
|
11420
11459
|
}
|
|
11421
11460
|
});
|
|
11422
11461
|
|
|
11423
|
-
// src/api/
|
|
11462
|
+
// src/api/reddit-routes.ts
|
|
11424
11463
|
import { Hono as Hono6 } from "hono";
|
|
11464
|
+
import { z as z15 } from "zod";
|
|
11465
|
+
var RedditThreadBodySchema = z15.object({
|
|
11466
|
+
url: z15.string().trim().min(1, "url is required"),
|
|
11467
|
+
maxComments: z15.number().int().min(1).max(2e3).optional()
|
|
11468
|
+
});
|
|
11469
|
+
function invalidRequest3(message) {
|
|
11470
|
+
return { error_code: "invalid_request", message };
|
|
11471
|
+
}
|
|
11472
|
+
function toOldReddit(url) {
|
|
11473
|
+
try {
|
|
11474
|
+
const u = new URL(url.trim());
|
|
11475
|
+
if (!/(^|\.)reddit\.com$/.test(u.hostname.toLowerCase()) && u.hostname.toLowerCase() !== "redd.it") return null;
|
|
11476
|
+
u.hostname = "old.reddit.com";
|
|
11477
|
+
u.protocol = "https:";
|
|
11478
|
+
return u.toString();
|
|
11479
|
+
} catch {
|
|
11480
|
+
return null;
|
|
11481
|
+
}
|
|
11482
|
+
}
|
|
11483
|
+
async function residentialProxyId(attemptIndex) {
|
|
11484
|
+
try {
|
|
11485
|
+
const r = await resolveKernelProxyId({
|
|
11486
|
+
kernelApiKey: browserServiceApiKey(),
|
|
11487
|
+
proxyMode: "location",
|
|
11488
|
+
location: "Austin, TX",
|
|
11489
|
+
gl: "us",
|
|
11490
|
+
attemptIndex,
|
|
11491
|
+
fresh: true
|
|
11492
|
+
});
|
|
11493
|
+
return r.kernelProxyId;
|
|
11494
|
+
} catch {
|
|
11495
|
+
return void 0;
|
|
11496
|
+
}
|
|
11497
|
+
}
|
|
11498
|
+
var PARSE_REDDIT = `(() => {
|
|
11499
|
+
const txt = el => ((el && el.innerText) || '').trim();
|
|
11500
|
+
const attr = (el, a) => ((el && el.getAttribute(a)) || '').trim();
|
|
11501
|
+
const bodyText = (document.body && document.body.innerText) || '';
|
|
11502
|
+
const blocked = /whoa there|blocked by network|you've been blocked|network (policy|security)|log in to your reddit account/i.test(bodyText);
|
|
11503
|
+
const post = document.querySelector('#siteTable .thing.link') || document.querySelector('.thing.link');
|
|
11504
|
+
const title = txt(document.querySelector('a.title')) || (document.title || '');
|
|
11505
|
+
const author = attr(post, 'data-author');
|
|
11506
|
+
const scoreEl = post && post.querySelector('.score.unvoted');
|
|
11507
|
+
const score = scoreEl ? ((scoreEl.getAttribute('title') || scoreEl.textContent || '').trim()) : '';
|
|
11508
|
+
const postBody = txt(post && post.querySelector('.usertext-body .md'));
|
|
11509
|
+
const comments = [];
|
|
11510
|
+
document.querySelectorAll('.commentarea .comment').forEach(c => {
|
|
11511
|
+
const entry = c.querySelector(':scope > .entry');
|
|
11512
|
+
if (!entry) return;
|
|
11513
|
+
const a = attr(c, 'data-author');
|
|
11514
|
+
const b = txt(entry.querySelector('.usertext-body .md'));
|
|
11515
|
+
if (!b) return;
|
|
11516
|
+
const s = ((entry.querySelector('.score.unvoted') || {}).textContent || '').trim();
|
|
11517
|
+
let depth = 0, p = c.parentElement;
|
|
11518
|
+
while (p) { if (p.classList && p.classList.contains('comment')) depth++; p = p.parentElement; }
|
|
11519
|
+
comments.push({ author: a, score: s, depth: depth, body: b });
|
|
11520
|
+
});
|
|
11521
|
+
return { title: title, author: author, score: score, postBody: postBody, blocked: blocked, numComments: comments.length, comments: comments };
|
|
11522
|
+
})()`;
|
|
11523
|
+
var redditApp = new Hono6();
|
|
11524
|
+
redditApp.post("/thread", createApiKeyAuth(), async (c) => {
|
|
11525
|
+
const raw = await c.req.json().catch(() => ({}));
|
|
11526
|
+
const parsed = RedditThreadBodySchema.safeParse(raw);
|
|
11527
|
+
if (!parsed.success) return c.json(invalidRequest3(parsed.error.issues[0]?.message ?? "Invalid request"), 400);
|
|
11528
|
+
const body = parsed.data;
|
|
11529
|
+
const oldUrl = toOldReddit(body.url);
|
|
11530
|
+
if (!oldUrl) return c.json(invalidRequest3("url must be a reddit.com link"), 400);
|
|
11531
|
+
const user = c.get("user");
|
|
11532
|
+
const gate = await acquireConcurrencyGate(user, "reddit_thread", {
|
|
11533
|
+
reuseLockId: c.req.header("x-mcp-scraper-concurrency-lock"),
|
|
11534
|
+
metadata: { url: body.url }
|
|
11535
|
+
});
|
|
11536
|
+
if (!gate.ok) return c.json(concurrencyLimitExceededResponse(gate), 429, { "Retry-After": String(gate.retryAfterSeconds) });
|
|
11537
|
+
let debited = false;
|
|
11538
|
+
let refunded = false;
|
|
11539
|
+
try {
|
|
11540
|
+
const { ok, balance_mc } = await debitMc(user.id, MC_COSTS.reddit_thread, LedgerOperation.REDDIT_THREAD, body.url);
|
|
11541
|
+
if (!ok) return c.json(insufficientBalanceResponse(balance_mc, MC_COSTS.reddit_thread), 402);
|
|
11542
|
+
debited = true;
|
|
11543
|
+
let result = null;
|
|
11544
|
+
for (let attempt = 0; attempt < 3 && !result; attempt++) {
|
|
11545
|
+
const proxyId = await residentialProxyId(attempt);
|
|
11546
|
+
const driver = new BrowserDriver();
|
|
11547
|
+
try {
|
|
11548
|
+
await driver.launch({
|
|
11549
|
+
headless: true,
|
|
11550
|
+
headlessMode: "headful",
|
|
11551
|
+
kernelApiKey: browserServiceApiKey(),
|
|
11552
|
+
kernelProxyId: proxyId,
|
|
11553
|
+
viewport: { width: 1280, height: 900 },
|
|
11554
|
+
locale: "en-US"
|
|
11555
|
+
});
|
|
11556
|
+
const page = driver.getPage();
|
|
11557
|
+
await page.goto(oldUrl, { waitUntil: "domcontentloaded", timeout: 45e3 });
|
|
11558
|
+
await page.waitForTimeout(2500);
|
|
11559
|
+
const data = await page.evaluate(PARSE_REDDIT);
|
|
11560
|
+
if (!data.blocked && (data.postBody || data.comments.length > 0)) {
|
|
11561
|
+
result = { ...data, sourceUrl: body.url, oldRedditUrl: oldUrl, attempts: attempt + 1 };
|
|
11562
|
+
}
|
|
11563
|
+
} catch {
|
|
11564
|
+
} finally {
|
|
11565
|
+
await driver.close();
|
|
11566
|
+
}
|
|
11567
|
+
}
|
|
11568
|
+
if (!result) {
|
|
11569
|
+
await creditMc(user.id, MC_COSTS.reddit_thread, LedgerOperation.REDDIT_THREAD_REFUND, "reddit blocked/empty after retries");
|
|
11570
|
+
refunded = true;
|
|
11571
|
+
await logRequestEvent({ userId: user.id, source: "reddit_thread", status: "failed", query: body.url, error: "reddit blocked/empty after retries" });
|
|
11572
|
+
return c.json({ error: "Reddit blocked the request after retries (refunded)" }, 503);
|
|
11573
|
+
}
|
|
11574
|
+
if (body.maxComments && Array.isArray(result.comments)) {
|
|
11575
|
+
result.comments = result.comments.slice(0, body.maxComments);
|
|
11576
|
+
result.numComments = result.comments.length;
|
|
11577
|
+
}
|
|
11578
|
+
await logRequestEvent({ userId: user.id, source: "reddit_thread", status: "done", query: body.url, resultCount: Number(result.numComments ?? 0), result });
|
|
11579
|
+
return c.json(result);
|
|
11580
|
+
} catch (err) {
|
|
11581
|
+
if (debited && !refunded) await creditMc(user.id, MC_COSTS.reddit_thread, LedgerOperation.REDDIT_THREAD_REFUND, "failed call");
|
|
11582
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
11583
|
+
await logRequestEvent({ userId: user.id, source: "reddit_thread", status: "failed", query: body.url, error: msg });
|
|
11584
|
+
return c.json({ error: msg }, 500);
|
|
11585
|
+
} finally {
|
|
11586
|
+
await releaseConcurrencyGate(gate.lockId);
|
|
11587
|
+
}
|
|
11588
|
+
});
|
|
11589
|
+
|
|
11590
|
+
// src/api/maps-routes.ts
|
|
11591
|
+
import { Hono as Hono7 } from "hono";
|
|
11425
11592
|
|
|
11426
11593
|
// src/extractor/MapsNavigator.ts
|
|
11427
11594
|
var MapsNavigator = class {
|
|
@@ -11596,7 +11763,7 @@ var MapsReviewCollector = class {
|
|
|
11596
11763
|
};
|
|
11597
11764
|
|
|
11598
11765
|
// src/extractor/MapsExtractor.ts
|
|
11599
|
-
import { z as
|
|
11766
|
+
import { z as z16 } from "zod";
|
|
11600
11767
|
var MapsExtractor = class {
|
|
11601
11768
|
constructor(driver) {
|
|
11602
11769
|
this.driver = driver;
|
|
@@ -11782,7 +11949,7 @@ var MapsExtractor = class {
|
|
|
11782
11949
|
});
|
|
11783
11950
|
return rows;
|
|
11784
11951
|
}, { hoursTable: MapsSelectors.hoursTable, hoursTableAlt: MapsSelectors.hoursTableAlt });
|
|
11785
|
-
const result =
|
|
11952
|
+
const result = z16.array(RawMapsHoursRowSchema).safeParse(raw);
|
|
11786
11953
|
if (!result.success) {
|
|
11787
11954
|
console.warn("[MapsExtractor] hours parse failed", result.error.flatten());
|
|
11788
11955
|
return [];
|
|
@@ -11846,7 +12013,7 @@ var MapsExtractor = class {
|
|
|
11846
12013
|
});
|
|
11847
12014
|
return results;
|
|
11848
12015
|
});
|
|
11849
|
-
const result =
|
|
12016
|
+
const result = z16.array(RawMapsAboutAttributeSchema).safeParse(raw);
|
|
11850
12017
|
if (!result.success) {
|
|
11851
12018
|
console.warn("[MapsExtractor] about parse failed", result.error.flatten());
|
|
11852
12019
|
return [];
|
|
@@ -12107,7 +12274,7 @@ async function cleanupDisposableProxy(kernelApiKey, proxyId, eventName) {
|
|
|
12107
12274
|
}
|
|
12108
12275
|
async function runMapsSearchWithRotation(options) {
|
|
12109
12276
|
const attempts = [];
|
|
12110
|
-
const maxAttempts = mapsSearchMaxAttemptsForProxyMode(options.proxyMode);
|
|
12277
|
+
const maxAttempts = mapsSearchMaxAttemptsForProxyMode(options.proxyMode) + 1;
|
|
12111
12278
|
const started = Date.now();
|
|
12112
12279
|
let lastError = null;
|
|
12113
12280
|
for (let attemptIndex = 0; attemptIndex < maxAttempts; attemptIndex += 1) {
|
|
@@ -12129,12 +12296,15 @@ async function runMapsSearchWithRotation(options) {
|
|
|
12129
12296
|
fresh: options.proxyMode === "location"
|
|
12130
12297
|
});
|
|
12131
12298
|
disposableProxyId = resolution.disposableProxyId;
|
|
12132
|
-
const
|
|
12299
|
+
const extractArgs = {
|
|
12133
12300
|
...options,
|
|
12134
12301
|
proxyZip,
|
|
12135
12302
|
kernelProxyId: options.proxyMode === "none" ? void 0 : resolution.kernelProxyId,
|
|
12136
12303
|
kernelProxyResolution: resolution.resolution
|
|
12137
|
-
}
|
|
12304
|
+
};
|
|
12305
|
+
const isHeadfulEscalation = attemptIndex === maxAttempts - 1;
|
|
12306
|
+
const baseCtx = currentCostContext();
|
|
12307
|
+
const result = isHeadfulEscalation && baseCtx ? await runWithCostContext({ ...baseCtx, forceHeadful: true }, () => extractor.extract(extractArgs)) : await extractor.extract(extractArgs);
|
|
12138
12308
|
const attempt = attemptFromDebug({
|
|
12139
12309
|
attemptIndex,
|
|
12140
12310
|
maxAttempts,
|
|
@@ -12219,7 +12389,7 @@ function mapsErrorResponse(c, err, errorCode) {
|
|
|
12219
12389
|
attempts: rotationError?.attempts ?? void 0
|
|
12220
12390
|
}, retryable ? 503 : 500);
|
|
12221
12391
|
}
|
|
12222
|
-
var mapsApp = new
|
|
12392
|
+
var mapsApp = new Hono7();
|
|
12223
12393
|
mapsApp.post("/search", createApiKeyAuth(), async (c) => {
|
|
12224
12394
|
const user = c.get("user");
|
|
12225
12395
|
const body = await c.req.json().catch(() => ({}));
|
|
@@ -12357,12 +12527,12 @@ mapsApp.post("/place", createApiKeyAuth(), async (c) => {
|
|
|
12357
12527
|
});
|
|
12358
12528
|
|
|
12359
12529
|
// src/api/directory-routes.ts
|
|
12360
|
-
import { Hono as
|
|
12530
|
+
import { Hono as Hono8 } from "hono";
|
|
12361
12531
|
|
|
12362
12532
|
// src/directory/directory-workflow.ts
|
|
12363
12533
|
import { mkdir as mkdir2, writeFile } from "fs/promises";
|
|
12364
12534
|
import { join as join6 } from "path";
|
|
12365
|
-
import { z as
|
|
12535
|
+
import { z as z17 } from "zod";
|
|
12366
12536
|
|
|
12367
12537
|
// src/directory/location-db.ts
|
|
12368
12538
|
import { access, readFile } from "fs/promises";
|
|
@@ -12523,24 +12693,24 @@ async function resolveDirectoryMarkets(options) {
|
|
|
12523
12693
|
}
|
|
12524
12694
|
|
|
12525
12695
|
// src/directory/directory-workflow.ts
|
|
12526
|
-
var DirectoryWorkflowOptionsSchema =
|
|
12527
|
-
query:
|
|
12528
|
-
state:
|
|
12529
|
-
minPopulation:
|
|
12530
|
-
populationYear:
|
|
12531
|
-
maxCities:
|
|
12532
|
-
maxResultsPerCity:
|
|
12533
|
-
concurrency:
|
|
12534
|
-
includeZipGroups:
|
|
12535
|
-
usZipsCsvPath:
|
|
12536
|
-
saveCsv:
|
|
12537
|
-
gl:
|
|
12538
|
-
hl:
|
|
12539
|
-
proxyMode:
|
|
12540
|
-
proxyZip:
|
|
12541
|
-
debug:
|
|
12542
|
-
headless:
|
|
12543
|
-
kernelApiKey:
|
|
12696
|
+
var DirectoryWorkflowOptionsSchema = z17.object({
|
|
12697
|
+
query: z17.string().min(1),
|
|
12698
|
+
state: z17.string().min(2).default("TN"),
|
|
12699
|
+
minPopulation: z17.number().int().min(0).default(1e5),
|
|
12700
|
+
populationYear: z17.union(POPULATION_YEARS.map((year) => z17.literal(year))).default(2025),
|
|
12701
|
+
maxCities: z17.number().int().min(1).max(100).default(25),
|
|
12702
|
+
maxResultsPerCity: z17.number().int().min(1).max(50).default(50),
|
|
12703
|
+
concurrency: z17.number().int().min(1).max(5).default(5),
|
|
12704
|
+
includeZipGroups: z17.boolean().default(true),
|
|
12705
|
+
usZipsCsvPath: z17.string().optional(),
|
|
12706
|
+
saveCsv: z17.boolean().default(true),
|
|
12707
|
+
gl: z17.string().length(2).default("us"),
|
|
12708
|
+
hl: z17.string().length(2).default("en"),
|
|
12709
|
+
proxyMode: z17.enum(["location", "configured", "none"]).default(DEFAULT_MAPS_PROXY_MODE),
|
|
12710
|
+
proxyZip: z17.string().regex(/^\d{5}$/).optional(),
|
|
12711
|
+
debug: z17.boolean().default(false),
|
|
12712
|
+
headless: z17.boolean().default(true),
|
|
12713
|
+
kernelApiKey: z17.string().optional()
|
|
12544
12714
|
});
|
|
12545
12715
|
function errorMessage(err) {
|
|
12546
12716
|
return err instanceof Error ? err.message : String(err);
|
|
@@ -12761,7 +12931,7 @@ async function runDirectoryWorkflowFromPlan(options, plan) {
|
|
|
12761
12931
|
}
|
|
12762
12932
|
|
|
12763
12933
|
// src/api/directory-routes.ts
|
|
12764
|
-
var directoryApp = new
|
|
12934
|
+
var directoryApp = new Hono8();
|
|
12765
12935
|
directoryApp.post("/run", createApiKeyAuth(), async (c) => {
|
|
12766
12936
|
const user = c.get("user");
|
|
12767
12937
|
const body = await c.req.json().catch(() => ({}));
|
|
@@ -12834,35 +13004,35 @@ directoryApp.post("/run", createApiKeyAuth(), async (c) => {
|
|
|
12834
13004
|
// src/api/workflow-routes.ts
|
|
12835
13005
|
import { createHmac } from "crypto";
|
|
12836
13006
|
import { readFile as readFile2 } from "fs/promises";
|
|
12837
|
-
import { Hono as
|
|
12838
|
-
import { z as
|
|
12839
|
-
var workflowApp = new
|
|
12840
|
-
var WorkflowInputSchema =
|
|
12841
|
-
var WorkflowIdSchema =
|
|
12842
|
-
var CadenceSchema =
|
|
12843
|
-
var ScheduleStatusSchema =
|
|
12844
|
-
var RunBodySchema =
|
|
13007
|
+
import { Hono as Hono9 } from "hono";
|
|
13008
|
+
import { z as z18 } from "zod";
|
|
13009
|
+
var workflowApp = new Hono9();
|
|
13010
|
+
var WorkflowInputSchema = z18.record(z18.unknown()).default({});
|
|
13011
|
+
var WorkflowIdSchema = z18.string().min(1);
|
|
13012
|
+
var CadenceSchema = z18.enum(["daily", "weekly", "monthly"]);
|
|
13013
|
+
var ScheduleStatusSchema = z18.enum(["active", "paused"]);
|
|
13014
|
+
var RunBodySchema = z18.object({
|
|
12845
13015
|
workflowId: WorkflowIdSchema,
|
|
12846
13016
|
input: WorkflowInputSchema,
|
|
12847
|
-
webhookUrl:
|
|
13017
|
+
webhookUrl: z18.string().url().optional()
|
|
12848
13018
|
});
|
|
12849
|
-
var ScheduleCreateSchema =
|
|
13019
|
+
var ScheduleCreateSchema = z18.object({
|
|
12850
13020
|
workflowId: WorkflowIdSchema,
|
|
12851
|
-
name:
|
|
13021
|
+
name: z18.string().min(1).max(120).optional(),
|
|
12852
13022
|
input: WorkflowInputSchema,
|
|
12853
13023
|
cadence: CadenceSchema.default("weekly"),
|
|
12854
|
-
timezone:
|
|
12855
|
-
webhookUrl:
|
|
12856
|
-
nextRunAt:
|
|
13024
|
+
timezone: z18.string().min(1).max(64).default("UTC"),
|
|
13025
|
+
webhookUrl: z18.string().url().optional(),
|
|
13026
|
+
nextRunAt: z18.string().datetime().optional()
|
|
12857
13027
|
});
|
|
12858
|
-
var SchedulePatchSchema =
|
|
12859
|
-
name:
|
|
13028
|
+
var SchedulePatchSchema = z18.object({
|
|
13029
|
+
name: z18.string().min(1).max(120).optional(),
|
|
12860
13030
|
status: ScheduleStatusSchema.optional(),
|
|
12861
13031
|
input: WorkflowInputSchema.optional(),
|
|
12862
13032
|
cadence: CadenceSchema.optional(),
|
|
12863
|
-
timezone:
|
|
12864
|
-
webhookUrl:
|
|
12865
|
-
nextRunAt:
|
|
13033
|
+
timezone: z18.string().min(1).max(64).optional(),
|
|
13034
|
+
webhookUrl: z18.string().url().nullable().optional(),
|
|
13035
|
+
nextRunAt: z18.string().datetime().nullable().optional()
|
|
12866
13036
|
});
|
|
12867
13037
|
function hostedWorkflowOutputDir() {
|
|
12868
13038
|
return workflowOutputBaseDir(process.env.MCP_SCRAPER_WORKFLOW_OUTPUT_DIR?.trim() || "/tmp/mcp-scraper-workflows");
|
|
@@ -13366,7 +13536,7 @@ workflowApp.post("/cron/dispatch", async (c) => {
|
|
|
13366
13536
|
});
|
|
13367
13537
|
|
|
13368
13538
|
// src/api/serp-intelligence-routes.ts
|
|
13369
|
-
import { Hono as
|
|
13539
|
+
import { Hono as Hono10 } from "hono";
|
|
13370
13540
|
|
|
13371
13541
|
// src/serp-intelligence/page-snapshot-extractor.ts
|
|
13372
13542
|
import { createHash as createHash2 } from "crypto";
|
|
@@ -13678,7 +13848,7 @@ async function capturePageSnapshots(targets, options = {}) {
|
|
|
13678
13848
|
}
|
|
13679
13849
|
|
|
13680
13850
|
// src/serp-intelligence/schemas.ts
|
|
13681
|
-
import { z as
|
|
13851
|
+
import { z as z19 } from "zod";
|
|
13682
13852
|
var SerpIntelligenceDeviceValues = ["desktop", "mobile"];
|
|
13683
13853
|
var SerpIntelligenceProxyModeValues = ["location", "configured", "none"];
|
|
13684
13854
|
var SerpIntelligenceAttemptOutcomeValues = [
|
|
@@ -13740,171 +13910,171 @@ function isPublicHttpUrl(value) {
|
|
|
13740
13910
|
return false;
|
|
13741
13911
|
}
|
|
13742
13912
|
}
|
|
13743
|
-
var SerpIntelligencePublicHttpUrlSchema =
|
|
13744
|
-
var SerpIntelligenceCaptureBodySchema =
|
|
13745
|
-
query:
|
|
13746
|
-
location:
|
|
13747
|
-
gl:
|
|
13748
|
-
hl:
|
|
13749
|
-
device:
|
|
13750
|
-
proxyMode:
|
|
13751
|
-
proxyZip:
|
|
13752
|
-
pages:
|
|
13753
|
-
debug:
|
|
13754
|
-
includePageSnapshots:
|
|
13755
|
-
pageSnapshotLimit:
|
|
13913
|
+
var SerpIntelligencePublicHttpUrlSchema = z19.string().url().refine(isPublicHttpUrl, "url must be a public HTTP or HTTPS URL");
|
|
13914
|
+
var SerpIntelligenceCaptureBodySchema = z19.object({
|
|
13915
|
+
query: z19.string().trim().min(1, "query is required"),
|
|
13916
|
+
location: z19.string().trim().min(1).optional(),
|
|
13917
|
+
gl: z19.string().trim().length(2).default("us"),
|
|
13918
|
+
hl: z19.string().trim().length(2).default("en"),
|
|
13919
|
+
device: z19.enum(SerpIntelligenceDeviceValues).default("desktop"),
|
|
13920
|
+
proxyMode: z19.enum(SerpIntelligenceProxyModeValues).default(DEFAULT_PROXY_MODE),
|
|
13921
|
+
proxyZip: z19.string().regex(/^\d{5}$/).optional(),
|
|
13922
|
+
pages: z19.number().int().min(1).max(2).default(1),
|
|
13923
|
+
debug: z19.boolean().default(false),
|
|
13924
|
+
includePageSnapshots: z19.boolean().default(false),
|
|
13925
|
+
pageSnapshotLimit: z19.number().int().min(0).max(10).default(0)
|
|
13756
13926
|
}).strict();
|
|
13757
|
-
var SerpIntelligencePageSnapshotRequestSchema =
|
|
13927
|
+
var SerpIntelligencePageSnapshotRequestSchema = z19.object({
|
|
13758
13928
|
url: SerpIntelligencePublicHttpUrlSchema,
|
|
13759
|
-
sourceKind:
|
|
13760
|
-
sourcePosition:
|
|
13929
|
+
sourceKind: z19.enum(SerpPageSnapshotSourceKindValues).default("configured_target"),
|
|
13930
|
+
sourcePosition: z19.number().int().min(1).optional()
|
|
13761
13931
|
}).strict();
|
|
13762
|
-
var SerpIntelligencePageSnapshotsBodySchema =
|
|
13763
|
-
urls:
|
|
13764
|
-
targets:
|
|
13765
|
-
maxConcurrency:
|
|
13766
|
-
timeoutMs:
|
|
13767
|
-
debug:
|
|
13932
|
+
var SerpIntelligencePageSnapshotsBodySchema = z19.object({
|
|
13933
|
+
urls: z19.array(SerpIntelligencePublicHttpUrlSchema).min(1).max(25),
|
|
13934
|
+
targets: z19.array(SerpIntelligencePageSnapshotRequestSchema).min(1).max(25).optional(),
|
|
13935
|
+
maxConcurrency: z19.number().int().min(1).max(5).default(2),
|
|
13936
|
+
timeoutMs: z19.number().int().min(1e3).max(6e4).default(15e3),
|
|
13937
|
+
debug: z19.boolean().default(false)
|
|
13768
13938
|
}).strict();
|
|
13769
|
-
var SerpIntelligenceAICitationSchema =
|
|
13770
|
-
text:
|
|
13771
|
-
href:
|
|
13939
|
+
var SerpIntelligenceAICitationSchema = z19.object({
|
|
13940
|
+
text: z19.string(),
|
|
13941
|
+
href: z19.string()
|
|
13772
13942
|
}).strict();
|
|
13773
|
-
var SerpIntelligenceOrganicResultSchema =
|
|
13774
|
-
position:
|
|
13775
|
-
title:
|
|
13776
|
-
url:
|
|
13777
|
-
domain:
|
|
13778
|
-
cite:
|
|
13779
|
-
snippet:
|
|
13780
|
-
isRedditStyle:
|
|
13781
|
-
inlineRating:
|
|
13782
|
-
value:
|
|
13783
|
-
count:
|
|
13943
|
+
var SerpIntelligenceOrganicResultSchema = z19.object({
|
|
13944
|
+
position: z19.number().int().min(1),
|
|
13945
|
+
title: z19.string(),
|
|
13946
|
+
url: z19.string(),
|
|
13947
|
+
domain: z19.string(),
|
|
13948
|
+
cite: z19.string().nullable(),
|
|
13949
|
+
snippet: z19.string().nullable(),
|
|
13950
|
+
isRedditStyle: z19.boolean(),
|
|
13951
|
+
inlineRating: z19.object({
|
|
13952
|
+
value: z19.string(),
|
|
13953
|
+
count: z19.string()
|
|
13784
13954
|
}).strict().nullable()
|
|
13785
13955
|
}).strict();
|
|
13786
|
-
var SerpIntelligenceLocationEvidenceSchema =
|
|
13787
|
-
status:
|
|
13788
|
-
expected:
|
|
13789
|
-
city:
|
|
13790
|
-
regionCode:
|
|
13791
|
-
canonicalLocation:
|
|
13956
|
+
var SerpIntelligenceLocationEvidenceSchema = z19.object({
|
|
13957
|
+
status: z19.enum(SerpIntelligenceLocalizationStatusValues),
|
|
13958
|
+
expected: z19.object({
|
|
13959
|
+
city: z19.string(),
|
|
13960
|
+
regionCode: z19.string().nullable(),
|
|
13961
|
+
canonicalLocation: z19.string()
|
|
13792
13962
|
}).strict().nullable(),
|
|
13793
|
-
candidates:
|
|
13794
|
-
city:
|
|
13795
|
-
regionCode:
|
|
13796
|
-
count:
|
|
13797
|
-
examples:
|
|
13963
|
+
candidates: z19.array(z19.object({
|
|
13964
|
+
city: z19.string(),
|
|
13965
|
+
regionCode: z19.string(),
|
|
13966
|
+
count: z19.number().int().min(0),
|
|
13967
|
+
examples: z19.array(z19.string())
|
|
13798
13968
|
}).strict())
|
|
13799
13969
|
}).strict();
|
|
13800
|
-
var SerpIntelligenceHarvestResultSchema =
|
|
13801
|
-
seed:
|
|
13802
|
-
location:
|
|
13803
|
-
extractedAt:
|
|
13804
|
-
totalQuestions:
|
|
13805
|
-
surface:
|
|
13806
|
-
aiOverview:
|
|
13807
|
-
detected:
|
|
13808
|
-
text:
|
|
13809
|
-
citations:
|
|
13810
|
-
expanded:
|
|
13811
|
-
fullyExpanded:
|
|
13812
|
-
sections:
|
|
13970
|
+
var SerpIntelligenceHarvestResultSchema = z19.object({
|
|
13971
|
+
seed: z19.string(),
|
|
13972
|
+
location: z19.string().nullable(),
|
|
13973
|
+
extractedAt: z19.string(),
|
|
13974
|
+
totalQuestions: z19.number().int().min(0),
|
|
13975
|
+
surface: z19.enum(["web", "aim", "unknown"]),
|
|
13976
|
+
aiOverview: z19.object({
|
|
13977
|
+
detected: z19.boolean(),
|
|
13978
|
+
text: z19.string().nullable(),
|
|
13979
|
+
citations: z19.array(SerpIntelligenceAICitationSchema),
|
|
13980
|
+
expanded: z19.boolean().optional(),
|
|
13981
|
+
fullyExpanded: z19.boolean().optional(),
|
|
13982
|
+
sections: z19.array(z19.string()).optional()
|
|
13813
13983
|
}).strict(),
|
|
13814
|
-
aiMode:
|
|
13815
|
-
detected:
|
|
13816
|
-
text:
|
|
13817
|
-
citations:
|
|
13984
|
+
aiMode: z19.object({
|
|
13985
|
+
detected: z19.boolean(),
|
|
13986
|
+
text: z19.string().nullable(),
|
|
13987
|
+
citations: z19.array(SerpIntelligenceAICitationSchema)
|
|
13818
13988
|
}).strict(),
|
|
13819
|
-
tree:
|
|
13820
|
-
flat:
|
|
13821
|
-
videos:
|
|
13822
|
-
forums:
|
|
13823
|
-
organicResults:
|
|
13824
|
-
localPack:
|
|
13825
|
-
entityIds:
|
|
13826
|
-
entities:
|
|
13827
|
-
name:
|
|
13828
|
-
kgId:
|
|
13829
|
-
cid:
|
|
13830
|
-
gcid:
|
|
13989
|
+
tree: z19.array(z19.unknown()),
|
|
13990
|
+
flat: z19.array(z19.unknown()),
|
|
13991
|
+
videos: z19.array(z19.unknown()),
|
|
13992
|
+
forums: z19.array(z19.unknown()),
|
|
13993
|
+
organicResults: z19.array(SerpIntelligenceOrganicResultSchema),
|
|
13994
|
+
localPack: z19.array(z19.unknown()),
|
|
13995
|
+
entityIds: z19.object({
|
|
13996
|
+
entities: z19.array(z19.object({
|
|
13997
|
+
name: z19.string(),
|
|
13998
|
+
kgId: z19.string().nullable(),
|
|
13999
|
+
cid: z19.string().nullable(),
|
|
14000
|
+
gcid: z19.string().nullable()
|
|
13831
14001
|
}).strict()),
|
|
13832
|
-
kgIds:
|
|
13833
|
-
cids:
|
|
13834
|
-
gcids:
|
|
14002
|
+
kgIds: z19.array(z19.string()),
|
|
14003
|
+
cids: z19.array(z19.string()),
|
|
14004
|
+
gcids: z19.array(z19.string())
|
|
13835
14005
|
}).strict(),
|
|
13836
|
-
stats:
|
|
13837
|
-
seed:
|
|
13838
|
-
totalQuestions:
|
|
13839
|
-
maxDepthReached:
|
|
13840
|
-
durationMs:
|
|
13841
|
-
errorCount:
|
|
14006
|
+
stats: z19.object({
|
|
14007
|
+
seed: z19.string(),
|
|
14008
|
+
totalQuestions: z19.number().int().min(0),
|
|
14009
|
+
maxDepthReached: z19.number().int().min(0),
|
|
14010
|
+
durationMs: z19.number().min(0),
|
|
14011
|
+
errorCount: z19.number().int().min(0)
|
|
13842
14012
|
}).strict(),
|
|
13843
|
-
diagnostics:
|
|
13844
|
-
completionStatus:
|
|
13845
|
-
problem:
|
|
13846
|
-
warnings:
|
|
13847
|
-
debug:
|
|
14013
|
+
diagnostics: z19.object({
|
|
14014
|
+
completionStatus: z19.enum(["paa_found", "no_paa", "serp_only"]),
|
|
14015
|
+
problem: z19.null(),
|
|
14016
|
+
warnings: z19.array(z19.unknown()).optional(),
|
|
14017
|
+
debug: z19.object({
|
|
13848
14018
|
locationEvidence: SerpIntelligenceLocationEvidenceSchema.optional()
|
|
13849
14019
|
}).passthrough().optional()
|
|
13850
14020
|
}).passthrough(),
|
|
13851
|
-
whatPeopleSaying:
|
|
14021
|
+
whatPeopleSaying: z19.array(z19.unknown())
|
|
13852
14022
|
}).strict();
|
|
13853
|
-
var SerpIntelligenceCaptureAttemptSchema =
|
|
13854
|
-
attemptNumber:
|
|
13855
|
-
outcome:
|
|
13856
|
-
startedAt:
|
|
13857
|
-
completedAt:
|
|
13858
|
-
durationMs:
|
|
13859
|
-
problemCode:
|
|
13860
|
-
message:
|
|
13861
|
-
kernelSessionId:
|
|
13862
|
-
cleanupSucceeded:
|
|
14023
|
+
var SerpIntelligenceCaptureAttemptSchema = z19.object({
|
|
14024
|
+
attemptNumber: z19.number().int().min(1),
|
|
14025
|
+
outcome: z19.enum(SerpIntelligenceAttemptOutcomeValues),
|
|
14026
|
+
startedAt: z19.string().optional(),
|
|
14027
|
+
completedAt: z19.string().optional(),
|
|
14028
|
+
durationMs: z19.number().min(0).optional(),
|
|
14029
|
+
problemCode: z19.string().optional(),
|
|
14030
|
+
message: z19.string().optional(),
|
|
14031
|
+
kernelSessionId: z19.string().nullable().optional(),
|
|
14032
|
+
cleanupSucceeded: z19.boolean().nullable().optional()
|
|
13863
14033
|
}).strict();
|
|
13864
|
-
var SerpPageSnapshotCaptureSchema =
|
|
14034
|
+
var SerpPageSnapshotCaptureSchema = z19.object({
|
|
13865
14035
|
url: SerpIntelligencePublicHttpUrlSchema,
|
|
13866
14036
|
requestedUrl: SerpIntelligencePublicHttpUrlSchema,
|
|
13867
14037
|
finalUrl: SerpIntelligencePublicHttpUrlSchema.nullable(),
|
|
13868
|
-
sourceKind:
|
|
13869
|
-
sourcePosition:
|
|
13870
|
-
status:
|
|
13871
|
-
fetchedVia:
|
|
13872
|
-
httpStatus:
|
|
13873
|
-
contentType:
|
|
13874
|
-
title:
|
|
14038
|
+
sourceKind: z19.enum(SerpPageSnapshotSourceKindValues),
|
|
14039
|
+
sourcePosition: z19.number().int().min(1).nullable(),
|
|
14040
|
+
status: z19.enum(SerpPageFetchStatusValues),
|
|
14041
|
+
fetchedVia: z19.enum(SerpPageFetchedViaValues).nullable(),
|
|
14042
|
+
httpStatus: z19.number().int().min(100).max(599).nullable(),
|
|
14043
|
+
contentType: z19.string().nullable(),
|
|
14044
|
+
title: z19.string().nullable(),
|
|
13875
14045
|
canonicalUrl: SerpIntelligencePublicHttpUrlSchema.nullable(),
|
|
13876
|
-
metaDescription:
|
|
13877
|
-
headings:
|
|
13878
|
-
level:
|
|
13879
|
-
text:
|
|
14046
|
+
metaDescription: z19.string().nullable(),
|
|
14047
|
+
headings: z19.array(z19.object({
|
|
14048
|
+
level: z19.number().int().min(1).max(6),
|
|
14049
|
+
text: z19.string()
|
|
13880
14050
|
}).strict()).default([]),
|
|
13881
|
-
artifact:
|
|
13882
|
-
htmlBlobUrl:
|
|
13883
|
-
textBlobUrl:
|
|
13884
|
-
markdownBlobUrl:
|
|
13885
|
-
screenshotBlobUrl:
|
|
13886
|
-
contentSha256:
|
|
13887
|
-
capturedAt:
|
|
14051
|
+
artifact: z19.object({
|
|
14052
|
+
htmlBlobUrl: z19.string().url().nullable(),
|
|
14053
|
+
textBlobUrl: z19.string().url().nullable(),
|
|
14054
|
+
markdownBlobUrl: z19.string().url().nullable(),
|
|
14055
|
+
screenshotBlobUrl: z19.string().url().nullable(),
|
|
14056
|
+
contentSha256: z19.string().nullable(),
|
|
14057
|
+
capturedAt: z19.string().nullable()
|
|
13888
14058
|
}).strict(),
|
|
13889
|
-
error:
|
|
13890
|
-
code:
|
|
13891
|
-
message:
|
|
14059
|
+
error: z19.object({
|
|
14060
|
+
code: z19.string(),
|
|
14061
|
+
message: z19.string()
|
|
13892
14062
|
}).strict().nullable()
|
|
13893
14063
|
}).strict();
|
|
13894
|
-
var SerpIntelligenceCaptureResponseSchema =
|
|
14064
|
+
var SerpIntelligenceCaptureResponseSchema = z19.object({
|
|
13895
14065
|
harvestResult: SerpIntelligenceHarvestResultSchema,
|
|
13896
|
-
attempts:
|
|
14066
|
+
attempts: z19.array(SerpIntelligenceCaptureAttemptSchema),
|
|
13897
14067
|
locationEvidence: SerpIntelligenceLocationEvidenceSchema.nullable(),
|
|
13898
|
-
pageSnapshotArtifacts:
|
|
13899
|
-
billing:
|
|
13900
|
-
creditsUsed:
|
|
13901
|
-
requestId:
|
|
13902
|
-
jobId:
|
|
14068
|
+
pageSnapshotArtifacts: z19.array(SerpPageSnapshotCaptureSchema),
|
|
14069
|
+
billing: z19.object({
|
|
14070
|
+
creditsUsed: z19.number().min(0).optional(),
|
|
14071
|
+
requestId: z19.string().optional(),
|
|
14072
|
+
jobId: z19.string().optional()
|
|
13903
14073
|
}).strict().optional()
|
|
13904
14074
|
}).strict();
|
|
13905
|
-
var SerpIntelligencePageSnapshotsResponseSchema =
|
|
13906
|
-
pageSnapshotArtifacts:
|
|
13907
|
-
attempts:
|
|
14075
|
+
var SerpIntelligencePageSnapshotsResponseSchema = z19.object({
|
|
14076
|
+
pageSnapshotArtifacts: z19.array(SerpPageSnapshotCaptureSchema),
|
|
14077
|
+
attempts: z19.array(SerpIntelligenceCaptureAttemptSchema).default([])
|
|
13908
14078
|
}).strict();
|
|
13909
14079
|
|
|
13910
14080
|
// src/serp-intelligence/serp-capture-service.ts
|
|
@@ -14076,7 +14246,7 @@ var SERP_INTELLIGENCE_RATE_LIMIT = 60;
|
|
|
14076
14246
|
var SERP_INTELLIGENCE_RATE_WINDOW_SECONDS = 60;
|
|
14077
14247
|
var POST_CAPTURE_ROUTE_LABEL = "POST /capture";
|
|
14078
14248
|
var POST_PAGE_SNAPSHOTS_ROUTE_LABEL = "POST /page-snapshots";
|
|
14079
|
-
var serpIntelligenceApp = new
|
|
14249
|
+
var serpIntelligenceApp = new Hono10();
|
|
14080
14250
|
serpIntelligenceApp.use("*", createApiKeyAuth());
|
|
14081
14251
|
function structuredError(input) {
|
|
14082
14252
|
return {
|
|
@@ -14245,41 +14415,95 @@ serpIntelligenceApp.post("/page-snapshots", async (c) => {
|
|
|
14245
14415
|
});
|
|
14246
14416
|
|
|
14247
14417
|
// src/mcp/mcp-routes.ts
|
|
14248
|
-
import { Hono as
|
|
14418
|
+
import { Hono as Hono11 } from "hono";
|
|
14249
14419
|
import { WebStandardStreamableHTTPServerTransport } from "@modelcontextprotocol/sdk/server/webStandardStreamableHttp.js";
|
|
14420
|
+
|
|
14421
|
+
// src/mcp/mcp-oauth.ts
|
|
14422
|
+
import { createRemoteJWKSet, jwtVerify } from "jose";
|
|
14423
|
+
function oauthIssuer() {
|
|
14424
|
+
return (process.env.OAUTH_ISSUER ?? "https://mcpscraper.dev").replace(/\/$/, "");
|
|
14425
|
+
}
|
|
14426
|
+
function oauthResourceUrl() {
|
|
14427
|
+
return (process.env.MCP_SCRAPER_RESOURCE_URL ?? "https://mcpscraper.dev/mcp").replace(/\/$/, "");
|
|
14428
|
+
}
|
|
14429
|
+
function oauthResourceBaseUrl() {
|
|
14430
|
+
return (process.env.OAUTH_RS_BASE_URL ?? "https://mcpscraper.dev").replace(/\/$/, "");
|
|
14431
|
+
}
|
|
14432
|
+
function protectedResourceMetadata() {
|
|
14433
|
+
return {
|
|
14434
|
+
resource: oauthResourceUrl(),
|
|
14435
|
+
authorization_servers: [oauthIssuer()],
|
|
14436
|
+
scopes_supported: ["scraper.use"],
|
|
14437
|
+
bearer_methods_supported: ["header"]
|
|
14438
|
+
};
|
|
14439
|
+
}
|
|
14440
|
+
function wwwAuthenticateValue() {
|
|
14441
|
+
return `Bearer resource_metadata="${oauthResourceBaseUrl()}/.well-known/oauth-protected-resource"`;
|
|
14442
|
+
}
|
|
14443
|
+
function isBearerJwt(token) {
|
|
14444
|
+
return !token.startsWith("sk_") && token.split(".").length === 3;
|
|
14445
|
+
}
|
|
14446
|
+
var jwksSingleton = null;
|
|
14447
|
+
function getJwks() {
|
|
14448
|
+
if (jwksSingleton) return jwksSingleton;
|
|
14449
|
+
jwksSingleton = createRemoteJWKSet(new URL(`${oauthIssuer()}/.well-known/jwks.json`));
|
|
14450
|
+
return jwksSingleton;
|
|
14451
|
+
}
|
|
14452
|
+
async function verifyScraperOAuthJwt(token) {
|
|
14453
|
+
const { payload } = await jwtVerify(token, getJwks(), {
|
|
14454
|
+
issuer: oauthIssuer(),
|
|
14455
|
+
audience: oauthResourceUrl()
|
|
14456
|
+
});
|
|
14457
|
+
return payload;
|
|
14458
|
+
}
|
|
14459
|
+
|
|
14460
|
+
// src/mcp/mcp-routes.ts
|
|
14250
14461
|
configureReportSaving(false);
|
|
14251
|
-
function mcpAuthError() {
|
|
14462
|
+
function mcpAuthError(message = "authentication required") {
|
|
14252
14463
|
const body = JSON.stringify({
|
|
14253
14464
|
jsonrpc: "2.0",
|
|
14254
14465
|
id: null,
|
|
14255
14466
|
error: {
|
|
14256
14467
|
code: -32001,
|
|
14257
|
-
message
|
|
14468
|
+
message
|
|
14258
14469
|
}
|
|
14259
14470
|
});
|
|
14260
14471
|
return new Response(body, {
|
|
14261
14472
|
status: 401,
|
|
14262
14473
|
headers: {
|
|
14263
14474
|
"Content-Type": "application/json",
|
|
14264
|
-
"WWW-Authenticate":
|
|
14475
|
+
"WWW-Authenticate": wwwAuthenticateValue()
|
|
14265
14476
|
}
|
|
14266
14477
|
});
|
|
14267
14478
|
}
|
|
14268
14479
|
async function requireMcpCallerKey(c) {
|
|
14269
14480
|
const xApiKey = c.req.header("x-api-key")?.trim();
|
|
14270
14481
|
const authHeader = c.req.header("Authorization")?.trim();
|
|
14271
|
-
const
|
|
14272
|
-
|
|
14482
|
+
const bearerToken = authHeader?.startsWith("Bearer ") ? authHeader.slice(7).trim() : void 0;
|
|
14483
|
+
if (!xApiKey && bearerToken && isBearerJwt(bearerToken)) {
|
|
14484
|
+
let identity;
|
|
14485
|
+
try {
|
|
14486
|
+
const payload = await verifyScraperOAuthJwt(bearerToken);
|
|
14487
|
+
identity = typeof payload.sub === "string" ? payload.sub : "";
|
|
14488
|
+
} catch {
|
|
14489
|
+
return mcpAuthError("Bearer token is invalid or expired");
|
|
14490
|
+
}
|
|
14491
|
+
if (!identity) return mcpAuthError("Bearer token has no subject");
|
|
14492
|
+
const user2 = await getUserByEmail(identity);
|
|
14493
|
+
if (!user2 || !user2.api_key || user2.key_active !== 1) return mcpAuthError("No active MCP Scraper account for this identity");
|
|
14494
|
+
return user2.api_key;
|
|
14495
|
+
}
|
|
14496
|
+
const callerKey = xApiKey ?? bearerToken;
|
|
14273
14497
|
if (!callerKey) return mcpAuthError();
|
|
14274
14498
|
const user = await getUserByApiKey(callerKey);
|
|
14275
14499
|
if (!user) return mcpAuthError();
|
|
14276
14500
|
return callerKey;
|
|
14277
14501
|
}
|
|
14278
|
-
var mcpApp = new
|
|
14502
|
+
var mcpApp = new Hono11();
|
|
14279
14503
|
function registerSerpIntelligenceCaptureTools(server, executor) {
|
|
14280
14504
|
server.registerTool("capture_serp_snapshot", {
|
|
14281
14505
|
title: "SERP Intelligence Snapshot",
|
|
14282
|
-
description: "Capture a structured SERP Intelligence Google snapshot through POST /serp-intelligence/capture, the same product capture path used by Phoenix. Split query from location and infer gl/hl.
|
|
14506
|
+
description: "Capture a structured SERP Intelligence Google snapshot through POST /serp-intelligence/capture, the same product capture path used by Phoenix. Split query from location and infer gl/hl. Routing is automatic \u2014 leave proxyMode unset. Set debug true when investigating location evidence, proxy behavior, CAPTCHA, or capture reliability.",
|
|
14283
14507
|
inputSchema: CaptureSerpSnapshotInputSchema,
|
|
14284
14508
|
outputSchema: CaptureSerpSnapshotOutputSchema,
|
|
14285
14509
|
annotations: liveWebToolAnnotations("SERP Intelligence Snapshot")
|
|
@@ -14313,7 +14537,7 @@ mcpApp.all("/", async (c) => {
|
|
|
14313
14537
|
});
|
|
14314
14538
|
|
|
14315
14539
|
// src/api/browser-agent-routes.ts
|
|
14316
|
-
import { Hono as
|
|
14540
|
+
import { Hono as Hono12 } from "hono";
|
|
14317
14541
|
|
|
14318
14542
|
// src/api/browser-agent-db.ts
|
|
14319
14543
|
import { randomUUID as randomUUID2 } from "crypto";
|
|
@@ -15747,7 +15971,7 @@ async function loadOpenSession(id, userId) {
|
|
|
15747
15971
|
return row;
|
|
15748
15972
|
}
|
|
15749
15973
|
function buildBrowserAgentRoutes() {
|
|
15750
|
-
const app2 = new
|
|
15974
|
+
const app2 = new Hono12();
|
|
15751
15975
|
app2.use("*", async (c, next) => {
|
|
15752
15976
|
await migrateBrowserAgent();
|
|
15753
15977
|
return next();
|
|
@@ -16453,9 +16677,149 @@ if (state.key) { refreshSessions(); if (state.current) selectSession(state.curre
|
|
|
16453
16677
|
|
|
16454
16678
|
// src/api/stripe-routes.ts
|
|
16455
16679
|
import Stripe from "stripe";
|
|
16456
|
-
import { Hono as
|
|
16680
|
+
import { Hono as Hono13 } from "hono";
|
|
16681
|
+
|
|
16682
|
+
// src/api/memory.ts
|
|
16683
|
+
import { createCipheriv, createDecipheriv, randomBytes, scryptSync } from "crypto";
|
|
16684
|
+
|
|
16685
|
+
// src/api/session.ts
|
|
16686
|
+
import { createHmac as createHmac2, timingSafeEqual } from "crypto";
|
|
16687
|
+
var isProduction = () => process.env.NODE_ENV === "production" || process.env.VERCEL === "1";
|
|
16688
|
+
function getSessionSecret() {
|
|
16689
|
+
const configured = process.env.SESSION_SECRET?.trim();
|
|
16690
|
+
if (configured) return configured;
|
|
16691
|
+
if (isProduction()) throw new Error("SESSION_SECRET is not set \u2014 add it to your Vercel env vars (see src/api/env.ts)");
|
|
16692
|
+
return "dev-secret-change-me";
|
|
16693
|
+
}
|
|
16694
|
+
var secret = () => getSessionSecret();
|
|
16695
|
+
function safeEqualHex(a, b) {
|
|
16696
|
+
if (a.length !== b.length) return false;
|
|
16697
|
+
try {
|
|
16698
|
+
return timingSafeEqual(Buffer.from(a, "hex"), Buffer.from(b, "hex"));
|
|
16699
|
+
} catch {
|
|
16700
|
+
return false;
|
|
16701
|
+
}
|
|
16702
|
+
}
|
|
16703
|
+
function signSession(userId) {
|
|
16704
|
+
const payload = String(userId);
|
|
16705
|
+
const sig = createHmac2("sha256", secret()).update(payload).digest("hex");
|
|
16706
|
+
return `${payload}.${sig}`;
|
|
16707
|
+
}
|
|
16708
|
+
function verifySession(token) {
|
|
16709
|
+
const dot = token.lastIndexOf(".");
|
|
16710
|
+
if (dot === -1) return null;
|
|
16711
|
+
const payload = token.slice(0, dot);
|
|
16712
|
+
const sig = token.slice(dot + 1);
|
|
16713
|
+
const expected = createHmac2("sha256", secret()).update(payload).digest("hex");
|
|
16714
|
+
if (!safeEqualHex(sig, expected)) return null;
|
|
16715
|
+
const id = parseInt(payload);
|
|
16716
|
+
return isNaN(id) ? null : id;
|
|
16717
|
+
}
|
|
16718
|
+
|
|
16719
|
+
// src/api/memory.ts
|
|
16720
|
+
var MEMORY_BASE_URL = () => (process.env.MCP_MEMORY_BASE_URL ?? "https://mcp-memory-omega.vercel.app").replace(/\/$/, "");
|
|
16721
|
+
var ADMIN_KEY = () => process.env.MCP_MEMORY_ADMIN_KEY?.trim() ?? "";
|
|
16722
|
+
function isMemoryOperator(email) {
|
|
16723
|
+
if (!email) return false;
|
|
16724
|
+
const ops = (process.env.MEMORY_OPERATOR_IDENTITIES ?? "").split(",").map((s) => s.trim().toLowerCase()).filter(Boolean);
|
|
16725
|
+
return ops.includes(email.trim().toLowerCase());
|
|
16726
|
+
}
|
|
16727
|
+
function encKey() {
|
|
16728
|
+
return scryptSync(getSessionSecret(), "mcp-memory-key-v1", 32);
|
|
16729
|
+
}
|
|
16730
|
+
function encryptMemoryKey(secret2) {
|
|
16731
|
+
const iv = randomBytes(12);
|
|
16732
|
+
const cipher = createCipheriv("aes-256-gcm", encKey(), iv);
|
|
16733
|
+
const enc = Buffer.concat([cipher.update(secret2, "utf8"), cipher.final()]);
|
|
16734
|
+
const tag = cipher.getAuthTag();
|
|
16735
|
+
return `${iv.toString("base64")}:${tag.toString("base64")}:${enc.toString("base64")}`;
|
|
16736
|
+
}
|
|
16737
|
+
function decryptMemoryKey(stored) {
|
|
16738
|
+
try {
|
|
16739
|
+
const [ivB, tagB, dataB] = stored.split(":");
|
|
16740
|
+
const decipher = createDecipheriv("aes-256-gcm", encKey(), Buffer.from(ivB, "base64"));
|
|
16741
|
+
decipher.setAuthTag(Buffer.from(tagB, "base64"));
|
|
16742
|
+
return Buffer.concat([decipher.update(Buffer.from(dataB, "base64")), decipher.final()]).toString("utf8");
|
|
16743
|
+
} catch {
|
|
16744
|
+
return null;
|
|
16745
|
+
}
|
|
16746
|
+
}
|
|
16747
|
+
async function memoryCall(toolName, args, userMemoryKey) {
|
|
16748
|
+
try {
|
|
16749
|
+
const res = await fetch(`${MEMORY_BASE_URL()}/api/mcp/thorbitMemoryServer/tools/${toolName}/execute`, {
|
|
16750
|
+
method: "POST",
|
|
16751
|
+
headers: { "content-type": "application/json" },
|
|
16752
|
+
body: JSON.stringify({ data: { ...args, apiKey: userMemoryKey } })
|
|
16753
|
+
});
|
|
16754
|
+
const body = await res.json().catch(() => null);
|
|
16755
|
+
if (!res.ok) return { ok: false, error: body?.error ?? `memory ${toolName} failed (${res.status})` };
|
|
16756
|
+
if (body?.result) return body.result;
|
|
16757
|
+
return { ok: false, error: body?.error ?? `memory ${toolName} returned no result` };
|
|
16758
|
+
} catch (err) {
|
|
16759
|
+
return { ok: false, error: err?.message ?? "memory call failed" };
|
|
16760
|
+
}
|
|
16761
|
+
}
|
|
16762
|
+
function personalVault(user) {
|
|
16763
|
+
return `mcp-${user.id}`;
|
|
16764
|
+
}
|
|
16765
|
+
function memoryIdentity(user) {
|
|
16766
|
+
return user.email;
|
|
16767
|
+
}
|
|
16768
|
+
function resolveMemoryIdentity(user) {
|
|
16769
|
+
return memoryIdentity(user);
|
|
16770
|
+
}
|
|
16771
|
+
function memoryPlanForReads(user) {
|
|
16772
|
+
if (isMemoryOperator(user.email)) return "unlimited";
|
|
16773
|
+
if (user.memory_plan === "pro" || user.memory_plan === "team") return user.memory_plan;
|
|
16774
|
+
return "free";
|
|
16775
|
+
}
|
|
16776
|
+
async function provisionMemoryForUser(user) {
|
|
16777
|
+
try {
|
|
16778
|
+
const { key } = await getOrCreateUserMemoryKey(user);
|
|
16779
|
+
if (key) await setMemoryProvisioned(user.id, true);
|
|
16780
|
+
} catch {
|
|
16781
|
+
}
|
|
16782
|
+
}
|
|
16783
|
+
async function getOrCreateUserMemoryKey(user) {
|
|
16784
|
+
const creds = await getUserMemoryCreds(user.id);
|
|
16785
|
+
if (creds.memory_key) {
|
|
16786
|
+
const decrypted = decryptMemoryKey(creds.memory_key);
|
|
16787
|
+
if (decrypted) return { key: decrypted };
|
|
16788
|
+
}
|
|
16789
|
+
const admin = ADMIN_KEY();
|
|
16790
|
+
if (!admin) return { key: null, error: "memory not configured (MCP_MEMORY_ADMIN_KEY missing)" };
|
|
16791
|
+
const identity = memoryIdentity(user);
|
|
16792
|
+
const plan = isMemoryOperator(user.email) ? "unlimited" : (await getMemoryPlan(user.id)).plan;
|
|
16793
|
+
const provisioned = await memoryCall(
|
|
16794
|
+
"provisionDefaultsTool",
|
|
16795
|
+
{ granteeIdentity: identity, issueKey: true, plan },
|
|
16796
|
+
admin
|
|
16797
|
+
);
|
|
16798
|
+
if (!provisioned.ok || !provisioned.secret) {
|
|
16799
|
+
return { key: null, error: provisioned.error ?? "failed to provision default vaults" };
|
|
16800
|
+
}
|
|
16801
|
+
await setUserMemoryCreds(user.id, encryptMemoryKey(provisioned.secret), identity);
|
|
16802
|
+
return { key: provisioned.secret };
|
|
16803
|
+
}
|
|
16804
|
+
async function syncMemoryKeyPlan(user, plan) {
|
|
16805
|
+
const effectivePlan = isMemoryOperator(user.email) ? "unlimited" : plan;
|
|
16806
|
+
const { key, error } = await getOrCreateUserMemoryKey(user);
|
|
16807
|
+
if (!key) return { ok: false, error: error ?? "memory unavailable" };
|
|
16808
|
+
const listed = await memoryCall(
|
|
16809
|
+
"listKeysTool",
|
|
16810
|
+
{},
|
|
16811
|
+
key
|
|
16812
|
+
);
|
|
16813
|
+
if (!listed.ok || !listed.keys?.length) return { ok: false, error: listed.error ?? "no memory keys found" };
|
|
16814
|
+
const target = listed.keys[0];
|
|
16815
|
+
if (target.plan === effectivePlan) return { ok: true };
|
|
16816
|
+
const res = await memoryCall("setScopeTool", { keyId: target.keyId, plan: effectivePlan }, key);
|
|
16817
|
+
return { ok: res.ok, error: res.error };
|
|
16818
|
+
}
|
|
16819
|
+
|
|
16820
|
+
// src/api/stripe-routes.ts
|
|
16457
16821
|
var stripe = new Stripe(process.env.STRIPE_SECRET_KEY, { apiVersion: "2026-02-25.clover" });
|
|
16458
|
-
var stripeApp = new
|
|
16822
|
+
var stripeApp = new Hono13();
|
|
16459
16823
|
stripeApp.post("/webhooks", async (c) => {
|
|
16460
16824
|
const sig = c.req.header("stripe-signature");
|
|
16461
16825
|
const body = await c.req.text();
|
|
@@ -16469,6 +16833,8 @@ stripeApp.post("/webhooks", async (c) => {
|
|
|
16469
16833
|
if (!isNew) return c.json({ received: true });
|
|
16470
16834
|
if (event.type === "invoice.paid" || event.type === "invoice.payment_succeeded") {
|
|
16471
16835
|
const invoice = event.data.object;
|
|
16836
|
+
const memLineId = invoice.lines.data.map(linePriceId).find((id) => id && id in MEMORY_PLANS);
|
|
16837
|
+
if (memLineId) return c.json({ received: true });
|
|
16472
16838
|
const lineTierId = invoice.lines.data.map(linePriceId).find((id) => id && id in SUBSCRIPTION_TIERS);
|
|
16473
16839
|
if (lineTierId) {
|
|
16474
16840
|
const tier = SUBSCRIPTION_TIERS[lineTierId];
|
|
@@ -16482,6 +16848,16 @@ stripeApp.post("/webhooks", async (c) => {
|
|
|
16482
16848
|
if (event.type === "customer.subscription.created" || event.type === "customer.subscription.updated") {
|
|
16483
16849
|
const sub = event.data.object;
|
|
16484
16850
|
const priceId = sub.items.data[0]?.price?.id;
|
|
16851
|
+
if (priceId && priceId in MEMORY_PLANS) {
|
|
16852
|
+
const mem = MEMORY_PLANS[priceId];
|
|
16853
|
+
const user2 = await resolveUser(sub.customer, sub.customer_email);
|
|
16854
|
+
if (user2) {
|
|
16855
|
+
const live = sub.status === "active" || sub.status === "trialing";
|
|
16856
|
+
await setMemoryPlan(user2.id, live ? mem.plan : "free", live ? sub.id : null);
|
|
16857
|
+
await syncMemoryKeyPlan(user2, live ? mem.plan : "free");
|
|
16858
|
+
}
|
|
16859
|
+
return c.json({ received: true });
|
|
16860
|
+
}
|
|
16485
16861
|
const user = await getUserByStripeCustomerId(sub.customer);
|
|
16486
16862
|
if (!user) return c.json({ received: true });
|
|
16487
16863
|
if (priceId && priceId in SUBSCRIPTION_TIERS) {
|
|
@@ -16496,6 +16872,14 @@ stripeApp.post("/webhooks", async (c) => {
|
|
|
16496
16872
|
if (event.type === "customer.subscription.deleted") {
|
|
16497
16873
|
const sub = event.data.object;
|
|
16498
16874
|
const priceId = sub.items.data[0]?.price?.id;
|
|
16875
|
+
if (priceId && priceId in MEMORY_PLANS || await resolveUserByMemorySubscription(sub.id)) {
|
|
16876
|
+
const user2 = await resolveUserByMemorySubscription(sub.id) ?? await getUserByStripeCustomerId(sub.customer);
|
|
16877
|
+
if (user2) {
|
|
16878
|
+
await setMemoryPlan(user2.id, "free", null);
|
|
16879
|
+
await syncMemoryKeyPlan(user2, "free");
|
|
16880
|
+
}
|
|
16881
|
+
return c.json({ received: true });
|
|
16882
|
+
}
|
|
16499
16883
|
const user = await getUserByStripeCustomerId(sub.customer);
|
|
16500
16884
|
if (!user) return c.json({ received: true });
|
|
16501
16885
|
if (priceId && priceId in SUBSCRIPTION_TIERS) {
|
|
@@ -16520,6 +16904,417 @@ async function resolveUser(customerId, emailFallback) {
|
|
|
16520
16904
|
return user;
|
|
16521
16905
|
}
|
|
16522
16906
|
|
|
16907
|
+
// src/api/oauth-routes.ts
|
|
16908
|
+
import { Hono as Hono14 } from "hono";
|
|
16909
|
+
import { getCookie, setCookie } from "hono/cookie";
|
|
16910
|
+
import { createHash as createHash3, randomBytes as randomBytes2, randomUUID as randomUUID3 } from "crypto";
|
|
16911
|
+
import { importPKCS8, exportJWK, SignJWT } from "jose";
|
|
16912
|
+
var ISSUER = "https://mcpscraper.dev";
|
|
16913
|
+
var RESOURCE = () => process.env.MCP_MEMORY_RESOURCE_URL ?? "https://mcp-memory-omega.vercel.app/mcp/thorbitMemoryServer/mcp";
|
|
16914
|
+
var SCRAPER_RESOURCE = () => (process.env.MCP_SCRAPER_RESOURCE_URL ?? "https://mcpscraper.dev/mcp").replace(/\/$/, "");
|
|
16915
|
+
var MEMORY_SCOPES = ["memory.read", "memory.write", "memory.admin"];
|
|
16916
|
+
var SCRAPER_SCOPES = ["scraper.use"];
|
|
16917
|
+
var SUPPORTED_SCOPES = [...MEMORY_SCOPES, ...SCRAPER_SCOPES];
|
|
16918
|
+
function isScraperResource(resource) {
|
|
16919
|
+
if (!resource) return false;
|
|
16920
|
+
return resource.replace(/\/$/, "") === SCRAPER_RESOURCE();
|
|
16921
|
+
}
|
|
16922
|
+
var ACCESS_TTL_SECONDS = 3600;
|
|
16923
|
+
var REFRESH_TTL_SECONDS = 60 * 60 * 24 * 30;
|
|
16924
|
+
var CODE_TTL_SECONDS = 60;
|
|
16925
|
+
var secureCookies = process.env.NODE_ENV === "production" || process.env.VERCEL === "1";
|
|
16926
|
+
var sessionCookieOptions = {
|
|
16927
|
+
httpOnly: true,
|
|
16928
|
+
secure: secureCookies,
|
|
16929
|
+
path: "/",
|
|
16930
|
+
maxAge: 60 * 60 * 24 * 30,
|
|
16931
|
+
sameSite: "Lax"
|
|
16932
|
+
};
|
|
16933
|
+
var cachedKeys = null;
|
|
16934
|
+
async function getKeys() {
|
|
16935
|
+
if (cachedKeys) return cachedKeys;
|
|
16936
|
+
const raw = process.env.OAUTH_SIGNING_KEY;
|
|
16937
|
+
if (!raw?.trim()) throw new Error("OAUTH_SIGNING_KEY is not set");
|
|
16938
|
+
const pem = raw.includes("BEGIN") ? raw : Buffer.from(raw, "base64").toString("utf8");
|
|
16939
|
+
const privateKey = await importPKCS8(pem, "RS256", { extractable: true });
|
|
16940
|
+
const full = await exportJWK(privateKey);
|
|
16941
|
+
const publicJwk = { kty: full.kty, n: full.n, e: full.e };
|
|
16942
|
+
const kid = createHash3("sha256").update(JSON.stringify({ e: publicJwk.e, kty: publicJwk.kty, n: publicJwk.n })).digest("base64url").slice(0, 16);
|
|
16943
|
+
publicJwk.kid = kid;
|
|
16944
|
+
publicJwk.alg = "RS256";
|
|
16945
|
+
publicJwk.use = "sig";
|
|
16946
|
+
cachedKeys = { privateKey, publicJwk, kid };
|
|
16947
|
+
return cachedKeys;
|
|
16948
|
+
}
|
|
16949
|
+
function isAllowedRedirect(uri) {
|
|
16950
|
+
let u;
|
|
16951
|
+
try {
|
|
16952
|
+
u = new URL(uri);
|
|
16953
|
+
} catch {
|
|
16954
|
+
return false;
|
|
16955
|
+
}
|
|
16956
|
+
if (u.protocol !== "https:") return false;
|
|
16957
|
+
const h = u.hostname.toLowerCase();
|
|
16958
|
+
if (["claude.ai", "www.claude.ai", "claude.com", "www.claude.com"].includes(h)) return true;
|
|
16959
|
+
if (h.endsWith(".claude.ai") || h.endsWith(".claude.com")) return true;
|
|
16960
|
+
const extra = (process.env.OAUTH_EXTRA_REDIRECT_HOSTS ?? "").split(",").map((s) => s.trim().toLowerCase()).filter(Boolean);
|
|
16961
|
+
return extra.includes(h);
|
|
16962
|
+
}
|
|
16963
|
+
function resolvePlan(user) {
|
|
16964
|
+
if (isMemoryOperator(user.email)) return "unlimited";
|
|
16965
|
+
if (user.memory_plan === "pro" || user.memory_plan === "team") return user.memory_plan;
|
|
16966
|
+
return "free";
|
|
16967
|
+
}
|
|
16968
|
+
function negotiateScope(requested, user, resource) {
|
|
16969
|
+
if (isScraperResource(resource)) return "scraper.use";
|
|
16970
|
+
const operator = isMemoryOperator(user.email);
|
|
16971
|
+
const wanted = (requested ?? "").split(/\s+/).map((s) => s.trim()).filter(Boolean);
|
|
16972
|
+
const filtered = wanted.filter((s) => MEMORY_SCOPES.includes(s));
|
|
16973
|
+
const base = filtered.length > 0 ? filtered : ["memory.read", "memory.write"];
|
|
16974
|
+
const allowed = base.filter((s) => s === "memory.admin" ? operator : true);
|
|
16975
|
+
const unique = [...new Set(allowed)];
|
|
16976
|
+
if (operator && !unique.includes("memory.admin")) unique.push("memory.admin");
|
|
16977
|
+
return unique.join(" ");
|
|
16978
|
+
}
|
|
16979
|
+
async function sessionUser(token) {
|
|
16980
|
+
if (!token) return null;
|
|
16981
|
+
const userId = verifySession(token);
|
|
16982
|
+
if (!userId) return null;
|
|
16983
|
+
const user = await getUserById(userId);
|
|
16984
|
+
return user ?? null;
|
|
16985
|
+
}
|
|
16986
|
+
function escapeHtml(value) {
|
|
16987
|
+
return value.replace(/&/g, "&").replace(/</g, "<").replace(/>/g, ">").replace(/"/g, """).replace(/'/g, "'");
|
|
16988
|
+
}
|
|
16989
|
+
function hiddenFields(p) {
|
|
16990
|
+
const fields = [
|
|
16991
|
+
["response_type", p.response_type],
|
|
16992
|
+
["client_id", p.client_id],
|
|
16993
|
+
["redirect_uri", p.redirect_uri],
|
|
16994
|
+
["code_challenge", p.code_challenge],
|
|
16995
|
+
["code_challenge_method", p.code_challenge_method],
|
|
16996
|
+
["scope", p.scope],
|
|
16997
|
+
["state", p.state],
|
|
16998
|
+
["resource", p.resource]
|
|
16999
|
+
];
|
|
17000
|
+
return fields.map(([k, v]) => `<input type="hidden" name="${k}" value="${escapeHtml(v)}" />`).join("");
|
|
17001
|
+
}
|
|
17002
|
+
function scopeLabel(scope) {
|
|
17003
|
+
const map = {
|
|
17004
|
+
"memory.read": "Read your vaults, notes and search",
|
|
17005
|
+
"memory.write": "Create and edit notes in your vaults",
|
|
17006
|
+
"memory.admin": "Full administrative access",
|
|
17007
|
+
"scraper.use": "Run web scraping, search, maps and social tools (billed to your account)"
|
|
17008
|
+
};
|
|
17009
|
+
return scope.split(/\s+/).filter(Boolean).map((s) => map[s] ?? s).map((l) => `<li>${escapeHtml(l)}</li>`).join("");
|
|
17010
|
+
}
|
|
17011
|
+
function resourceLabel(resource) {
|
|
17012
|
+
return isScraperResource(resource) ? "MCP Scraper" : "mcp-memory";
|
|
17013
|
+
}
|
|
17014
|
+
function pageShell(inner) {
|
|
17015
|
+
return `<!doctype html><html lang="en"><head><meta charset="utf-8" /><meta name="viewport" content="width=device-width,initial-scale=1" /><title>Authorize Claude \xB7 mcp-memory</title><style>
|
|
17016
|
+
:root{color-scheme:dark}
|
|
17017
|
+
*{box-sizing:border-box}
|
|
17018
|
+
body{margin:0;min-height:100vh;display:flex;align-items:center;justify-content:center;background:#0b0d10;color:#e8eaed;font-family:-apple-system,BlinkMacSystemFont,"Segoe UI",Inter,system-ui,sans-serif;padding:24px}
|
|
17019
|
+
.card{width:100%;max-width:420px;background:#14171c;border:1px solid #232830;border-radius:16px;padding:32px}
|
|
17020
|
+
h1{font-size:20px;margin:0 0 4px}
|
|
17021
|
+
p.sub{color:#9aa3ad;font-size:14px;margin:0 0 24px}
|
|
17022
|
+
label{display:block;font-size:13px;color:#9aa3ad;margin:14px 0 6px}
|
|
17023
|
+
input[type=email],input[type=password]{width:100%;padding:11px 13px;background:#0b0d10;border:1px solid #2b313a;border-radius:9px;color:#e8eaed;font-size:14px}
|
|
17024
|
+
input:focus{outline:none;border-color:#5b8cff}
|
|
17025
|
+
button{width:100%;margin-top:20px;padding:12px;border:0;border-radius:9px;font-size:14px;font-weight:600;cursor:pointer}
|
|
17026
|
+
.primary{background:#5b8cff;color:#fff}
|
|
17027
|
+
.ghost{background:transparent;color:#9aa3ad;border:1px solid #2b313a;margin-top:10px}
|
|
17028
|
+
ul{margin:14px 0;padding-left:18px;color:#cbd2da;font-size:13px;line-height:1.7}
|
|
17029
|
+
.err{background:#3a1b1f;border:1px solid #6b2b30;color:#ffb4b4;padding:10px 12px;border-radius:9px;font-size:13px;margin-bottom:16px}
|
|
17030
|
+
.who{font-size:13px;color:#9aa3ad;margin-bottom:8px}
|
|
17031
|
+
</style></head><body><div class="card">${inner}</div></body></html>`;
|
|
17032
|
+
}
|
|
17033
|
+
function renderLogin(p, error) {
|
|
17034
|
+
return pageShell(`
|
|
17035
|
+
<h1>Sign in to ${escapeHtml(resourceLabel(p.resource))}</h1>
|
|
17036
|
+
<p class="sub">Authorize Claude to connect to your ${escapeHtml(resourceLabel(p.resource))} on mcpscraper.dev.</p>
|
|
17037
|
+
${error ? `<div class="err">${escapeHtml(error)}</div>` : ""}
|
|
17038
|
+
<form method="post" action="/oauth/authorize">
|
|
17039
|
+
${hiddenFields(p)}
|
|
17040
|
+
<input type="hidden" name="action" value="login" />
|
|
17041
|
+
<label for="email">Email</label>
|
|
17042
|
+
<input id="email" name="email" type="email" autocomplete="username" required />
|
|
17043
|
+
<label for="password">Password</label>
|
|
17044
|
+
<input id="password" name="password" type="password" autocomplete="current-password" required />
|
|
17045
|
+
<button class="primary" type="submit">Sign in</button>
|
|
17046
|
+
</form>`);
|
|
17047
|
+
}
|
|
17048
|
+
function renderConsent(p, email, error) {
|
|
17049
|
+
return pageShell(`
|
|
17050
|
+
<h1>Connect Claude to ${escapeHtml(resourceLabel(p.resource))}</h1>
|
|
17051
|
+
<p class="sub">Allow Claude to access your ${escapeHtml(resourceLabel(p.resource))}?</p>
|
|
17052
|
+
${error ? `<div class="err">${escapeHtml(error)}</div>` : ""}
|
|
17053
|
+
<div class="who">Signed in as <strong>${escapeHtml(email)}</strong></div>
|
|
17054
|
+
<ul>${scopeLabel(p.scope)}</ul>
|
|
17055
|
+
<form method="post" action="/oauth/authorize">
|
|
17056
|
+
${hiddenFields(p)}
|
|
17057
|
+
<input type="hidden" name="action" value="approve" />
|
|
17058
|
+
<button class="primary" type="submit">Allow</button>
|
|
17059
|
+
</form>
|
|
17060
|
+
<form method="post" action="/oauth/authorize">
|
|
17061
|
+
${hiddenFields(p)}
|
|
17062
|
+
<input type="hidden" name="action" value="deny" />
|
|
17063
|
+
<button class="ghost" type="submit">Deny</button>
|
|
17064
|
+
</form>`);
|
|
17065
|
+
}
|
|
17066
|
+
function redirectWithError(redirectUri, state, error) {
|
|
17067
|
+
const u = new URL(redirectUri);
|
|
17068
|
+
u.searchParams.set("error", error);
|
|
17069
|
+
if (state) u.searchParams.set("state", state);
|
|
17070
|
+
return new Response(null, { status: 302, headers: { Location: u.toString() } });
|
|
17071
|
+
}
|
|
17072
|
+
var oauthApp = new Hono14();
|
|
17073
|
+
oauthApp.use("*", async (c, next) => {
|
|
17074
|
+
c.header("Access-Control-Allow-Origin", "*");
|
|
17075
|
+
c.header("Access-Control-Allow-Methods", "GET, POST, OPTIONS");
|
|
17076
|
+
c.header("Access-Control-Allow-Headers", "Content-Type, Authorization, mcp-protocol-version");
|
|
17077
|
+
if (c.req.method === "OPTIONS") return c.body(null, 204);
|
|
17078
|
+
return next();
|
|
17079
|
+
});
|
|
17080
|
+
function authServerMetadata() {
|
|
17081
|
+
return {
|
|
17082
|
+
issuer: ISSUER,
|
|
17083
|
+
authorization_endpoint: `${ISSUER}/oauth/authorize`,
|
|
17084
|
+
token_endpoint: `${ISSUER}/oauth/token`,
|
|
17085
|
+
registration_endpoint: `${ISSUER}/oauth/register`,
|
|
17086
|
+
jwks_uri: `${ISSUER}/.well-known/jwks.json`,
|
|
17087
|
+
scopes_supported: [...SUPPORTED_SCOPES],
|
|
17088
|
+
response_types_supported: ["code"],
|
|
17089
|
+
grant_types_supported: ["authorization_code", "refresh_token"],
|
|
17090
|
+
code_challenge_methods_supported: ["S256"],
|
|
17091
|
+
token_endpoint_auth_methods_supported: ["none"]
|
|
17092
|
+
};
|
|
17093
|
+
}
|
|
17094
|
+
function openidConfiguration() {
|
|
17095
|
+
return {
|
|
17096
|
+
...authServerMetadata(),
|
|
17097
|
+
subject_types_supported: ["public"],
|
|
17098
|
+
id_token_signing_alg_values_supported: ["RS256"]
|
|
17099
|
+
};
|
|
17100
|
+
}
|
|
17101
|
+
oauthApp.get("/.well-known/oauth-authorization-server", (c) => {
|
|
17102
|
+
console.log("[oauth-dcr] GET /.well-known/oauth-authorization-server ua=%s", c.req.header("user-agent") ?? "");
|
|
17103
|
+
return c.json(authServerMetadata());
|
|
17104
|
+
});
|
|
17105
|
+
oauthApp.get("/.well-known/oauth-authorization-server/*", (c) => {
|
|
17106
|
+
console.log("[oauth-dcr] GET %s (path-aware AS) ua=%s", c.req.path, c.req.header("user-agent") ?? "");
|
|
17107
|
+
return c.json(authServerMetadata());
|
|
17108
|
+
});
|
|
17109
|
+
oauthApp.get("/.well-known/openid-configuration", (c) => {
|
|
17110
|
+
console.log("[oauth-dcr] GET /.well-known/openid-configuration ua=%s", c.req.header("user-agent") ?? "");
|
|
17111
|
+
return c.json(openidConfiguration());
|
|
17112
|
+
});
|
|
17113
|
+
oauthApp.get("/.well-known/openid-configuration/*", (c) => {
|
|
17114
|
+
console.log("[oauth-dcr] GET %s (path-aware OIDC) ua=%s", c.req.path, c.req.header("user-agent") ?? "");
|
|
17115
|
+
return c.json(openidConfiguration());
|
|
17116
|
+
});
|
|
17117
|
+
oauthApp.get("/.well-known/jwks.json", async (c) => {
|
|
17118
|
+
const { publicJwk } = await getKeys();
|
|
17119
|
+
c.header("Cache-Control", "public, max-age=3600");
|
|
17120
|
+
return c.json({ keys: [publicJwk] });
|
|
17121
|
+
});
|
|
17122
|
+
oauthApp.get("/.well-known/oauth-protected-resource", (c) => {
|
|
17123
|
+
return c.json(protectedResourceMetadata());
|
|
17124
|
+
});
|
|
17125
|
+
oauthApp.get("/.well-known/oauth-protected-resource/*", (c) => {
|
|
17126
|
+
return c.json(protectedResourceMetadata());
|
|
17127
|
+
});
|
|
17128
|
+
oauthApp.post("/oauth/register", async (c) => {
|
|
17129
|
+
const rawBody = await c.req.text();
|
|
17130
|
+
console.log("[oauth-dcr] POST /oauth/register ua=%s ct=%s body=%s", c.req.header("user-agent") ?? "", c.req.header("content-type") ?? "", rawBody.slice(0, 2e3));
|
|
17131
|
+
let body;
|
|
17132
|
+
try {
|
|
17133
|
+
body = rawBody ? JSON.parse(rawBody) : null;
|
|
17134
|
+
} catch {
|
|
17135
|
+
body = null;
|
|
17136
|
+
}
|
|
17137
|
+
if (!body) {
|
|
17138
|
+
console.log("[oauth-dcr] register REJECT reason=body_not_json");
|
|
17139
|
+
return c.json({ error: "invalid_client_metadata", error_description: "Body must be JSON" }, 400);
|
|
17140
|
+
}
|
|
17141
|
+
console.log("[oauth-dcr] register body keys=%s redirect_uris=%s", Object.keys(body).join(","), JSON.stringify(body.redirect_uris));
|
|
17142
|
+
const redirectUris = Array.isArray(body.redirect_uris) ? body.redirect_uris.filter((u) => typeof u === "string") : [];
|
|
17143
|
+
if (redirectUris.length === 0) {
|
|
17144
|
+
console.log("[oauth-dcr] register REJECT reason=no_redirect_uris");
|
|
17145
|
+
return c.json({ error: "invalid_redirect_uri", error_description: "redirect_uris is required" }, 400);
|
|
17146
|
+
}
|
|
17147
|
+
for (const uri of redirectUris) {
|
|
17148
|
+
if (!isAllowedRedirect(uri)) {
|
|
17149
|
+
console.log("[oauth-dcr] register REJECT reason=redirect_uri_not_allowed uri=%s", uri);
|
|
17150
|
+
return c.json({ error: "invalid_redirect_uri", error_description: `redirect_uri not allowed: ${uri}` }, 400);
|
|
17151
|
+
}
|
|
17152
|
+
}
|
|
17153
|
+
const clientName = typeof body.client_name === "string" ? body.client_name : null;
|
|
17154
|
+
const clientId = `client_${randomBytes2(16).toString("hex")}`;
|
|
17155
|
+
await registerClient(clientId, redirectUris, clientName);
|
|
17156
|
+
console.log("[oauth-dcr] register OK client_id=%s redirect_uris=%s", clientId, JSON.stringify(redirectUris));
|
|
17157
|
+
return c.json({
|
|
17158
|
+
client_id: clientId,
|
|
17159
|
+
redirect_uris: redirectUris,
|
|
17160
|
+
client_name: clientName ?? void 0,
|
|
17161
|
+
token_endpoint_auth_method: "none",
|
|
17162
|
+
grant_types: ["authorization_code", "refresh_token"],
|
|
17163
|
+
response_types: ["code"],
|
|
17164
|
+
client_id_issued_at: Math.floor(Date.now() / 1e3)
|
|
17165
|
+
}, 201);
|
|
17166
|
+
});
|
|
17167
|
+
function parseAuthParams(get) {
|
|
17168
|
+
return {
|
|
17169
|
+
response_type: get("response_type") ?? "",
|
|
17170
|
+
client_id: get("client_id") ?? "",
|
|
17171
|
+
redirect_uri: get("redirect_uri") ?? "",
|
|
17172
|
+
code_challenge: get("code_challenge") ?? "",
|
|
17173
|
+
code_challenge_method: get("code_challenge_method") ?? "",
|
|
17174
|
+
scope: get("scope") ?? "",
|
|
17175
|
+
state: get("state") ?? "",
|
|
17176
|
+
resource: get("resource") ?? ""
|
|
17177
|
+
};
|
|
17178
|
+
}
|
|
17179
|
+
async function validateAuthRequest(p) {
|
|
17180
|
+
if (p.response_type !== "code") return { ok: false, status: 400, message: "response_type must be code" };
|
|
17181
|
+
if (!p.client_id) return { ok: false, status: 400, message: "client_id is required" };
|
|
17182
|
+
const client2 = await getClient(p.client_id);
|
|
17183
|
+
if (!client2) return { ok: false, status: 400, message: "unknown client_id" };
|
|
17184
|
+
if (!p.redirect_uri || !client2.redirect_uris.includes(p.redirect_uri)) return { ok: false, status: 400, message: "redirect_uri does not match the registered set" };
|
|
17185
|
+
if (!isAllowedRedirect(p.redirect_uri)) return { ok: false, status: 400, message: "redirect_uri host not allowed" };
|
|
17186
|
+
if (!p.code_challenge || p.code_challenge_method !== "S256") return { ok: false, status: 400, message: "PKCE S256 code_challenge is required" };
|
|
17187
|
+
return { ok: true };
|
|
17188
|
+
}
|
|
17189
|
+
oauthApp.get("/oauth/authorize", async (c) => {
|
|
17190
|
+
const p = parseAuthParams((k) => c.req.query(k));
|
|
17191
|
+
console.log("[oauth-dcr] GET /oauth/authorize client_id=%s redirect_uri=%s scope=%s resource=%s", p.client_id, p.redirect_uri, p.scope, p.resource);
|
|
17192
|
+
const valid = await validateAuthRequest(p);
|
|
17193
|
+
if (!valid.ok) {
|
|
17194
|
+
console.log("[oauth-dcr] authorize REJECT reason=%s", valid.message);
|
|
17195
|
+
return c.text(valid.message, valid.status);
|
|
17196
|
+
}
|
|
17197
|
+
const user = await sessionUser(getCookie(c, "session"));
|
|
17198
|
+
if (!user) return c.html(renderLogin(p));
|
|
17199
|
+
const negotiated = { ...p, scope: negotiateScope(p.scope, user, p.resource) };
|
|
17200
|
+
return c.html(renderConsent(negotiated, user.email));
|
|
17201
|
+
});
|
|
17202
|
+
oauthApp.post("/oauth/authorize", async (c) => {
|
|
17203
|
+
const form = await c.req.parseBody();
|
|
17204
|
+
const get = (k) => {
|
|
17205
|
+
const v = form[k];
|
|
17206
|
+
return typeof v === "string" ? v : void 0;
|
|
17207
|
+
};
|
|
17208
|
+
const p = parseAuthParams(get);
|
|
17209
|
+
const action = get("action") ?? "";
|
|
17210
|
+
const valid = await validateAuthRequest(p);
|
|
17211
|
+
if (!valid.ok) return c.text(valid.message, valid.status);
|
|
17212
|
+
if (action === "login") {
|
|
17213
|
+
const email = (get("email") ?? "").trim().toLowerCase();
|
|
17214
|
+
const password = get("password") ?? "";
|
|
17215
|
+
const user2 = email ? await getUserByEmail(email) : void 0;
|
|
17216
|
+
const okPw = !!(user2 && user2.password_hash && verifyPassword(password, user2.password_hash));
|
|
17217
|
+
console.log("[oauth-dcr] LOGIN email=%s found=%s hasHash=%s pwLen=%d ok=%s", email, !!user2, !!(user2 && user2.password_hash), password.length, okPw);
|
|
17218
|
+
if (!okPw) {
|
|
17219
|
+
return c.html(renderLogin(p, "Invalid email or password."));
|
|
17220
|
+
}
|
|
17221
|
+
setCookie(c, "session", signSession(user2.id), sessionCookieOptions);
|
|
17222
|
+
return c.html(renderConsent({ ...p, scope: negotiateScope(p.scope, user2, p.resource) }, user2.email));
|
|
17223
|
+
}
|
|
17224
|
+
const user = await sessionUser(getCookie(c, "session"));
|
|
17225
|
+
if (!user) return c.html(renderLogin(p, "Your session expired. Please sign in again."));
|
|
17226
|
+
if (action === "deny") return redirectWithError(p.redirect_uri, p.state, "access_denied");
|
|
17227
|
+
if (action !== "approve") return c.text("unsupported action", 400);
|
|
17228
|
+
const scope = negotiateScope(p.scope, user, p.resource);
|
|
17229
|
+
const code = `code_${randomBytes2(32).toString("base64url")}`;
|
|
17230
|
+
const expiresAt = new Date(Date.now() + CODE_TTL_SECONDS * 1e3).toISOString();
|
|
17231
|
+
await putCode({
|
|
17232
|
+
code,
|
|
17233
|
+
client_id: p.client_id,
|
|
17234
|
+
redirect_uri: p.redirect_uri,
|
|
17235
|
+
code_challenge: p.code_challenge,
|
|
17236
|
+
identity: user.email,
|
|
17237
|
+
scope,
|
|
17238
|
+
resource: p.resource || RESOURCE(),
|
|
17239
|
+
expires_at: expiresAt
|
|
17240
|
+
});
|
|
17241
|
+
const u = new URL(p.redirect_uri);
|
|
17242
|
+
u.searchParams.set("code", code);
|
|
17243
|
+
if (p.state) u.searchParams.set("state", p.state);
|
|
17244
|
+
return new Response(null, { status: 302, headers: { Location: u.toString() } });
|
|
17245
|
+
});
|
|
17246
|
+
function pkceMatches(verifier, challenge) {
|
|
17247
|
+
if (!verifier) return false;
|
|
17248
|
+
const computed = createHash3("sha256").update(verifier).digest("base64url");
|
|
17249
|
+
return computed === challenge;
|
|
17250
|
+
}
|
|
17251
|
+
async function mintAccessToken(identity, scope, plan, audience) {
|
|
17252
|
+
const { privateKey, kid } = await getKeys();
|
|
17253
|
+
return new SignJWT({ scope, plan }).setProtectedHeader({ alg: "RS256", kid }).setIssuer(ISSUER).setSubject(identity).setAudience(audience).setIssuedAt().setJti(randomUUID3()).setExpirationTime(`${ACCESS_TTL_SECONDS}s`).sign(privateKey);
|
|
17254
|
+
}
|
|
17255
|
+
function tokenErrorResponse(c, error, description, status) {
|
|
17256
|
+
return c.json({ error, error_description: description }, status);
|
|
17257
|
+
}
|
|
17258
|
+
oauthApp.post("/oauth/token", async (c) => {
|
|
17259
|
+
const form = await c.req.parseBody();
|
|
17260
|
+
const get = (k) => {
|
|
17261
|
+
const v = form[k];
|
|
17262
|
+
return typeof v === "string" ? v : void 0;
|
|
17263
|
+
};
|
|
17264
|
+
const grantType = get("grant_type");
|
|
17265
|
+
if (grantType === "authorization_code") {
|
|
17266
|
+
const code = get("code") ?? "";
|
|
17267
|
+
const clientId = get("client_id") ?? "";
|
|
17268
|
+
const redirectUri = get("redirect_uri") ?? "";
|
|
17269
|
+
const verifier = get("code_verifier") ?? "";
|
|
17270
|
+
const record = await consumeCode(code);
|
|
17271
|
+
if (!record) return tokenErrorResponse(c, "invalid_grant", "Authorization code is invalid or already used", 400);
|
|
17272
|
+
if (new Date(record.expires_at).getTime() < Date.now()) return tokenErrorResponse(c, "invalid_grant", "Authorization code has expired", 400);
|
|
17273
|
+
if (record.client_id !== clientId) return tokenErrorResponse(c, "invalid_grant", "client_id mismatch", 400);
|
|
17274
|
+
if (record.redirect_uri !== redirectUri) return tokenErrorResponse(c, "invalid_grant", "redirect_uri mismatch", 400);
|
|
17275
|
+
if (!pkceMatches(verifier, record.code_challenge)) return tokenErrorResponse(c, "invalid_grant", "PKCE verification failed", 400);
|
|
17276
|
+
const user = await getUserByEmail(record.identity);
|
|
17277
|
+
const plan = user ? resolvePlan(user) : "free";
|
|
17278
|
+
const audience = record.resource ?? RESOURCE();
|
|
17279
|
+
const accessToken = await mintAccessToken(record.identity, record.scope, plan, audience);
|
|
17280
|
+
const refreshToken = `rt_${randomBytes2(40).toString("base64url")}`;
|
|
17281
|
+
await putRefresh({
|
|
17282
|
+
refresh_token: refreshToken,
|
|
17283
|
+
client_id: clientId,
|
|
17284
|
+
identity: record.identity,
|
|
17285
|
+
scope: record.scope,
|
|
17286
|
+
resource: audience,
|
|
17287
|
+
expires_at: new Date(Date.now() + REFRESH_TTL_SECONDS * 1e3).toISOString(),
|
|
17288
|
+
revoked: 0
|
|
17289
|
+
});
|
|
17290
|
+
return c.json({ access_token: accessToken, token_type: "Bearer", expires_in: ACCESS_TTL_SECONDS, refresh_token: refreshToken, scope: record.scope });
|
|
17291
|
+
}
|
|
17292
|
+
if (grantType === "refresh_token") {
|
|
17293
|
+
const refreshToken = get("refresh_token") ?? "";
|
|
17294
|
+
const clientId = get("client_id") ?? "";
|
|
17295
|
+
const record = await getRefresh(refreshToken);
|
|
17296
|
+
if (!record || record.revoked) return tokenErrorResponse(c, "invalid_grant", "Refresh token is invalid or revoked", 400);
|
|
17297
|
+
if (new Date(record.expires_at).getTime() < Date.now()) return tokenErrorResponse(c, "invalid_grant", "Refresh token has expired", 400);
|
|
17298
|
+
if (clientId && record.client_id !== clientId) return tokenErrorResponse(c, "invalid_grant", "client_id mismatch", 400);
|
|
17299
|
+
const user = await getUserByEmail(record.identity);
|
|
17300
|
+
const plan = user ? resolvePlan(user) : "free";
|
|
17301
|
+
const audience = record.resource ?? RESOURCE();
|
|
17302
|
+
const accessToken = await mintAccessToken(record.identity, record.scope, plan, audience);
|
|
17303
|
+
const nextRefresh = `rt_${randomBytes2(40).toString("base64url")}`;
|
|
17304
|
+
await rotateRefresh(refreshToken, {
|
|
17305
|
+
refresh_token: nextRefresh,
|
|
17306
|
+
client_id: record.client_id,
|
|
17307
|
+
identity: record.identity,
|
|
17308
|
+
scope: record.scope,
|
|
17309
|
+
resource: audience,
|
|
17310
|
+
expires_at: new Date(Date.now() + REFRESH_TTL_SECONDS * 1e3).toISOString(),
|
|
17311
|
+
revoked: 0
|
|
17312
|
+
});
|
|
17313
|
+
return c.json({ access_token: accessToken, token_type: "Bearer", expires_in: ACCESS_TTL_SECONDS, refresh_token: nextRefresh, scope: record.scope });
|
|
17314
|
+
}
|
|
17315
|
+
return tokenErrorResponse(c, "unsupported_grant_type", "grant_type must be authorization_code or refresh_token", 400);
|
|
17316
|
+
});
|
|
17317
|
+
|
|
16523
17318
|
// src/api/site-audit-worker.ts
|
|
16524
17319
|
var MAX_CONCURRENT_SITE_AUDIT = 1;
|
|
16525
17320
|
async function drainSiteAuditQueue(budget) {
|
|
@@ -16555,12 +17350,115 @@ function startSiteAuditWorker() {
|
|
|
16555
17350
|
|
|
16556
17351
|
// src/api/server.ts
|
|
16557
17352
|
import { createMiddleware as createMiddleware3 } from "hono/factory";
|
|
16558
|
-
import { getCookie, setCookie, deleteCookie } from "hono/cookie";
|
|
17353
|
+
import { getCookie as getCookie2, setCookie as setCookie2, deleteCookie } from "hono/cookie";
|
|
16559
17354
|
import Stripe2 from "stripe";
|
|
16560
17355
|
|
|
17356
|
+
// src/api/scrape-vault-sink.ts
|
|
17357
|
+
var MAX_DEPOSIT_CHARS = 5e6;
|
|
17358
|
+
var SCRAPE_FALLBACK_PREFIX = "scrape-fallback/";
|
|
17359
|
+
var SCRAPE_BLOB_TTL_MS = 24 * 60 * 60 * 1e3;
|
|
17360
|
+
async function depositScrapeToVault(user, opts) {
|
|
17361
|
+
try {
|
|
17362
|
+
const content = (opts.content ?? "").trim();
|
|
17363
|
+
if (!content) return { deposited: false, error: "empty content" };
|
|
17364
|
+
const clipped = content.length > MAX_DEPOSIT_CHARS ? content.slice(0, MAX_DEPOSIT_CHARS) : content;
|
|
17365
|
+
const { key, error } = await getOrCreateUserMemoryKey(user);
|
|
17366
|
+
if (!key) return { deposited: false, error: error ?? "memory unavailable" };
|
|
17367
|
+
const vault = opts.vault?.trim() || personalVault(user);
|
|
17368
|
+
const title = (opts.title?.trim() || opts.source).slice(0, 200);
|
|
17369
|
+
const res = await memoryCall(
|
|
17370
|
+
"libraryIngestTool",
|
|
17371
|
+
{ title, content: clipped, source: opts.source, vault },
|
|
17372
|
+
key
|
|
17373
|
+
);
|
|
17374
|
+
if (!res.ok) return { deposited: false, vault, error: res.error ?? "ingest failed" };
|
|
17375
|
+
return { deposited: true, vault, noteId: res.noteId, path: res.path, chunks: res.chunks };
|
|
17376
|
+
} catch (err) {
|
|
17377
|
+
return { deposited: false, error: err?.message ?? "deposit failed" };
|
|
17378
|
+
}
|
|
17379
|
+
}
|
|
17380
|
+
async function uploadFallbackBlob(opts) {
|
|
17381
|
+
try {
|
|
17382
|
+
const content = (opts.content ?? "").trim();
|
|
17383
|
+
if (!content) return null;
|
|
17384
|
+
const slug = (slugify(opts.title || opts.source) || "scrape").slice(0, 60);
|
|
17385
|
+
const key = `${SCRAPE_FALLBACK_PREFIX}${slug}.md`;
|
|
17386
|
+
const doc = `# ${opts.title}
|
|
17387
|
+
Source: ${opts.source}
|
|
17388
|
+
|
|
17389
|
+
${content}`;
|
|
17390
|
+
const stored = await getBlobStore().put(key, doc, "text/markdown");
|
|
17391
|
+
return { fileUrl: stored.url, fileExpiresAt: new Date(Date.now() + SCRAPE_BLOB_TTL_MS).toISOString() };
|
|
17392
|
+
} catch {
|
|
17393
|
+
return null;
|
|
17394
|
+
}
|
|
17395
|
+
}
|
|
17396
|
+
async function persistScrapeBody(user, opts) {
|
|
17397
|
+
const deposit = await depositScrapeToVault(user, opts);
|
|
17398
|
+
if (deposit.deposited) return deposit;
|
|
17399
|
+
const fallback = await uploadFallbackBlob(opts);
|
|
17400
|
+
if (!fallback) return deposit;
|
|
17401
|
+
return { ...deposit, ...fallback };
|
|
17402
|
+
}
|
|
17403
|
+
|
|
17404
|
+
// src/api/scrape-blob-cleanup.ts
|
|
17405
|
+
import { readdir, stat, unlink } from "fs/promises";
|
|
17406
|
+
import { homedir as homedir4 } from "os";
|
|
17407
|
+
import { join as join7 } from "path";
|
|
17408
|
+
async function cleanupVercel(token, cutoff) {
|
|
17409
|
+
const { list, del } = await import("@vercel/blob");
|
|
17410
|
+
let cursor;
|
|
17411
|
+
let deleted = 0;
|
|
17412
|
+
do {
|
|
17413
|
+
const res = await list({ prefix: SCRAPE_FALLBACK_PREFIX, cursor, token });
|
|
17414
|
+
const expired = res.blobs.filter((b) => new Date(b.uploadedAt).getTime() < cutoff);
|
|
17415
|
+
if (expired.length) {
|
|
17416
|
+
await del(expired.map((b) => b.url), { token });
|
|
17417
|
+
deleted += expired.length;
|
|
17418
|
+
}
|
|
17419
|
+
cursor = res.hasMore ? res.cursor : void 0;
|
|
17420
|
+
} while (cursor);
|
|
17421
|
+
return { deleted, store: "vercel-blob" };
|
|
17422
|
+
}
|
|
17423
|
+
async function cleanupLocal(cutoff) {
|
|
17424
|
+
const baseDir = process.env.MCP_SCRAPER_OUTPUT_DIR?.trim() || join7(homedir4(), "Downloads", "mcp-scraper");
|
|
17425
|
+
const dir = join7(baseDir, "blobs", SCRAPE_FALLBACK_PREFIX.replace(/\/$/, ""));
|
|
17426
|
+
let deleted = 0;
|
|
17427
|
+
let entries;
|
|
17428
|
+
try {
|
|
17429
|
+
entries = await readdir(dir);
|
|
17430
|
+
} catch {
|
|
17431
|
+
return { deleted: 0, store: "local" };
|
|
17432
|
+
}
|
|
17433
|
+
for (const name of entries) {
|
|
17434
|
+
const path5 = join7(dir, name);
|
|
17435
|
+
try {
|
|
17436
|
+
const s = await stat(path5);
|
|
17437
|
+
if (s.isFile() && s.mtimeMs < cutoff) {
|
|
17438
|
+
await unlink(path5);
|
|
17439
|
+
deleted++;
|
|
17440
|
+
}
|
|
17441
|
+
} catch {
|
|
17442
|
+
}
|
|
17443
|
+
}
|
|
17444
|
+
return { deleted, store: "local" };
|
|
17445
|
+
}
|
|
17446
|
+
async function cleanupExpiredScrapeBlobs(maxAgeMs = SCRAPE_BLOB_TTL_MS) {
|
|
17447
|
+
const cutoff = Date.now() - maxAgeMs;
|
|
17448
|
+
try {
|
|
17449
|
+
const token = process.env.BLOB_READ_WRITE_TOKEN;
|
|
17450
|
+
return token ? await cleanupVercel(token, cutoff) : await cleanupLocal(cutoff);
|
|
17451
|
+
} catch {
|
|
17452
|
+
return { deleted: 0, store: "none" };
|
|
17453
|
+
}
|
|
17454
|
+
}
|
|
17455
|
+
|
|
16561
17456
|
// src/api/credit-operations.ts
|
|
16562
|
-
async function grantSignupCredit(
|
|
16563
|
-
return;
|
|
17457
|
+
async function grantSignupCredit(userId) {
|
|
17458
|
+
if (!FREE_SIGNUP_MC || FREE_SIGNUP_MC <= 0) return;
|
|
17459
|
+
const already = await ledgerExistsForOperation(userId, LedgerOperation.SIGNUP_GRANT);
|
|
17460
|
+
if (already) return;
|
|
17461
|
+
await creditMc(userId, FREE_SIGNUP_MC, LedgerOperation.SIGNUP_GRANT, "Welcome credits");
|
|
16564
17462
|
}
|
|
16565
17463
|
async function applyMonthlyFreeRefresh(user) {
|
|
16566
17464
|
return user;
|
|
@@ -16598,46 +17496,168 @@ async function getFreeCreditBreakdown(userId) {
|
|
|
16598
17496
|
};
|
|
16599
17497
|
}
|
|
16600
17498
|
|
|
16601
|
-
// src/api/
|
|
16602
|
-
import {
|
|
16603
|
-
var
|
|
16604
|
-
function
|
|
16605
|
-
|
|
16606
|
-
|
|
16607
|
-
if (
|
|
16608
|
-
|
|
17499
|
+
// src/api/memory-db.ts
|
|
17500
|
+
import { neon } from "@neondatabase/serverless";
|
|
17501
|
+
var _sql = null;
|
|
17502
|
+
function sql() {
|
|
17503
|
+
if (_sql) return _sql;
|
|
17504
|
+
const url = process.env.MEMORY_DATABASE_URL?.trim();
|
|
17505
|
+
if (!url) throw new Error("MEMORY_DATABASE_URL is not set (paa-crawler needs mcp-memory Neon URL for reads)");
|
|
17506
|
+
_sql = neon(url);
|
|
17507
|
+
return _sql;
|
|
17508
|
+
}
|
|
17509
|
+
function physicalVault(owner, vault) {
|
|
17510
|
+
return `${owner} ${vault}`;
|
|
17511
|
+
}
|
|
17512
|
+
function isUnlimited(plan) {
|
|
17513
|
+
return plan === "unlimited";
|
|
17514
|
+
}
|
|
17515
|
+
function quotaBytesForPlan(plan) {
|
|
17516
|
+
return MEMORY_PLAN_QUOTA[plan] ?? MEMORY_PLAN_QUOTA.free;
|
|
17517
|
+
}
|
|
17518
|
+
function toGb(bytes) {
|
|
17519
|
+
return Math.round(bytes / 1e9 * 100) / 100;
|
|
17520
|
+
}
|
|
17521
|
+
async function entitledVaultRows(identity) {
|
|
17522
|
+
const rows = await sql().query(
|
|
17523
|
+
`SELECT vault, owner_identity FROM mem_vault_registry WHERE grantee = $1 ORDER BY created_at ASC`,
|
|
17524
|
+
[identity]
|
|
17525
|
+
);
|
|
17526
|
+
const seen = /* @__PURE__ */ new Map();
|
|
17527
|
+
for (const r of rows) {
|
|
17528
|
+
if (seen.has(r.vault)) continue;
|
|
17529
|
+
seen.set(r.vault, { vault: r.vault, owner: r.owner_identity, physical: physicalVault(r.owner_identity, r.vault) });
|
|
17530
|
+
}
|
|
17531
|
+
return [...seen.values()];
|
|
17532
|
+
}
|
|
17533
|
+
async function storageByPhysical(physicals) {
|
|
17534
|
+
if (physicals.length === 0) return /* @__PURE__ */ new Map();
|
|
17535
|
+
const rows = await sql().query(
|
|
17536
|
+
`SELECT vault, COUNT(*)::int AS notes, COALESCE(SUM(octet_length(content)), 0)::bigint AS bytes
|
|
17537
|
+
FROM mem_notes WHERE vault = ANY($1) GROUP BY vault`,
|
|
17538
|
+
[physicals]
|
|
17539
|
+
);
|
|
17540
|
+
const m = /* @__PURE__ */ new Map();
|
|
17541
|
+
for (const r of rows) m.set(r.vault, { notes: Number(r.notes), bytes: Number(r.bytes) });
|
|
17542
|
+
return m;
|
|
17543
|
+
}
|
|
17544
|
+
async function dbListVaults(identity) {
|
|
17545
|
+
const ents = await entitledVaultRows(identity);
|
|
17546
|
+
const storage = await storageByPhysical(ents.map((e) => e.physical));
|
|
17547
|
+
return ents.map((e) => {
|
|
17548
|
+
const st = storage.get(e.physical);
|
|
17549
|
+
const isOwner = e.owner === identity;
|
|
17550
|
+
return {
|
|
17551
|
+
vault: e.vault,
|
|
17552
|
+
role: isOwner ? "owner" : "shared",
|
|
17553
|
+
...isOwner ? {} : { sharedBy: e.owner },
|
|
17554
|
+
notes: st?.notes ?? 0,
|
|
17555
|
+
bytes: st?.bytes ?? 0
|
|
17556
|
+
};
|
|
17557
|
+
});
|
|
16609
17558
|
}
|
|
16610
|
-
|
|
16611
|
-
|
|
16612
|
-
|
|
16613
|
-
|
|
16614
|
-
|
|
16615
|
-
|
|
16616
|
-
|
|
16617
|
-
|
|
17559
|
+
async function dbStorage(identity, plan) {
|
|
17560
|
+
const ents = await entitledVaultRows(identity);
|
|
17561
|
+
const storage = await storageByPhysical(ents.map((e) => e.physical));
|
|
17562
|
+
let totalBytes = 0;
|
|
17563
|
+
const perVault = ents.map((e) => {
|
|
17564
|
+
const st = storage.get(e.physical);
|
|
17565
|
+
const bytes = st?.bytes ?? 0;
|
|
17566
|
+
totalBytes += bytes;
|
|
17567
|
+
return { vault: e.vault, bytes, gb: toGb(bytes), notes: st?.notes ?? 0 };
|
|
17568
|
+
});
|
|
17569
|
+
const unlimited = isUnlimited(plan);
|
|
17570
|
+
const quotaBytes = unlimited ? null : quotaBytesForPlan(plan);
|
|
17571
|
+
return {
|
|
17572
|
+
ok: true,
|
|
17573
|
+
totalBytes,
|
|
17574
|
+
totalGb: toGb(totalBytes),
|
|
17575
|
+
quotaBytes,
|
|
17576
|
+
quotaGb: quotaBytes == null ? null : toGb(quotaBytes),
|
|
17577
|
+
unlimited,
|
|
17578
|
+
perVault
|
|
17579
|
+
};
|
|
16618
17580
|
}
|
|
16619
|
-
function
|
|
16620
|
-
const
|
|
16621
|
-
|
|
16622
|
-
|
|
17581
|
+
async function resolvePhysical(identity, vault) {
|
|
17582
|
+
const rows = await sql().query(
|
|
17583
|
+
`SELECT owner_identity FROM mem_vault_registry WHERE grantee = $1 AND vault = $2 LIMIT 1`,
|
|
17584
|
+
[identity, vault]
|
|
17585
|
+
);
|
|
17586
|
+
if (!rows[0]) return null;
|
|
17587
|
+
return physicalVault(rows[0].owner_identity, vault);
|
|
17588
|
+
}
|
|
17589
|
+
async function dbNotes(identity, vault) {
|
|
17590
|
+
const physical = await resolvePhysical(identity, vault);
|
|
17591
|
+
if (!physical) return [];
|
|
17592
|
+
const rows = await sql().query(
|
|
17593
|
+
`SELECT path, title, kind, source, updated_at, octet_length(content) AS bytes, props
|
|
17594
|
+
FROM mem_notes WHERE vault = $1 ORDER BY updated_at DESC`,
|
|
17595
|
+
[physical]
|
|
17596
|
+
);
|
|
17597
|
+
return rows.map((r) => ({
|
|
17598
|
+
path: r.path,
|
|
17599
|
+
title: r.title,
|
|
17600
|
+
kind: r.kind,
|
|
17601
|
+
source: r.source,
|
|
17602
|
+
updatedAt: r.updated_at,
|
|
17603
|
+
bytes: Number(r.bytes),
|
|
17604
|
+
props: r.props && typeof r.props === "object" ? r.props : {}
|
|
17605
|
+
}));
|
|
16623
17606
|
}
|
|
16624
|
-
function
|
|
16625
|
-
const
|
|
16626
|
-
if (
|
|
16627
|
-
const
|
|
16628
|
-
|
|
16629
|
-
|
|
16630
|
-
|
|
16631
|
-
|
|
16632
|
-
|
|
17607
|
+
async function dbNote(identity, vault, path5) {
|
|
17608
|
+
const physical = await resolvePhysical(identity, vault);
|
|
17609
|
+
if (!physical) return null;
|
|
17610
|
+
const rows = await sql().query(
|
|
17611
|
+
`SELECT path, title, content, kind, source, updated_at, captured_at
|
|
17612
|
+
FROM mem_notes WHERE vault = $1 AND path = $2 LIMIT 1`,
|
|
17613
|
+
[physical, path5]
|
|
17614
|
+
);
|
|
17615
|
+
const r = rows[0];
|
|
17616
|
+
if (!r) return null;
|
|
17617
|
+
return {
|
|
17618
|
+
path: r.path,
|
|
17619
|
+
title: r.title,
|
|
17620
|
+
content: r.content,
|
|
17621
|
+
kind: r.kind,
|
|
17622
|
+
source: r.source,
|
|
17623
|
+
updatedAt: r.updated_at,
|
|
17624
|
+
capturedAt: r.captured_at
|
|
17625
|
+
};
|
|
17626
|
+
}
|
|
17627
|
+
var FREE_COST_CAP_USD = 1;
|
|
17628
|
+
async function dbUsage(identity, plan) {
|
|
17629
|
+
const now = /* @__PURE__ */ new Date();
|
|
17630
|
+
const period = `${now.getUTCFullYear()}-${String(now.getUTCMonth() + 1).padStart(2, "0")}`;
|
|
17631
|
+
const rows = await sql().query(
|
|
17632
|
+
`SELECT source, COALESCE(SUM(cost_usd), 0)::float8 AS total
|
|
17633
|
+
FROM mem_usage_ledger WHERE identity = $1 AND period = $2 GROUP BY source`,
|
|
17634
|
+
[identity, period]
|
|
17635
|
+
);
|
|
17636
|
+
const bySource = {};
|
|
17637
|
+
let costUsd = 0;
|
|
17638
|
+
for (const r of rows) {
|
|
17639
|
+
const v = Number(r.total);
|
|
17640
|
+
bySource[r.source] = v;
|
|
17641
|
+
costUsd += v;
|
|
17642
|
+
}
|
|
17643
|
+
return {
|
|
17644
|
+
ok: true,
|
|
17645
|
+
period,
|
|
17646
|
+
plan,
|
|
17647
|
+
costUsd,
|
|
17648
|
+
bySource,
|
|
17649
|
+
freeCapUsd: FREE_COST_CAP_USD,
|
|
17650
|
+
freeCapReached: plan === "free" && costUsd >= FREE_COST_CAP_USD,
|
|
17651
|
+
unlimited: isUnlimited(plan)
|
|
17652
|
+
};
|
|
16633
17653
|
}
|
|
16634
17654
|
|
|
16635
17655
|
// src/api/server.ts
|
|
16636
|
-
var
|
|
16637
|
-
var isProduction2 =
|
|
16638
|
-
var
|
|
17656
|
+
var secureCookies2 = process.env.NODE_ENV === "production" || process.env.VERCEL === "1";
|
|
17657
|
+
var isProduction2 = secureCookies2;
|
|
17658
|
+
var sessionCookieOptions2 = {
|
|
16639
17659
|
httpOnly: true,
|
|
16640
|
-
secure:
|
|
17660
|
+
secure: secureCookies2,
|
|
16641
17661
|
path: "/",
|
|
16642
17662
|
maxAge: 60 * 60 * 24 * 30,
|
|
16643
17663
|
sameSite: "Strict"
|
|
@@ -16698,7 +17718,7 @@ var adminAuth = createMiddleware3(async (c, next) => {
|
|
|
16698
17718
|
return next();
|
|
16699
17719
|
});
|
|
16700
17720
|
var sessionAuth = createMiddleware3(async (c, next) => {
|
|
16701
|
-
const token =
|
|
17721
|
+
const token = getCookie2(c, "session");
|
|
16702
17722
|
if (!token) return c.json({ error: "Not authenticated" }, 401);
|
|
16703
17723
|
const userId = verifySession(token);
|
|
16704
17724
|
if (!userId) return c.json({ error: "Session expired" }, 401);
|
|
@@ -16709,7 +17729,7 @@ var sessionAuth = createMiddleware3(async (c, next) => {
|
|
|
16709
17729
|
c.set("sessionUser", { ...refreshed, balance_mc: balanceMc });
|
|
16710
17730
|
return next();
|
|
16711
17731
|
});
|
|
16712
|
-
var app = new
|
|
17732
|
+
var app = new Hono15();
|
|
16713
17733
|
var STRIPE_API_VERSION = "2026-02-25.clover";
|
|
16714
17734
|
function requireStripeSecret() {
|
|
16715
17735
|
const secret2 = process.env.STRIPE_SECRET_KEY?.trim();
|
|
@@ -16737,6 +17757,33 @@ app.use("*", async (c, next) => {
|
|
|
16737
17757
|
c.header("X-Content-Type-Options", "nosniff");
|
|
16738
17758
|
c.header("Referrer-Policy", "same-origin");
|
|
16739
17759
|
});
|
|
17760
|
+
function opForCostPath(p) {
|
|
17761
|
+
if (p === "/harvest" || p === "/harvest/sync") return "harvest";
|
|
17762
|
+
if (p === "/extract-url") return "page_scrape";
|
|
17763
|
+
if (p === "/extract-site") return "extract_site";
|
|
17764
|
+
if (p === "/map-urls") return "url_map";
|
|
17765
|
+
if (p === "/youtube/harvest") return "yt_channel";
|
|
17766
|
+
if (p === "/youtube/transcribe") return "yt_transcription";
|
|
17767
|
+
if (p === "/maps/search") return "maps_search";
|
|
17768
|
+
if (p === "/maps/place") return "maps_place";
|
|
17769
|
+
if (p.startsWith("/facebook/")) {
|
|
17770
|
+
if (p.endsWith("transcribe")) return "fb_transcribe";
|
|
17771
|
+
if (p.endsWith("/search")) return "fb_search";
|
|
17772
|
+
return "fb_ad";
|
|
17773
|
+
}
|
|
17774
|
+
if (p.startsWith("/instagram/")) return "instagram";
|
|
17775
|
+
if (p.startsWith("/api/internal/site-architecture-auditor")) return "audit_site";
|
|
17776
|
+
if (p.startsWith("/agent/")) return "browser_session";
|
|
17777
|
+
return null;
|
|
17778
|
+
}
|
|
17779
|
+
app.use("*", (c, next) => {
|
|
17780
|
+
const op = opForCostPath(new URL(c.req.url).pathname);
|
|
17781
|
+
if (!op) return next();
|
|
17782
|
+
const probeRunId = c.req.header("x-cost-probe-id") ?? null;
|
|
17783
|
+
const forceHeadless = c.req.header("x-kernel-headless") === "true";
|
|
17784
|
+
return runWithCostContext({ op, probeRunId, forceHeadless }, () => next());
|
|
17785
|
+
});
|
|
17786
|
+
void migrateCostTelemetry();
|
|
16740
17787
|
app.post("/auth/register", requireAllowedOrigin, async (c) => {
|
|
16741
17788
|
const { email, password } = await c.req.json();
|
|
16742
17789
|
const normalizedEmail = email?.trim().toLowerCase();
|
|
@@ -16768,8 +17815,16 @@ app.post("/auth/register", requireAllowedOrigin, async (c) => {
|
|
|
16768
17815
|
}
|
|
16769
17816
|
const newKey = await rotateApiKey(user.id);
|
|
16770
17817
|
await grantSignupCredit(user.id);
|
|
17818
|
+
void (async () => {
|
|
17819
|
+
try {
|
|
17820
|
+
const full = await getUserById(user.id);
|
|
17821
|
+
if (full) await provisionMemoryForUser(full);
|
|
17822
|
+
} catch (err) {
|
|
17823
|
+
console.warn("[auth/register] memory provision failed (will retry on first write):", err instanceof Error ? err.message : String(err));
|
|
17824
|
+
}
|
|
17825
|
+
})();
|
|
16771
17826
|
const token = signSession(user.id);
|
|
16772
|
-
|
|
17827
|
+
setCookie2(c, "session", token, sessionCookieOptions2);
|
|
16773
17828
|
return c.json({ email: user.email, api_key: newKey }, 201);
|
|
16774
17829
|
} catch (err) {
|
|
16775
17830
|
const msg = err instanceof Error ? err.message : "";
|
|
@@ -16788,11 +17843,11 @@ app.post("/auth/login", requireAllowedOrigin, async (c) => {
|
|
|
16788
17843
|
return c.json({ error: "Invalid email or password" }, 401);
|
|
16789
17844
|
}
|
|
16790
17845
|
const token = signSession(user.id);
|
|
16791
|
-
|
|
17846
|
+
setCookie2(c, "session", token, sessionCookieOptions2);
|
|
16792
17847
|
return c.json({ email: user.email, api_key: user.key_active ? user.api_key : null });
|
|
16793
17848
|
});
|
|
16794
17849
|
app.post("/auth/logout", requireAllowedOrigin, (c) => {
|
|
16795
|
-
deleteCookie(c, "session", { path: "/", secure:
|
|
17850
|
+
deleteCookie(c, "session", { path: "/", secure: secureCookies2, sameSite: "Strict" });
|
|
16796
17851
|
return c.json({ ok: true });
|
|
16797
17852
|
});
|
|
16798
17853
|
app.post("/auth/forgot-password", requireAllowedOrigin, async (c) => {
|
|
@@ -16846,16 +17901,16 @@ app.get("/catalog", (c) => {
|
|
|
16846
17901
|
return c.json(CATALOG);
|
|
16847
17902
|
});
|
|
16848
17903
|
app.get("/me", async (c) => {
|
|
16849
|
-
const token =
|
|
17904
|
+
const token = getCookie2(c, "session");
|
|
16850
17905
|
if (!token) return c.json({ authenticated: false });
|
|
16851
17906
|
const userId = verifySession(token);
|
|
16852
17907
|
if (!userId) {
|
|
16853
|
-
deleteCookie(c, "session", { path: "/", secure:
|
|
17908
|
+
deleteCookie(c, "session", { path: "/", secure: secureCookies2, sameSite: "Strict" });
|
|
16854
17909
|
return c.json({ authenticated: false });
|
|
16855
17910
|
}
|
|
16856
17911
|
const foundUser = await getUserById(userId);
|
|
16857
17912
|
if (!foundUser) {
|
|
16858
|
-
deleteCookie(c, "session", { path: "/", secure:
|
|
17913
|
+
deleteCookie(c, "session", { path: "/", secure: secureCookies2, sameSite: "Strict" });
|
|
16859
17914
|
return c.json({ authenticated: false });
|
|
16860
17915
|
}
|
|
16861
17916
|
const refreshed = await applyMonthlyFreeRefresh(foundUser);
|
|
@@ -16874,6 +17929,8 @@ app.get("/me", async (c) => {
|
|
|
16874
17929
|
has_concurrency_sub: !!user.concurrency_stripe_sub_id,
|
|
16875
17930
|
subscription_tier: user.subscription_tier,
|
|
16876
17931
|
subscription_concurrency: user.subscription_concurrency,
|
|
17932
|
+
memory_plan: user.memory_plan ?? "free",
|
|
17933
|
+
has_memory_sub: !!user.memory_subscription_id,
|
|
16877
17934
|
...stats
|
|
16878
17935
|
});
|
|
16879
17936
|
});
|
|
@@ -16887,6 +17944,227 @@ app.delete("/api-key", requireAllowedOrigin, sessionAuth, async (c) => {
|
|
|
16887
17944
|
await revokeApiKey(user.id);
|
|
16888
17945
|
return c.json({ ok: true });
|
|
16889
17946
|
});
|
|
17947
|
+
function maybeProvisionInBackground(user, vaultCount) {
|
|
17948
|
+
if (vaultCount > 0 || user.memory_provisioned === 1) return;
|
|
17949
|
+
void (async () => {
|
|
17950
|
+
try {
|
|
17951
|
+
if (await getMemoryProvisioned(user.id)) return;
|
|
17952
|
+
await provisionMemoryForUser(user);
|
|
17953
|
+
} catch (err) {
|
|
17954
|
+
console.warn("[memory] background provision failed:", err instanceof Error ? err.message : String(err));
|
|
17955
|
+
}
|
|
17956
|
+
})();
|
|
17957
|
+
}
|
|
17958
|
+
app.get("/memory/overview", auth2, async (c) => {
|
|
17959
|
+
const user = c.get("user");
|
|
17960
|
+
const identity = resolveMemoryIdentity(user);
|
|
17961
|
+
const plan = memoryPlanForReads(user);
|
|
17962
|
+
const requestedVault = c.req.query("vault")?.trim() || null;
|
|
17963
|
+
try {
|
|
17964
|
+
const [vaults, storage, usage] = await Promise.all([
|
|
17965
|
+
dbListVaults(identity),
|
|
17966
|
+
dbStorage(identity, plan),
|
|
17967
|
+
dbUsage(identity, plan)
|
|
17968
|
+
]);
|
|
17969
|
+
maybeProvisionInBackground(user, vaults.length);
|
|
17970
|
+
const activeVault = requestedVault && vaults.some((v) => v.vault === requestedVault) ? requestedVault : vaults[0]?.vault ?? null;
|
|
17971
|
+
const notes = activeVault ? await dbNotes(identity, activeVault) : [];
|
|
17972
|
+
const billing = {
|
|
17973
|
+
ok: true,
|
|
17974
|
+
memory_plan: user.memory_plan ?? "free",
|
|
17975
|
+
has_memory_sub: !!user.memory_subscription_id,
|
|
17976
|
+
free_quota_gb: MEMORY_PLAN_QUOTA.free / 1e9,
|
|
17977
|
+
plans: Object.values(MEMORY_PLANS).map((p) => ({ plan: p.plan, label: p.label, price_id: p.price_id, interval: p.interval, monthly_usd: p.monthly_usd, quota_gb: p.quota_bytes / 1e9 }))
|
|
17978
|
+
};
|
|
17979
|
+
return c.json({ ok: true, vaults, storage, activeVault, notes, usage, billing });
|
|
17980
|
+
} catch (err) {
|
|
17981
|
+
const message = err instanceof Error ? err.message : "memory overview failed";
|
|
17982
|
+
console.error("[memory/overview]", message);
|
|
17983
|
+
return c.json({ ok: false, error: message }, 502);
|
|
17984
|
+
}
|
|
17985
|
+
});
|
|
17986
|
+
app.get("/memory/vaults", auth2, async (c) => {
|
|
17987
|
+
const user = c.get("user");
|
|
17988
|
+
try {
|
|
17989
|
+
const vaults = await dbListVaults(resolveMemoryIdentity(user));
|
|
17990
|
+
maybeProvisionInBackground(user, vaults.length);
|
|
17991
|
+
return c.json({ ok: true, vaults });
|
|
17992
|
+
} catch (err) {
|
|
17993
|
+
return c.json({ ok: false, error: err instanceof Error ? err.message : "could not load vaults" }, 502);
|
|
17994
|
+
}
|
|
17995
|
+
});
|
|
17996
|
+
app.get("/memory/connect", auth2, async (c) => {
|
|
17997
|
+
const { key, error } = await getOrCreateUserMemoryKey(c.get("user"));
|
|
17998
|
+
if (!key) return c.json({ error: error ?? "memory unavailable" }, 502);
|
|
17999
|
+
const command = `MCP_MEMORY_KEY=${key} npx -y -p @vilovieta/mcp-memory@latest mcp-memory`;
|
|
18000
|
+
const config = {
|
|
18001
|
+
mcpServers: {
|
|
18002
|
+
"mcp-memory": {
|
|
18003
|
+
command: "npx",
|
|
18004
|
+
args: ["-y", "-p", "@vilovieta/mcp-memory@latest", "mcp-memory"],
|
|
18005
|
+
env: { MCP_MEMORY_KEY: key }
|
|
18006
|
+
}
|
|
18007
|
+
}
|
|
18008
|
+
};
|
|
18009
|
+
return c.json({ ok: true, command, config });
|
|
18010
|
+
});
|
|
18011
|
+
app.post("/memory/vault", auth2, async (c) => {
|
|
18012
|
+
const { key, error } = await getOrCreateUserMemoryKey(c.get("user"));
|
|
18013
|
+
if (!key) return c.json({ error: error ?? "memory unavailable" }, 502);
|
|
18014
|
+
const body = await c.req.json().catch(() => ({}));
|
|
18015
|
+
const vault = body.vault?.trim();
|
|
18016
|
+
if (!vault) return c.json({ error: "vault name required" }, 400);
|
|
18017
|
+
const result = await memoryCall("addVaultTool", { vault }, key);
|
|
18018
|
+
return c.json(result, result.ok ? 200 : 502);
|
|
18019
|
+
});
|
|
18020
|
+
app.get("/memory/notes", auth2, async (c) => {
|
|
18021
|
+
const user = c.get("user");
|
|
18022
|
+
const identity = resolveMemoryIdentity(user);
|
|
18023
|
+
try {
|
|
18024
|
+
let vault = c.req.query("vault")?.trim() || null;
|
|
18025
|
+
if (!vault) {
|
|
18026
|
+
const vaults = await dbListVaults(identity);
|
|
18027
|
+
vault = vaults[0]?.vault ?? null;
|
|
18028
|
+
}
|
|
18029
|
+
if (!vault) return c.json({ ok: true, notes: [], vault: null });
|
|
18030
|
+
const notes = await dbNotes(identity, vault);
|
|
18031
|
+
return c.json({ ok: true, vault, notes });
|
|
18032
|
+
} catch (err) {
|
|
18033
|
+
return c.json({ ok: false, error: err instanceof Error ? err.message : "could not load notes" }, 502);
|
|
18034
|
+
}
|
|
18035
|
+
});
|
|
18036
|
+
app.get("/memory/note", auth2, async (c) => {
|
|
18037
|
+
const user = c.get("user");
|
|
18038
|
+
const vault = c.req.query("vault")?.trim();
|
|
18039
|
+
const path5 = c.req.query("path");
|
|
18040
|
+
if (!path5) return c.json({ error: "path required" }, 400);
|
|
18041
|
+
if (!vault) return c.json({ ok: false, error: "vault required" }, 400);
|
|
18042
|
+
try {
|
|
18043
|
+
const note = await dbNote(resolveMemoryIdentity(user), vault, path5);
|
|
18044
|
+
if (!note) return c.json({ ok: false, error: "note not found" }, 404);
|
|
18045
|
+
return c.json({ ok: true, note });
|
|
18046
|
+
} catch (err) {
|
|
18047
|
+
return c.json({ ok: false, error: err instanceof Error ? err.message : "could not load note" }, 502);
|
|
18048
|
+
}
|
|
18049
|
+
});
|
|
18050
|
+
app.put("/memory/note", auth2, async (c) => {
|
|
18051
|
+
const { key, error } = await getOrCreateUserMemoryKey(c.get("user"));
|
|
18052
|
+
if (!key) return c.json({ error: error ?? "memory unavailable" }, 502);
|
|
18053
|
+
const body = await c.req.json().catch(() => ({}));
|
|
18054
|
+
if (!body.path) return c.json({ error: "path required" }, 400);
|
|
18055
|
+
const result = await memoryCall("putTool", { vault: body.vault, path: body.path, title: body.title, content: body.content }, key);
|
|
18056
|
+
return c.json(result, result.ok ? 200 : 502);
|
|
18057
|
+
});
|
|
18058
|
+
app.delete("/memory/note", auth2, async (c) => {
|
|
18059
|
+
const { key, error } = await getOrCreateUserMemoryKey(c.get("user"));
|
|
18060
|
+
if (!key) return c.json({ error: error ?? "memory unavailable" }, 502);
|
|
18061
|
+
const vault = c.req.query("vault");
|
|
18062
|
+
const path5 = c.req.query("path");
|
|
18063
|
+
if (!path5) return c.json({ error: "path required" }, 400);
|
|
18064
|
+
const result = await memoryCall("deleteNoteTool", { vault, path: path5 }, key);
|
|
18065
|
+
return c.json(result, result.ok ? 200 : 502);
|
|
18066
|
+
});
|
|
18067
|
+
app.get("/memory/note/download", auth2, async (c) => {
|
|
18068
|
+
const user = c.get("user");
|
|
18069
|
+
const vault = c.req.query("vault")?.trim();
|
|
18070
|
+
const path5 = c.req.query("path");
|
|
18071
|
+
if (!path5) return c.json({ error: "path required" }, 400);
|
|
18072
|
+
if (!vault) return c.json({ error: "vault required" }, 400);
|
|
18073
|
+
const note = await dbNote(resolveMemoryIdentity(user), vault, path5);
|
|
18074
|
+
if (!note) return c.json({ error: "note not found" }, 404);
|
|
18075
|
+
const filename = (path5.split("/").pop() || "note").replace(/"/g, "");
|
|
18076
|
+
return new Response(note.content ?? "", {
|
|
18077
|
+
status: 200,
|
|
18078
|
+
headers: {
|
|
18079
|
+
"Content-Type": "text/markdown; charset=utf-8",
|
|
18080
|
+
"Content-Disposition": `attachment; filename="${filename.endsWith(".md") ? filename : filename + ".md"}"`
|
|
18081
|
+
}
|
|
18082
|
+
});
|
|
18083
|
+
});
|
|
18084
|
+
app.get("/memory/storage", auth2, async (c) => {
|
|
18085
|
+
const user = c.get("user");
|
|
18086
|
+
try {
|
|
18087
|
+
const result = await dbStorage(resolveMemoryIdentity(user), memoryPlanForReads(user));
|
|
18088
|
+
return c.json(result);
|
|
18089
|
+
} catch (err) {
|
|
18090
|
+
return c.json({ ok: false, error: err instanceof Error ? err.message : "could not load storage" }, 502);
|
|
18091
|
+
}
|
|
18092
|
+
});
|
|
18093
|
+
app.get("/memory/billing", auth2, async (c) => {
|
|
18094
|
+
const user = c.get("user");
|
|
18095
|
+
const plans = Object.values(MEMORY_PLANS).map((p) => ({ plan: p.plan, label: p.label, price_id: p.price_id, interval: p.interval, monthly_usd: p.monthly_usd, quota_gb: p.quota_bytes / 1e9 }));
|
|
18096
|
+
return c.json({
|
|
18097
|
+
ok: true,
|
|
18098
|
+
memory_plan: user.memory_plan ?? "free",
|
|
18099
|
+
has_memory_sub: !!user.memory_subscription_id,
|
|
18100
|
+
free_quota_gb: MEMORY_PLAN_QUOTA.free / 1e9,
|
|
18101
|
+
plans
|
|
18102
|
+
});
|
|
18103
|
+
});
|
|
18104
|
+
app.post("/memory/checkout", auth2, async (c) => {
|
|
18105
|
+
try {
|
|
18106
|
+
const user = c.get("user");
|
|
18107
|
+
const body = await c.req.json().catch(() => ({}));
|
|
18108
|
+
const priceId = body.priceId?.trim();
|
|
18109
|
+
if (!priceId || !MEMORY_PLANS[priceId]) return c.json({ error: "Invalid priceId \u2014 must be a memory plan price." }, 400);
|
|
18110
|
+
const secret2 = process.env.STRIPE_SECRET_KEY?.trim();
|
|
18111
|
+
if (!secret2) return c.json({ error: "Stripe is not configured." }, 503);
|
|
18112
|
+
const stripeClient = new Stripe2(secret2, { apiVersion: STRIPE_API_VERSION });
|
|
18113
|
+
let customerId = user.stripe_customer_id;
|
|
18114
|
+
if (!customerId) {
|
|
18115
|
+
const customer = await stripeClient.customers.create({ email: user.email });
|
|
18116
|
+
customerId = customer.id;
|
|
18117
|
+
await setStripeCustomerId(user.id, customerId);
|
|
18118
|
+
}
|
|
18119
|
+
const session = await stripeClient.checkout.sessions.create({
|
|
18120
|
+
mode: "subscription",
|
|
18121
|
+
customer: customerId,
|
|
18122
|
+
line_items: [{ price: priceId, quantity: 1 }],
|
|
18123
|
+
client_reference_id: String(user.id),
|
|
18124
|
+
metadata: { product: "mcp-memory", userId: String(user.id) },
|
|
18125
|
+
subscription_data: { metadata: { product: "mcp-memory", userId: String(user.id) } },
|
|
18126
|
+
automatic_tax: { enabled: true },
|
|
18127
|
+
billing_address_collection: "required",
|
|
18128
|
+
customer_update: { address: "auto", name: "auto" },
|
|
18129
|
+
tax_id_collection: { enabled: true },
|
|
18130
|
+
success_url: `${appOrigin()}/memory?billing=success`,
|
|
18131
|
+
cancel_url: `${appOrigin()}/memory?billing=cancel`
|
|
18132
|
+
});
|
|
18133
|
+
if (!session.url) return c.json({ error: "Stripe did not return a checkout URL." }, 502);
|
|
18134
|
+
return c.json({ url: session.url });
|
|
18135
|
+
} catch (err) {
|
|
18136
|
+
const message = err instanceof Error ? err.message : "Unable to start memory checkout.";
|
|
18137
|
+
console.error("[memory/checkout]", message);
|
|
18138
|
+
return c.json({ error: message }, 500);
|
|
18139
|
+
}
|
|
18140
|
+
});
|
|
18141
|
+
app.post("/memory/portal", auth2, async (c) => {
|
|
18142
|
+
try {
|
|
18143
|
+
const user = c.get("user");
|
|
18144
|
+
if (!user.stripe_customer_id) return c.json({ error: "No billing account yet \u2014 subscribe first." }, 409);
|
|
18145
|
+
const secret2 = process.env.STRIPE_SECRET_KEY?.trim();
|
|
18146
|
+
if (!secret2) return c.json({ error: "Stripe is not configured." }, 503);
|
|
18147
|
+
const stripeClient = new Stripe2(secret2, { apiVersion: STRIPE_API_VERSION });
|
|
18148
|
+
const session = await stripeClient.billingPortal.sessions.create({
|
|
18149
|
+
customer: user.stripe_customer_id,
|
|
18150
|
+
return_url: `${appOrigin()}/memory`
|
|
18151
|
+
});
|
|
18152
|
+
return c.json({ url: session.url });
|
|
18153
|
+
} catch (err) {
|
|
18154
|
+
const message = err instanceof Error ? err.message : "Unable to open billing portal.";
|
|
18155
|
+
console.error("[memory/portal]", message);
|
|
18156
|
+
return c.json({ error: message }, 500);
|
|
18157
|
+
}
|
|
18158
|
+
});
|
|
18159
|
+
app.get("/memory/usage", auth2, async (c) => {
|
|
18160
|
+
const user = c.get("user");
|
|
18161
|
+
try {
|
|
18162
|
+
const result = await dbUsage(resolveMemoryIdentity(user), memoryPlanForReads(user));
|
|
18163
|
+
return c.json(result);
|
|
18164
|
+
} catch (err) {
|
|
18165
|
+
return c.json({ ok: false, error: err instanceof Error ? err.message : "could not load usage" }, 502);
|
|
18166
|
+
}
|
|
18167
|
+
});
|
|
16890
18168
|
var SYNC_HARVEST_TIMEOUT_OVERRIDE_MS = (() => {
|
|
16891
18169
|
const raw = process.env.SYNC_HARVEST_TIMEOUT_MS;
|
|
16892
18170
|
const parsed = raw === void 0 ? NaN : Number(raw);
|
|
@@ -17013,6 +18291,7 @@ app.post("/harvest/sync", auth2, async (c) => {
|
|
|
17013
18291
|
format: "json",
|
|
17014
18292
|
outputDir: "/tmp/paa-output-api",
|
|
17015
18293
|
signal: syncSignal,
|
|
18294
|
+
softDeadlineMs: Date.now() + Math.floor(syncTimeoutMs * 0.8),
|
|
17016
18295
|
onAttemptEvent: recordAttempt
|
|
17017
18296
|
});
|
|
17018
18297
|
await completeJob(jobId, result);
|
|
@@ -17165,8 +18444,14 @@ app.post("/extract-url", auth2, async (c) => {
|
|
|
17165
18444
|
screenshotMeta = { base64: screenshotBuf.toString("base64"), sizeBytes: screenshotBuf.length, device };
|
|
17166
18445
|
}
|
|
17167
18446
|
const mediaMeta = downloadMedia ? await harvestPageMedia(result.bodyHtml, canonicalUrl, { types: mediaTypes ?? ["image", "video", "audio"], outputDir: null }) : null;
|
|
18447
|
+
const memory = bodyResult.data.depositToVault ? await persistScrapeBody(user, {
|
|
18448
|
+
title: result.title ?? canonicalUrl,
|
|
18449
|
+
content: result.bodyMarkdown ?? "",
|
|
18450
|
+
source: canonicalUrl,
|
|
18451
|
+
vault: bodyResult.data.vaultName
|
|
18452
|
+
}) : void 0;
|
|
17168
18453
|
await logRequestEvent({ userId: user.id, source: "extract_url", status: "done", query: canonicalUrl, resultCount: result.headings.length, result });
|
|
17169
|
-
return c.json({ ...result, screenshot: screenshotMeta, branding: brandingData, media: mediaMeta });
|
|
18454
|
+
return c.json({ ...result, screenshot: screenshotMeta, branding: brandingData, media: mediaMeta, memory });
|
|
17170
18455
|
} catch (err) {
|
|
17171
18456
|
const msg = err instanceof Error ? err.message : String(err);
|
|
17172
18457
|
if (debited) await creditMc(user.id, MC_COSTS.page_scrape, LedgerOperation.EXTRACT_URL, "failed call");
|
|
@@ -17584,16 +18869,17 @@ app.get("/cron/tick", async (c) => {
|
|
|
17584
18869
|
if (!process.env.CRON_SECRET || secret2 !== `Bearer ${process.env.CRON_SECRET}`) {
|
|
17585
18870
|
return c.json({ error: "Unauthorized" }, 401);
|
|
17586
18871
|
}
|
|
17587
|
-
const { drainQueue } = await import("./worker-
|
|
18872
|
+
const { drainQueue } = await import("./worker-UC6D2756.js");
|
|
17588
18873
|
const budget = { maxJobs: 10, deadlineMs: Date.now() + 28e4 };
|
|
17589
18874
|
const workflowDispatchResult = await dispatchDueWorkflowSchedules(`${new URL(c.req.url).protocol}//${new URL(c.req.url).host}`);
|
|
17590
|
-
const [results, sweepResult, reapResult, expiredResult] = await Promise.all([
|
|
18875
|
+
const [results, sweepResult, reapResult, expiredResult, blobCleanup] = await Promise.all([
|
|
17591
18876
|
drainQueue(budget),
|
|
17592
18877
|
runMonthlyRefreshSweep(),
|
|
17593
18878
|
reapIdleBrowserSessions(120).catch(() => ({ reaped: 0 })),
|
|
17594
|
-
expireOldLots().catch(() => ({ expired_lots: 0, expired_mc: 0 }))
|
|
18879
|
+
expireOldLots().catch(() => ({ expired_lots: 0, expired_mc: 0 })),
|
|
18880
|
+
cleanupExpiredScrapeBlobs().catch(() => ({ deleted: 0, store: "none" }))
|
|
17595
18881
|
]);
|
|
17596
|
-
return c.json({ drained: results.length, results, sweepResult, reaped: reapResult, expired: expiredResult, workflowDispatch: workflowDispatchResult });
|
|
18882
|
+
return c.json({ drained: results.length, results, sweepResult, reaped: reapResult, expired: expiredResult, blobCleanup, workflowDispatch: workflowDispatchResult });
|
|
17597
18883
|
});
|
|
17598
18884
|
app.on(["GET", "POST", "PUT"], "/api/inngest", serveInngest({ client: inngest, functions: [siteAuditFn, siteExtractFn] }));
|
|
17599
18885
|
app.route("/api/internal/site-architecture-auditor", siteAuditApp);
|
|
@@ -17601,6 +18887,7 @@ app.route("/youtube", youtubeApp);
|
|
|
17601
18887
|
app.route("/screenshot", screenshotApp);
|
|
17602
18888
|
app.route("/facebook", facebookAdApp);
|
|
17603
18889
|
app.route("/instagram", instagramApp);
|
|
18890
|
+
app.route("/reddit", redditApp);
|
|
17604
18891
|
app.route("/maps", mapsApp);
|
|
17605
18892
|
app.route("/directory", directoryApp);
|
|
17606
18893
|
app.route("/workflows", workflowApp);
|
|
@@ -17623,6 +18910,7 @@ app.get("/console/auth/:id/go", async (c) => {
|
|
|
17623
18910
|
});
|
|
17624
18911
|
app.get("/console/:id", (c) => c.html(renderConsoleHtml(c.req.param("id"))));
|
|
17625
18912
|
app.route("/stripe", stripeApp);
|
|
18913
|
+
app.route("/", oauthApp);
|
|
17626
18914
|
if (!process.env.INNGEST_EVENT_KEY) {
|
|
17627
18915
|
startSiteAuditWorker();
|
|
17628
18916
|
}
|
|
@@ -17729,4 +19017,4 @@ app.get("/blog/:slug/", (c) => {
|
|
|
17729
19017
|
export {
|
|
17730
19018
|
app
|
|
17731
19019
|
};
|
|
17732
|
-
//# sourceMappingURL=server-
|
|
19020
|
+
//# sourceMappingURL=server-HIGHKTEV.js.map
|