@j0hanz/superfetch 1.2.5 → 2.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +131 -156
- package/dist/config/auth-config.d.ts +16 -0
- package/dist/config/auth-config.js +53 -0
- package/dist/config/constants.d.ts +11 -13
- package/dist/config/constants.js +1 -3
- package/dist/config/env-parsers.d.ts +7 -0
- package/dist/config/env-parsers.js +84 -0
- package/dist/config/formatting.d.ts +2 -2
- package/dist/config/index.d.ts +47 -53
- package/dist/config/index.js +35 -64
- package/dist/config/types/content.d.ts +1 -49
- package/dist/config/types/runtime.d.ts +8 -16
- package/dist/config/types/tools.d.ts +2 -28
- package/dist/http/accept-policy.d.ts +3 -0
- package/dist/http/accept-policy.js +45 -0
- package/dist/http/async-handler.d.ts +2 -0
- package/dist/http/async-handler.js +5 -0
- package/dist/http/auth-introspection.d.ts +2 -0
- package/dist/http/auth-introspection.js +141 -0
- package/dist/http/auth-static.d.ts +2 -0
- package/dist/http/auth-static.js +23 -0
- package/dist/http/auth.d.ts +3 -2
- package/dist/http/auth.js +254 -23
- package/dist/http/cors.d.ts +6 -6
- package/dist/http/cors.js +7 -42
- package/dist/http/download-routes.d.ts +0 -12
- package/dist/http/download-routes.js +21 -58
- package/dist/http/host-allowlist.d.ts +3 -0
- package/dist/http/host-allowlist.js +117 -0
- package/dist/http/jsonrpc-http.d.ts +2 -0
- package/dist/http/jsonrpc-http.js +10 -0
- package/dist/http/mcp-routes.d.ts +8 -3
- package/dist/http/mcp-routes.js +137 -31
- package/dist/http/mcp-session-eviction.d.ts +3 -0
- package/dist/http/mcp-session-eviction.js +24 -0
- package/dist/http/mcp-session-helpers.d.ts +0 -1
- package/dist/http/mcp-session-helpers.js +1 -1
- package/dist/http/mcp-session-init.d.ts +7 -0
- package/dist/http/mcp-session-init.js +94 -0
- package/dist/http/mcp-session-slots.d.ts +17 -0
- package/dist/http/mcp-session-slots.js +55 -0
- package/dist/http/mcp-session-transport-init.d.ts +7 -0
- package/dist/http/mcp-session-transport-init.js +41 -0
- package/dist/http/mcp-session-transport.d.ts +7 -0
- package/dist/http/mcp-session-transport.js +57 -0
- package/dist/http/mcp-session-types.d.ts +5 -0
- package/dist/http/mcp-session-types.js +1 -0
- package/dist/http/mcp-session.d.ts +9 -9
- package/dist/http/mcp-session.js +15 -137
- package/dist/http/mcp-sessions.d.ts +43 -0
- package/dist/http/mcp-sessions.js +392 -0
- package/dist/http/mcp-validation.d.ts +1 -0
- package/dist/http/mcp-validation.js +11 -10
- package/dist/http/protocol-policy.d.ts +2 -0
- package/dist/http/protocol-policy.js +31 -0
- package/dist/http/rate-limit.js +7 -4
- package/dist/http/server-config.d.ts +1 -0
- package/dist/http/server-config.js +40 -0
- package/dist/http/server-middleware.d.ts +7 -9
- package/dist/http/server-middleware.js +9 -70
- package/dist/http/server-shutdown.d.ts +4 -0
- package/dist/http/server-shutdown.js +43 -0
- package/dist/http/server.d.ts +10 -0
- package/dist/http/server.js +546 -61
- package/dist/http/session-cleanup.js +8 -5
- package/dist/middleware/error-handler.d.ts +1 -1
- package/dist/middleware/error-handler.js +32 -33
- package/dist/resources/cached-content-params.d.ts +5 -0
- package/dist/resources/cached-content-params.js +36 -0
- package/dist/resources/cached-content.js +67 -125
- package/dist/resources/index.js +0 -82
- package/dist/server.js +50 -29
- package/dist/services/cache-events.d.ts +8 -0
- package/dist/services/cache-events.js +19 -0
- package/dist/services/cache-keys.d.ts +7 -0
- package/dist/services/cache-keys.js +57 -0
- package/dist/services/cache.d.ts +4 -9
- package/dist/services/cache.js +77 -139
- package/dist/services/context.d.ts +0 -1
- package/dist/services/context.js +0 -7
- package/dist/services/extractor.js +55 -116
- package/dist/services/fetcher/agents.d.ts +2 -2
- package/dist/services/fetcher/agents.js +35 -96
- package/dist/services/fetcher/dns-selection.d.ts +2 -0
- package/dist/services/fetcher/dns-selection.js +72 -0
- package/dist/services/fetcher/interceptors.d.ts +0 -22
- package/dist/services/fetcher/interceptors.js +18 -32
- package/dist/services/fetcher/redirects.js +16 -7
- package/dist/services/fetcher/response.js +79 -34
- package/dist/services/fetcher.d.ts +22 -3
- package/dist/services/fetcher.js +544 -44
- package/dist/services/fifo-queue.d.ts +8 -0
- package/dist/services/fifo-queue.js +25 -0
- package/dist/services/logger.js +2 -2
- package/dist/services/metadata-collector.d.ts +1 -9
- package/dist/services/metadata-collector.js +71 -2
- package/dist/services/transform-worker-pool.d.ts +4 -14
- package/dist/services/transform-worker-pool.js +177 -129
- package/dist/services/transform-worker-types.d.ts +32 -0
- package/dist/services/transform-worker-types.js +14 -0
- package/dist/tools/handlers/fetch-markdown.tool.d.ts +3 -4
- package/dist/tools/handlers/fetch-markdown.tool.js +20 -72
- package/dist/tools/handlers/fetch-single.shared.d.ts +11 -22
- package/dist/tools/handlers/fetch-single.shared.js +175 -89
- package/dist/tools/handlers/fetch-url.tool.d.ts +7 -1
- package/dist/tools/handlers/fetch-url.tool.js +84 -119
- package/dist/tools/index.js +21 -40
- package/dist/tools/schemas.d.ts +1 -51
- package/dist/tools/schemas.js +1 -107
- package/dist/tools/utils/cached-markdown.d.ts +5 -0
- package/dist/tools/utils/cached-markdown.js +46 -0
- package/dist/tools/utils/content-shaping.d.ts +4 -0
- package/dist/tools/utils/content-shaping.js +67 -0
- package/dist/tools/utils/content-transform.d.ts +5 -17
- package/dist/tools/utils/content-transform.js +134 -114
- package/dist/tools/utils/fetch-pipeline.d.ts +0 -8
- package/dist/tools/utils/fetch-pipeline.js +57 -63
- package/dist/tools/utils/frontmatter.d.ts +3 -0
- package/dist/tools/utils/frontmatter.js +73 -0
- package/dist/tools/utils/inline-content.d.ts +1 -2
- package/dist/tools/utils/inline-content.js +4 -7
- package/dist/tools/utils/markdown-heuristics.d.ts +1 -0
- package/dist/tools/utils/markdown-heuristics.js +19 -0
- package/dist/tools/utils/markdown-signals.d.ts +1 -0
- package/dist/tools/utils/markdown-signals.js +19 -0
- package/dist/tools/utils/raw-markdown-frontmatter.d.ts +3 -0
- package/dist/tools/utils/raw-markdown-frontmatter.js +73 -0
- package/dist/tools/utils/raw-markdown.d.ts +6 -0
- package/dist/tools/utils/raw-markdown.js +135 -0
- package/dist/transformers/markdown/fenced-code-rule.d.ts +2 -0
- package/dist/transformers/markdown/fenced-code-rule.js +38 -0
- package/dist/transformers/markdown/frontmatter.d.ts +2 -0
- package/dist/transformers/markdown/frontmatter.js +45 -0
- package/dist/transformers/markdown/noise-rule.d.ts +2 -0
- package/dist/transformers/markdown/noise-rule.js +80 -0
- package/dist/transformers/markdown/turndown-instance.d.ts +2 -0
- package/dist/transformers/markdown/turndown-instance.js +19 -0
- package/dist/transformers/markdown.d.ts +2 -0
- package/dist/transformers/markdown.js +185 -0
- package/dist/transformers/markdown.transformer.js +5 -117
- package/dist/utils/cached-payload.d.ts +7 -0
- package/dist/utils/cached-payload.js +36 -0
- package/dist/utils/code-language-bash.d.ts +1 -0
- package/dist/utils/code-language-bash.js +48 -0
- package/dist/utils/code-language-core.d.ts +2 -0
- package/dist/utils/code-language-core.js +13 -0
- package/dist/utils/code-language-detectors.d.ts +5 -0
- package/dist/utils/code-language-detectors.js +142 -0
- package/dist/utils/code-language-helpers.d.ts +5 -0
- package/dist/utils/code-language-helpers.js +62 -0
- package/dist/utils/code-language-parsing.d.ts +5 -0
- package/dist/utils/code-language-parsing.js +62 -0
- package/dist/utils/code-language.d.ts +9 -0
- package/dist/utils/code-language.js +250 -46
- package/dist/utils/error-details.d.ts +3 -0
- package/dist/utils/error-details.js +12 -0
- package/dist/utils/error-utils.js +1 -1
- package/dist/utils/filename-generator.js +34 -12
- package/dist/utils/guards.d.ts +1 -0
- package/dist/utils/guards.js +3 -0
- package/dist/utils/header-normalizer.d.ts +0 -3
- package/dist/utils/header-normalizer.js +3 -3
- package/dist/utils/ip-address.d.ts +4 -0
- package/dist/utils/ip-address.js +6 -0
- package/dist/utils/tool-error-handler.d.ts +2 -2
- package/dist/utils/tool-error-handler.js +14 -46
- package/dist/utils/url-transformer.d.ts +7 -0
- package/dist/utils/url-transformer.js +147 -0
- package/dist/utils/url-validator.d.ts +1 -2
- package/dist/utils/url-validator.js +53 -114
- package/dist/workers/content-transform.worker.d.ts +1 -0
- package/dist/workers/content-transform.worker.js +40 -0
- package/package.json +17 -18
package/dist/http/server.js
CHANGED
|
@@ -1,31 +1,508 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { randomUUID } from 'node:crypto';
|
|
2
|
+
import { setInterval as setIntervalPromise } from 'node:timers/promises';
|
|
2
3
|
import { config, enableHttpMode } from '../config/index.js';
|
|
4
|
+
import { FetchError } from '../errors/app-error.js';
|
|
5
|
+
import * as cache from '../services/cache.js';
|
|
6
|
+
import { runWithRequestContext } from '../services/context.js';
|
|
3
7
|
import { destroyAgents } from '../services/fetcher.js';
|
|
4
|
-
import { logError, logInfo, logWarn } from '../services/logger.js';
|
|
5
|
-
import {
|
|
6
|
-
import {
|
|
7
|
-
import {
|
|
8
|
-
import { createAuthMiddleware } from './auth.js';
|
|
9
|
-
import { createCorsMiddleware } from './cors.js';
|
|
10
|
-
import { registerDownloadRoutes } from './download-routes.js';
|
|
8
|
+
import { logDebug, logError, logInfo, logWarn } from '../services/logger.js';
|
|
9
|
+
import { parseCachedPayload, resolveCachedPayloadContent, } from '../utils/cached-payload.js';
|
|
10
|
+
import { getErrorMessage } from '../utils/error-details.js';
|
|
11
|
+
import { generateSafeFilename } from '../utils/filename-generator.js';
|
|
12
|
+
import { createAuthMetadataRouter, createAuthMiddleware } from './auth.js';
|
|
11
13
|
import { registerMcpRoutes } from './mcp-routes.js';
|
|
12
|
-
import {
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
function
|
|
17
|
-
|
|
14
|
+
import { createSessionStore, getSessionId, startSessionCleanupLoop, } from './mcp-sessions.js';
|
|
15
|
+
function getRateLimitKey(req) {
|
|
16
|
+
return req.ip ?? req.socket.remoteAddress ?? 'unknown';
|
|
17
|
+
}
|
|
18
|
+
function createCleanupInterval(store, options) {
|
|
19
|
+
const controller = new AbortController();
|
|
20
|
+
void startCleanupLoop(store, options, controller.signal).catch(handleCleanupError);
|
|
21
|
+
return controller;
|
|
22
|
+
}
|
|
23
|
+
function createRateLimitMiddleware(options) {
|
|
24
|
+
const store = new Map();
|
|
25
|
+
const cleanupController = createCleanupInterval(store, options);
|
|
26
|
+
const stop = () => {
|
|
27
|
+
cleanupController.abort();
|
|
28
|
+
};
|
|
29
|
+
const middleware = createRateLimitHandler(store, options);
|
|
30
|
+
return { middleware, stop, store };
|
|
31
|
+
}
|
|
32
|
+
function createRateLimitHandler(store, options) {
|
|
33
|
+
return (req, res, next) => {
|
|
34
|
+
if (shouldSkipRateLimit(req, options)) {
|
|
35
|
+
next();
|
|
36
|
+
return;
|
|
37
|
+
}
|
|
38
|
+
const now = Date.now();
|
|
39
|
+
const key = getRateLimitKey(req);
|
|
40
|
+
const resolution = resolveRateLimitEntry(store, key, now, options);
|
|
41
|
+
if (resolution.isNew) {
|
|
42
|
+
next();
|
|
43
|
+
return;
|
|
44
|
+
}
|
|
45
|
+
if (handleRateLimitExceeded(res, resolution.entry, now, options)) {
|
|
46
|
+
return;
|
|
47
|
+
}
|
|
48
|
+
next();
|
|
49
|
+
};
|
|
50
|
+
}
|
|
51
|
+
async function startCleanupLoop(store, options, signal) {
|
|
52
|
+
for await (const getNow of setIntervalPromise(options.cleanupIntervalMs, Date.now, { signal, ref: false })) {
|
|
53
|
+
evictStaleEntries(store, options, getNow());
|
|
54
|
+
}
|
|
55
|
+
}
|
|
56
|
+
function evictStaleEntries(store, options, now) {
|
|
57
|
+
for (const [key, entry] of store.entries()) {
|
|
58
|
+
if (now - entry.lastAccessed > options.windowMs * 2) {
|
|
59
|
+
store.delete(key);
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
}
|
|
63
|
+
function isAbortError(error) {
|
|
64
|
+
return error instanceof Error && error.name === 'AbortError';
|
|
65
|
+
}
|
|
66
|
+
function handleCleanupError(error) {
|
|
67
|
+
if (isAbortError(error)) {
|
|
68
|
+
return;
|
|
69
|
+
}
|
|
70
|
+
}
|
|
71
|
+
function shouldSkipRateLimit(req, options) {
|
|
72
|
+
return !options.enabled || req.method === 'OPTIONS';
|
|
73
|
+
}
|
|
74
|
+
function resolveRateLimitEntry(store, key, now, options) {
|
|
75
|
+
const existing = store.get(key);
|
|
76
|
+
if (!existing || now > existing.resetTime) {
|
|
77
|
+
const entry = createNewEntry(now, options);
|
|
78
|
+
store.set(key, entry);
|
|
79
|
+
return { entry, isNew: true };
|
|
80
|
+
}
|
|
81
|
+
updateEntry(existing, now);
|
|
82
|
+
return { entry: existing, isNew: false };
|
|
83
|
+
}
|
|
84
|
+
function createNewEntry(now, options) {
|
|
85
|
+
return {
|
|
86
|
+
count: 1,
|
|
87
|
+
resetTime: now + options.windowMs,
|
|
88
|
+
lastAccessed: now,
|
|
89
|
+
};
|
|
90
|
+
}
|
|
91
|
+
function updateEntry(entry, now) {
|
|
92
|
+
entry.count += 1;
|
|
93
|
+
entry.lastAccessed = now;
|
|
94
|
+
}
|
|
95
|
+
function handleRateLimitExceeded(res, entry, now, options) {
|
|
96
|
+
if (entry.count <= options.maxRequests) {
|
|
97
|
+
return false;
|
|
98
|
+
}
|
|
99
|
+
const retryAfter = Math.max(1, Math.ceil((entry.resetTime - now) / 1000));
|
|
100
|
+
res.set('Retry-After', String(retryAfter));
|
|
101
|
+
res.status(429).json({
|
|
102
|
+
error: 'Rate limit exceeded',
|
|
103
|
+
retryAfter,
|
|
104
|
+
});
|
|
105
|
+
return true;
|
|
106
|
+
}
|
|
107
|
+
const LOOPBACK_HOSTS = new Set(['localhost', '127.0.0.1', '::1']);
|
|
108
|
+
function getNonEmptyStringHeader(value) {
|
|
109
|
+
if (typeof value !== 'string')
|
|
110
|
+
return null;
|
|
111
|
+
const trimmed = value.trim();
|
|
112
|
+
return trimmed === '' ? null : trimmed;
|
|
113
|
+
}
|
|
114
|
+
function respondHostNotAllowed(res) {
|
|
115
|
+
res.status(403).json({
|
|
116
|
+
error: 'Host not allowed',
|
|
117
|
+
code: 'HOST_NOT_ALLOWED',
|
|
118
|
+
});
|
|
119
|
+
}
|
|
120
|
+
function respondOriginNotAllowed(res) {
|
|
121
|
+
res.status(403).json({
|
|
122
|
+
error: 'Origin not allowed',
|
|
123
|
+
code: 'ORIGIN_NOT_ALLOWED',
|
|
124
|
+
});
|
|
125
|
+
}
|
|
126
|
+
function tryParseOriginHostname(originHeader) {
|
|
127
|
+
try {
|
|
128
|
+
return new URL(originHeader).hostname.toLowerCase();
|
|
129
|
+
}
|
|
130
|
+
catch {
|
|
131
|
+
return null;
|
|
132
|
+
}
|
|
133
|
+
}
|
|
134
|
+
function takeFirstHostValue(value) {
|
|
135
|
+
const first = value.split(',')[0];
|
|
136
|
+
if (!first)
|
|
137
|
+
return null;
|
|
138
|
+
const trimmed = first.trim();
|
|
139
|
+
return trimmed ? trimmed : null;
|
|
140
|
+
}
|
|
141
|
+
function stripIpv6Brackets(value) {
|
|
142
|
+
if (!value.startsWith('['))
|
|
143
|
+
return null;
|
|
144
|
+
const end = value.indexOf(']');
|
|
145
|
+
if (end === -1)
|
|
146
|
+
return null;
|
|
147
|
+
return value.slice(1, end);
|
|
148
|
+
}
|
|
149
|
+
function stripPortIfPresent(value) {
|
|
150
|
+
const colonIndex = value.indexOf(':');
|
|
151
|
+
if (colonIndex === -1)
|
|
152
|
+
return value;
|
|
153
|
+
return value.slice(0, colonIndex);
|
|
154
|
+
}
|
|
155
|
+
function normalizeHost(value) {
|
|
156
|
+
const trimmed = value.trim().toLowerCase();
|
|
157
|
+
if (!trimmed)
|
|
158
|
+
return null;
|
|
159
|
+
const first = takeFirstHostValue(trimmed);
|
|
160
|
+
if (!first)
|
|
161
|
+
return null;
|
|
162
|
+
const ipv6 = stripIpv6Brackets(first);
|
|
163
|
+
if (ipv6)
|
|
164
|
+
return ipv6;
|
|
165
|
+
return stripPortIfPresent(first);
|
|
166
|
+
}
|
|
167
|
+
function isWildcardHost(host) {
|
|
168
|
+
return host === '0.0.0.0' || host === '::';
|
|
169
|
+
}
|
|
170
|
+
function addLoopbackHosts(allowedHosts) {
|
|
171
|
+
for (const host of LOOPBACK_HOSTS) {
|
|
172
|
+
allowedHosts.add(host);
|
|
173
|
+
}
|
|
174
|
+
}
|
|
175
|
+
function addConfiguredHost(allowedHosts) {
|
|
176
|
+
const configuredHost = normalizeHost(config.server.host);
|
|
177
|
+
if (!configuredHost)
|
|
178
|
+
return;
|
|
179
|
+
if (isWildcardHost(configuredHost))
|
|
180
|
+
return;
|
|
181
|
+
allowedHosts.add(configuredHost);
|
|
182
|
+
}
|
|
183
|
+
function addExplicitAllowedHosts(allowedHosts) {
|
|
184
|
+
for (const host of config.security.allowedHosts) {
|
|
185
|
+
allowedHosts.add(host);
|
|
186
|
+
}
|
|
187
|
+
}
|
|
188
|
+
function buildAllowedHosts() {
|
|
189
|
+
const allowedHosts = new Set();
|
|
190
|
+
addLoopbackHosts(allowedHosts);
|
|
191
|
+
addConfiguredHost(allowedHosts);
|
|
192
|
+
addExplicitAllowedHosts(allowedHosts);
|
|
193
|
+
return allowedHosts;
|
|
194
|
+
}
|
|
195
|
+
function createHostValidationMiddleware() {
|
|
196
|
+
const allowedHosts = buildAllowedHosts();
|
|
197
|
+
return (req, res, next) => {
|
|
198
|
+
const hostHeader = typeof req.headers.host === 'string' ? req.headers.host : '';
|
|
199
|
+
const normalized = normalizeHost(hostHeader);
|
|
200
|
+
if (!normalized || !allowedHosts.has(normalized)) {
|
|
201
|
+
respondHostNotAllowed(res);
|
|
202
|
+
return;
|
|
203
|
+
}
|
|
204
|
+
next();
|
|
205
|
+
};
|
|
206
|
+
}
|
|
207
|
+
function createOriginValidationMiddleware() {
|
|
208
|
+
const allowedHosts = buildAllowedHosts();
|
|
209
|
+
return (req, res, next) => {
|
|
210
|
+
const originHeader = getNonEmptyStringHeader(req.headers.origin);
|
|
211
|
+
if (!originHeader) {
|
|
212
|
+
next();
|
|
213
|
+
return;
|
|
214
|
+
}
|
|
215
|
+
const originHostname = tryParseOriginHostname(originHeader);
|
|
216
|
+
if (!originHostname || !allowedHosts.has(originHostname)) {
|
|
217
|
+
respondOriginNotAllowed(res);
|
|
218
|
+
return;
|
|
219
|
+
}
|
|
220
|
+
next();
|
|
221
|
+
};
|
|
222
|
+
}
|
|
223
|
+
function createJsonParseErrorHandler() {
|
|
224
|
+
return (err, _req, res, next) => {
|
|
225
|
+
if (err instanceof SyntaxError && 'body' in err) {
|
|
226
|
+
res.status(400).json({
|
|
227
|
+
jsonrpc: '2.0',
|
|
228
|
+
error: {
|
|
229
|
+
code: -32700,
|
|
230
|
+
message: 'Parse error: Invalid JSON',
|
|
231
|
+
},
|
|
232
|
+
id: null,
|
|
233
|
+
});
|
|
234
|
+
return;
|
|
235
|
+
}
|
|
236
|
+
next();
|
|
237
|
+
};
|
|
238
|
+
}
|
|
239
|
+
function createContextMiddleware() {
|
|
240
|
+
return (req, _res, next) => {
|
|
241
|
+
const requestId = randomUUID();
|
|
242
|
+
const sessionId = getSessionId(req);
|
|
243
|
+
const context = sessionId === undefined ? { requestId } : { requestId, sessionId };
|
|
244
|
+
runWithRequestContext(context, () => {
|
|
245
|
+
next();
|
|
246
|
+
});
|
|
247
|
+
};
|
|
248
|
+
}
|
|
249
|
+
export function createCorsMiddleware() {
|
|
250
|
+
return (req, res, next) => {
|
|
251
|
+
// Handle OPTIONS preflight
|
|
252
|
+
if (req.method === 'OPTIONS') {
|
|
253
|
+
res.sendStatus(200);
|
|
254
|
+
return;
|
|
255
|
+
}
|
|
256
|
+
next();
|
|
257
|
+
};
|
|
258
|
+
}
|
|
259
|
+
function registerHealthRoute(app) {
|
|
260
|
+
app.get('/health', (_req, res) => {
|
|
261
|
+
res.json({
|
|
262
|
+
status: 'healthy',
|
|
263
|
+
name: config.server.name,
|
|
264
|
+
version: config.server.version,
|
|
265
|
+
uptime: process.uptime(),
|
|
266
|
+
});
|
|
267
|
+
});
|
|
268
|
+
}
|
|
269
|
+
export function attachBaseMiddleware(options) {
|
|
270
|
+
const { app, jsonParser, rateLimitMiddleware, corsMiddleware } = options;
|
|
271
|
+
app.use(createHostValidationMiddleware());
|
|
272
|
+
app.use(createOriginValidationMiddleware());
|
|
273
|
+
app.use(jsonParser);
|
|
274
|
+
app.use(createContextMiddleware());
|
|
275
|
+
app.use(createJsonParseErrorHandler());
|
|
276
|
+
app.use(corsMiddleware);
|
|
277
|
+
app.use('/mcp', rateLimitMiddleware);
|
|
278
|
+
registerHealthRoute(app);
|
|
279
|
+
}
|
|
280
|
+
const HASH_PATTERN = /^[a-f0-9.]+$/i;
|
|
281
|
+
function validateNamespace(namespace) {
|
|
282
|
+
return namespace === 'markdown';
|
|
283
|
+
}
|
|
284
|
+
function validateHash(hash) {
|
|
285
|
+
return HASH_PATTERN.test(hash) && hash.length >= 8 && hash.length <= 64;
|
|
286
|
+
}
|
|
287
|
+
function parseDownloadParams(req) {
|
|
288
|
+
const { namespace, hash } = req.params;
|
|
289
|
+
if (!namespace || !hash)
|
|
290
|
+
return null;
|
|
291
|
+
if (!validateNamespace(namespace))
|
|
292
|
+
return null;
|
|
293
|
+
if (!validateHash(hash))
|
|
294
|
+
return null;
|
|
295
|
+
return { namespace, hash };
|
|
296
|
+
}
|
|
297
|
+
function buildCacheKeyFromParams(params) {
|
|
298
|
+
return `${params.namespace}:${params.hash}`;
|
|
299
|
+
}
|
|
300
|
+
function respondBadRequest(res, message) {
|
|
301
|
+
res.status(400).json({
|
|
302
|
+
error: message,
|
|
303
|
+
code: 'BAD_REQUEST',
|
|
304
|
+
});
|
|
305
|
+
}
|
|
306
|
+
function respondNotFound(res) {
|
|
307
|
+
res.status(404).json({
|
|
308
|
+
error: 'Content not found or expired',
|
|
309
|
+
code: 'NOT_FOUND',
|
|
310
|
+
});
|
|
311
|
+
}
|
|
312
|
+
function respondServiceUnavailable(res) {
|
|
313
|
+
res.status(503).json({
|
|
314
|
+
error: 'Download service is disabled',
|
|
315
|
+
code: 'SERVICE_UNAVAILABLE',
|
|
316
|
+
});
|
|
317
|
+
}
|
|
318
|
+
function resolveDownloadPayload(params, cacheEntry) {
|
|
319
|
+
const payload = parseCachedPayload(cacheEntry.content);
|
|
320
|
+
if (!payload)
|
|
321
|
+
return null;
|
|
322
|
+
const content = resolveCachedPayloadContent(payload);
|
|
323
|
+
if (!content)
|
|
324
|
+
return null;
|
|
325
|
+
const safeTitle = typeof payload.title === 'string' ? payload.title : undefined;
|
|
326
|
+
const fileName = generateSafeFilename(cacheEntry.url, cacheEntry.title ?? safeTitle, params.hash, '.md');
|
|
327
|
+
return {
|
|
328
|
+
content,
|
|
329
|
+
contentType: 'text/markdown; charset=utf-8',
|
|
330
|
+
fileName,
|
|
331
|
+
};
|
|
332
|
+
}
|
|
333
|
+
function buildContentDisposition(fileName) {
|
|
334
|
+
const encodedName = encodeURIComponent(fileName).replace(/'/g, '%27');
|
|
335
|
+
return `attachment; filename="${fileName}"; filename*=UTF-8''${encodedName}`;
|
|
336
|
+
}
|
|
337
|
+
function sendDownloadPayload(res, payload) {
|
|
338
|
+
const disposition = buildContentDisposition(payload.fileName);
|
|
339
|
+
res.setHeader('Content-Type', payload.contentType);
|
|
340
|
+
res.setHeader('Content-Disposition', disposition);
|
|
341
|
+
res.setHeader('Cache-Control', `private, max-age=${config.cache.ttl}`);
|
|
342
|
+
res.setHeader('X-Content-Type-Options', 'nosniff');
|
|
343
|
+
res.send(payload.content);
|
|
344
|
+
}
|
|
345
|
+
function handleDownload(req, res) {
|
|
346
|
+
if (!config.cache.enabled) {
|
|
347
|
+
respondServiceUnavailable(res);
|
|
348
|
+
return;
|
|
349
|
+
}
|
|
350
|
+
const params = parseDownloadParams(req);
|
|
351
|
+
if (!params) {
|
|
352
|
+
respondBadRequest(res, 'Invalid namespace or hash format');
|
|
353
|
+
return;
|
|
354
|
+
}
|
|
355
|
+
const cacheKey = buildCacheKeyFromParams(params);
|
|
356
|
+
const cacheEntry = cache.get(cacheKey);
|
|
357
|
+
if (!cacheEntry) {
|
|
358
|
+
logDebug('Download request for missing cache key', { cacheKey });
|
|
359
|
+
respondNotFound(res);
|
|
360
|
+
return;
|
|
361
|
+
}
|
|
362
|
+
const payload = resolveDownloadPayload(params, cacheEntry);
|
|
363
|
+
if (!payload) {
|
|
364
|
+
logDebug('Download payload unavailable', { cacheKey });
|
|
365
|
+
respondNotFound(res);
|
|
366
|
+
return;
|
|
367
|
+
}
|
|
368
|
+
logDebug('Serving download', { cacheKey, fileName: payload.fileName });
|
|
369
|
+
sendDownloadPayload(res, payload);
|
|
370
|
+
}
|
|
371
|
+
export function registerDownloadRoutes(app) {
|
|
372
|
+
app.get('/mcp/downloads/:namespace/:hash', handleDownload);
|
|
373
|
+
}
|
|
374
|
+
function getStatusCode(fetchError) {
|
|
375
|
+
return fetchError ? fetchError.statusCode : 500;
|
|
376
|
+
}
|
|
377
|
+
function getErrorCode(fetchError) {
|
|
378
|
+
return fetchError ? fetchError.code : 'INTERNAL_ERROR';
|
|
379
|
+
}
|
|
380
|
+
function getFetchErrorMessage(fetchError) {
|
|
381
|
+
return fetchError ? fetchError.message : 'Internal Server Error';
|
|
382
|
+
}
|
|
383
|
+
function getErrorDetails(fetchError) {
|
|
384
|
+
if (fetchError && Object.keys(fetchError.details).length > 0) {
|
|
385
|
+
return fetchError.details;
|
|
386
|
+
}
|
|
387
|
+
return undefined;
|
|
388
|
+
}
|
|
389
|
+
function setRetryAfterHeader(res, fetchError) {
|
|
390
|
+
const retryAfter = resolveRetryAfter(fetchError);
|
|
391
|
+
if (retryAfter === undefined)
|
|
392
|
+
return;
|
|
393
|
+
res.set('Retry-After', retryAfter);
|
|
394
|
+
}
|
|
395
|
+
function buildErrorResponse(fetchError) {
|
|
396
|
+
const details = getErrorDetails(fetchError);
|
|
397
|
+
const response = {
|
|
398
|
+
error: {
|
|
399
|
+
message: getFetchErrorMessage(fetchError),
|
|
400
|
+
code: getErrorCode(fetchError),
|
|
401
|
+
statusCode: getStatusCode(fetchError),
|
|
402
|
+
...(details && { details }),
|
|
403
|
+
},
|
|
404
|
+
};
|
|
405
|
+
// Never expose stack traces in production
|
|
406
|
+
return response;
|
|
407
|
+
}
|
|
408
|
+
function resolveRetryAfter(fetchError) {
|
|
409
|
+
if (fetchError?.statusCode !== 429)
|
|
410
|
+
return undefined;
|
|
411
|
+
const { retryAfter } = fetchError.details;
|
|
412
|
+
return isRetryAfterValue(retryAfter) ? String(retryAfter) : undefined;
|
|
413
|
+
}
|
|
414
|
+
function isRetryAfterValue(value) {
|
|
415
|
+
return typeof value === 'number' || typeof value === 'string';
|
|
416
|
+
}
|
|
417
|
+
export function errorHandler(err, req, res, next) {
|
|
418
|
+
if (res.headersSent) {
|
|
419
|
+
next(err);
|
|
420
|
+
return;
|
|
421
|
+
}
|
|
422
|
+
const fetchError = err instanceof FetchError ? err : null;
|
|
423
|
+
const statusCode = getStatusCode(fetchError);
|
|
424
|
+
logError(`HTTP ${statusCode}: ${err.message} - ${req.method} ${req.path}`, err);
|
|
425
|
+
setRetryAfterHeader(res, fetchError);
|
|
426
|
+
res.status(statusCode).json(buildErrorResponse(fetchError));
|
|
18
427
|
}
|
|
19
428
|
function assertHttpConfiguration() {
|
|
20
|
-
|
|
429
|
+
ensureBindAllowed();
|
|
430
|
+
ensureStaticTokens();
|
|
431
|
+
if (config.auth.mode === 'oauth') {
|
|
432
|
+
ensureOauthConfiguration();
|
|
433
|
+
}
|
|
434
|
+
}
|
|
435
|
+
function ensureBindAllowed() {
|
|
436
|
+
const isLoopback = ['127.0.0.1', '::1', 'localhost'].includes(config.server.host);
|
|
437
|
+
if (!config.security.allowRemote && !isLoopback) {
|
|
21
438
|
logError('Refusing to bind to non-loopback host without ALLOW_REMOTE=true', { host: config.server.host });
|
|
22
439
|
process.exit(1);
|
|
23
440
|
}
|
|
24
|
-
if (
|
|
25
|
-
logError('
|
|
441
|
+
if (config.security.allowRemote && config.auth.mode !== 'oauth') {
|
|
442
|
+
logError('Remote HTTP mode requires OAuth configuration; refusing to start');
|
|
443
|
+
process.exit(1);
|
|
444
|
+
}
|
|
445
|
+
}
|
|
446
|
+
function ensureStaticTokens() {
|
|
447
|
+
if (config.auth.mode === 'static' && config.auth.staticTokens.length === 0) {
|
|
448
|
+
logError('At least one static access token is required for HTTP mode');
|
|
26
449
|
process.exit(1);
|
|
27
450
|
}
|
|
28
451
|
}
|
|
452
|
+
function ensureOauthConfiguration() {
|
|
453
|
+
if (!config.auth.issuerUrl || !config.auth.authorizationUrl) {
|
|
454
|
+
logError('OAUTH_ISSUER_URL and OAUTH_AUTHORIZATION_URL are required for OAuth mode');
|
|
455
|
+
process.exit(1);
|
|
456
|
+
}
|
|
457
|
+
if (!config.auth.tokenUrl) {
|
|
458
|
+
logError('OAUTH_TOKEN_URL is required for OAuth mode');
|
|
459
|
+
process.exit(1);
|
|
460
|
+
}
|
|
461
|
+
if (!config.auth.introspectionUrl) {
|
|
462
|
+
logError('OAUTH_INTROSPECTION_URL is required for OAuth mode');
|
|
463
|
+
process.exit(1);
|
|
464
|
+
}
|
|
465
|
+
}
|
|
466
|
+
function createShutdownHandler(server, sessionStore, sessionCleanupController, stopRateLimitCleanup) {
|
|
467
|
+
return (signal) => shutdownServer(signal, server, sessionStore, sessionCleanupController, stopRateLimitCleanup);
|
|
468
|
+
}
|
|
469
|
+
async function shutdownServer(signal, server, sessionStore, sessionCleanupController, stopRateLimitCleanup) {
|
|
470
|
+
logInfo(`${signal} received, shutting down gracefully...`);
|
|
471
|
+
stopRateLimitCleanup();
|
|
472
|
+
sessionCleanupController.abort();
|
|
473
|
+
await closeSessions(sessionStore);
|
|
474
|
+
destroyAgents();
|
|
475
|
+
closeServer(server);
|
|
476
|
+
scheduleForcedShutdown(10000);
|
|
477
|
+
}
|
|
478
|
+
async function closeSessions(sessionStore) {
|
|
479
|
+
const sessions = sessionStore.clear();
|
|
480
|
+
await Promise.allSettled(sessions.map((session) => session.transport.close().catch((error) => {
|
|
481
|
+
logWarn('Failed to close session during shutdown', {
|
|
482
|
+
error: getErrorMessage(error),
|
|
483
|
+
});
|
|
484
|
+
})));
|
|
485
|
+
}
|
|
486
|
+
function closeServer(server) {
|
|
487
|
+
server.close(() => {
|
|
488
|
+
logInfo('HTTP server closed');
|
|
489
|
+
process.exit(0);
|
|
490
|
+
});
|
|
491
|
+
}
|
|
492
|
+
function scheduleForcedShutdown(timeoutMs) {
|
|
493
|
+
setTimeout(() => {
|
|
494
|
+
logError('Forced shutdown after timeout');
|
|
495
|
+
process.exit(1);
|
|
496
|
+
}, timeoutMs).unref();
|
|
497
|
+
}
|
|
498
|
+
function registerSignalHandlers(shutdown) {
|
|
499
|
+
process.on('SIGINT', () => {
|
|
500
|
+
void shutdown('SIGINT');
|
|
501
|
+
});
|
|
502
|
+
process.on('SIGTERM', () => {
|
|
503
|
+
void shutdown('SIGTERM');
|
|
504
|
+
});
|
|
505
|
+
}
|
|
29
506
|
function startListening(app) {
|
|
30
507
|
return app
|
|
31
508
|
.listen(config.server.port, config.server.host, () => {
|
|
@@ -33,63 +510,74 @@ function startListening(app) {
|
|
|
33
510
|
host: config.server.host,
|
|
34
511
|
port: config.server.port,
|
|
35
512
|
});
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
process.stdout.write(`\n${styleText('dim', 'Run with --stdio flag for direct stdio integration')}\n`);
|
|
513
|
+
const baseUrl = `http://${config.server.host}:${config.server.port}`;
|
|
514
|
+
logInfo(`superFetch MCP server running at ${baseUrl} (health: ${baseUrl}/health, mcp: ${baseUrl}/mcp)`);
|
|
515
|
+
logInfo('Run with --stdio flag for direct stdio integration');
|
|
40
516
|
})
|
|
41
517
|
.on('error', (err) => {
|
|
42
518
|
logError('Failed to start server', err);
|
|
43
519
|
process.exit(1);
|
|
44
520
|
});
|
|
45
521
|
}
|
|
46
|
-
function
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
})));
|
|
57
|
-
destroyAgents();
|
|
58
|
-
destroyTransformWorkers();
|
|
59
|
-
server.close(() => {
|
|
60
|
-
logInfo('HTTP server closed');
|
|
61
|
-
process.exit(0);
|
|
62
|
-
});
|
|
63
|
-
setTimeout(() => {
|
|
64
|
-
logError('Forced shutdown after timeout');
|
|
65
|
-
process.exit(1);
|
|
66
|
-
}, 10000).unref();
|
|
522
|
+
function buildMiddleware() {
|
|
523
|
+
const { middleware: rateLimitMiddleware, stop: stopRateLimitCleanup } = createRateLimitMiddleware(config.rateLimit);
|
|
524
|
+
const authMiddleware = createAuthMiddleware();
|
|
525
|
+
// No CORS - MCP clients don't run in browsers
|
|
526
|
+
const corsMiddleware = createCorsMiddleware();
|
|
527
|
+
return {
|
|
528
|
+
rateLimitMiddleware,
|
|
529
|
+
stopRateLimitCleanup,
|
|
530
|
+
authMiddleware,
|
|
531
|
+
corsMiddleware,
|
|
67
532
|
};
|
|
68
533
|
}
|
|
69
|
-
function
|
|
70
|
-
process.on('SIGINT', () => {
|
|
71
|
-
void shutdown('SIGINT');
|
|
72
|
-
});
|
|
73
|
-
process.on('SIGTERM', () => {
|
|
74
|
-
void shutdown('SIGTERM');
|
|
75
|
-
});
|
|
76
|
-
}
|
|
77
|
-
export async function startHttpServer() {
|
|
78
|
-
enableHttpMode();
|
|
79
|
-
const { app, jsonParser } = await createExpressApp();
|
|
80
|
-
const corsOptions = buildCorsOptions();
|
|
81
|
-
const { middleware: rateLimitMiddleware, stop: stopRateLimitCleanup } = createRateLimitMiddleware(config.rateLimit);
|
|
82
|
-
const authMiddleware = createAuthMiddleware(config.security.apiKey ?? '');
|
|
83
|
-
attachBaseMiddleware(app, jsonParser, rateLimitMiddleware, authMiddleware, createCorsMiddleware(corsOptions));
|
|
84
|
-
assertHttpConfiguration();
|
|
534
|
+
function createSessionInfrastructure() {
|
|
85
535
|
const sessionStore = createSessionStore(config.server.sessionTtlMs);
|
|
86
536
|
const sessionCleanupController = startSessionCleanupLoop(sessionStore, config.server.sessionTtlMs);
|
|
537
|
+
return { sessionStore, sessionCleanupController };
|
|
538
|
+
}
|
|
539
|
+
function registerHttpRoutes(app, sessionStore, authMiddleware) {
|
|
540
|
+
app.use('/mcp', authMiddleware);
|
|
541
|
+
app.use('/mcp/downloads', authMiddleware);
|
|
87
542
|
registerMcpRoutes(app, {
|
|
88
543
|
sessionStore,
|
|
89
544
|
maxSessions: config.server.maxSessions,
|
|
90
545
|
});
|
|
91
546
|
registerDownloadRoutes(app);
|
|
92
547
|
app.use(errorHandler);
|
|
548
|
+
}
|
|
549
|
+
function attachAuthMetadata(app) {
|
|
550
|
+
const authMetadataRouter = createAuthMetadataRouter();
|
|
551
|
+
if (authMetadataRouter) {
|
|
552
|
+
app.use(authMetadataRouter);
|
|
553
|
+
}
|
|
554
|
+
}
|
|
555
|
+
async function buildServerContext() {
|
|
556
|
+
const { app, authMiddleware, stopRateLimitCleanup } = await createAppWithMiddleware();
|
|
557
|
+
const { sessionStore, sessionCleanupController } = attachSessionRoutes(app, authMiddleware);
|
|
558
|
+
return { app, sessionStore, sessionCleanupController, stopRateLimitCleanup };
|
|
559
|
+
}
|
|
560
|
+
async function createAppWithMiddleware() {
|
|
561
|
+
const { app, jsonParser } = await createExpressApp();
|
|
562
|
+
const { rateLimitMiddleware, stopRateLimitCleanup, authMiddleware, corsMiddleware, } = buildMiddleware();
|
|
563
|
+
attachBaseMiddleware({
|
|
564
|
+
app,
|
|
565
|
+
jsonParser,
|
|
566
|
+
rateLimitMiddleware,
|
|
567
|
+
corsMiddleware,
|
|
568
|
+
});
|
|
569
|
+
attachAuthMetadata(app);
|
|
570
|
+
assertHttpConfiguration();
|
|
571
|
+
return { app, authMiddleware, stopRateLimitCleanup };
|
|
572
|
+
}
|
|
573
|
+
function attachSessionRoutes(app, authMiddleware) {
|
|
574
|
+
const { sessionStore, sessionCleanupController } = createSessionInfrastructure();
|
|
575
|
+
registerHttpRoutes(app, sessionStore, authMiddleware);
|
|
576
|
+
return { sessionStore, sessionCleanupController };
|
|
577
|
+
}
|
|
578
|
+
export async function startHttpServer() {
|
|
579
|
+
enableHttpMode();
|
|
580
|
+
const { app, sessionStore, sessionCleanupController, stopRateLimitCleanup } = await buildServerContext();
|
|
93
581
|
const server = startListening(app);
|
|
94
582
|
const shutdown = createShutdownHandler(server, sessionStore, sessionCleanupController, stopRateLimitCleanup);
|
|
95
583
|
registerSignalHandlers(shutdown);
|
|
@@ -98,9 +586,6 @@ export async function startHttpServer() {
|
|
|
98
586
|
async function createExpressApp() {
|
|
99
587
|
const { default: express } = await import('express');
|
|
100
588
|
const app = express();
|
|
101
|
-
if (config.server.trustProxy) {
|
|
102
|
-
app.set('trust proxy', true);
|
|
103
|
-
}
|
|
104
589
|
const jsonParser = express.json({ limit: '1mb' });
|
|
105
590
|
return { app, jsonParser };
|
|
106
591
|
}
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { setInterval as setIntervalPromise } from 'node:timers/promises';
|
|
2
2
|
import { logInfo, logWarn } from '../services/logger.js';
|
|
3
|
-
import { evictExpiredSessions } from './mcp-
|
|
3
|
+
import { evictExpiredSessions } from './mcp-session-eviction.js';
|
|
4
4
|
export function startSessionCleanupLoop(store, sessionTtlMs) {
|
|
5
5
|
const controller = new AbortController();
|
|
6
6
|
void runSessionCleanupLoop(store, sessionTtlMs, controller.signal).catch(handleSessionCleanupError);
|
|
@@ -8,11 +8,11 @@ export function startSessionCleanupLoop(store, sessionTtlMs) {
|
|
|
8
8
|
}
|
|
9
9
|
async function runSessionCleanupLoop(store, sessionTtlMs, signal) {
|
|
10
10
|
const intervalMs = getCleanupIntervalMs(sessionTtlMs);
|
|
11
|
-
for await (const
|
|
11
|
+
for await (const getNow of setIntervalPromise(intervalMs, Date.now, {
|
|
12
12
|
signal,
|
|
13
13
|
ref: false,
|
|
14
14
|
})) {
|
|
15
|
-
handleSessionEvictions(store);
|
|
15
|
+
handleSessionEvictions(store, getNow());
|
|
16
16
|
}
|
|
17
17
|
}
|
|
18
18
|
function getCleanupIntervalMs(sessionTtlMs) {
|
|
@@ -21,10 +21,13 @@ function getCleanupIntervalMs(sessionTtlMs) {
|
|
|
21
21
|
function isAbortError(error) {
|
|
22
22
|
return error instanceof Error && error.name === 'AbortError';
|
|
23
23
|
}
|
|
24
|
-
function handleSessionEvictions(store) {
|
|
24
|
+
function handleSessionEvictions(store, now) {
|
|
25
25
|
const evicted = evictExpiredSessions(store);
|
|
26
26
|
if (evicted > 0) {
|
|
27
|
-
logInfo('Expired sessions evicted', {
|
|
27
|
+
logInfo('Expired sessions evicted', {
|
|
28
|
+
evicted,
|
|
29
|
+
timestamp: new Date(now).toISOString(),
|
|
30
|
+
});
|
|
28
31
|
}
|
|
29
32
|
}
|
|
30
33
|
function handleSessionCleanupError(error) {
|
|
@@ -1,2 +1,2 @@
|
|
|
1
1
|
import type { NextFunction, Request, Response } from 'express';
|
|
2
|
-
export declare function errorHandler(err: Error, req: Request, res: Response,
|
|
2
|
+
export declare function errorHandler(err: Error, req: Request, res: Response, next: NextFunction): void;
|