@j0hanz/superfetch 2.0.0 → 2.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +139 -46
- package/dist/cache.d.ts +42 -0
- package/dist/cache.js +565 -0
- package/dist/config/env-parsers.d.ts +1 -0
- package/dist/config/env-parsers.js +12 -0
- package/dist/config/index.d.ts +7 -0
- package/dist/config/index.js +20 -8
- package/dist/config/types/content.d.ts +1 -0
- package/dist/config.d.ts +77 -0
- package/dist/config.js +261 -0
- package/dist/crypto.d.ts +2 -0
- package/dist/crypto.js +32 -0
- package/dist/errors.d.ts +10 -0
- package/dist/errors.js +28 -0
- package/dist/fetch.d.ts +40 -0
- package/dist/fetch.js +910 -0
- package/dist/http/auth.js +161 -2
- package/dist/http/base-middleware.d.ts +7 -0
- package/dist/http/base-middleware.js +143 -0
- package/dist/http/cors.d.ts +0 -5
- package/dist/http/cors.js +0 -6
- package/dist/http/download-routes.js +6 -2
- package/dist/http/error-handler.d.ts +2 -0
- package/dist/http/error-handler.js +55 -0
- package/dist/http/host-allowlist.d.ts +3 -0
- package/dist/http/host-allowlist.js +117 -0
- package/dist/http/mcp-routes.d.ts +8 -2
- package/dist/http/mcp-routes.js +101 -8
- package/dist/http/mcp-session-eviction.d.ts +3 -0
- package/dist/http/mcp-session-eviction.js +24 -0
- package/dist/http/mcp-session-init.d.ts +7 -0
- package/dist/http/mcp-session-init.js +94 -0
- package/dist/http/mcp-session-slots.d.ts +17 -0
- package/dist/http/mcp-session-slots.js +55 -0
- package/dist/http/mcp-session-transport-init.d.ts +7 -0
- package/dist/http/mcp-session-transport-init.js +41 -0
- package/dist/http/mcp-session-types.d.ts +5 -0
- package/dist/http/mcp-session-types.js +1 -0
- package/dist/http/mcp-session.d.ts +9 -9
- package/dist/http/mcp-session.js +5 -114
- package/dist/http/mcp-sessions.d.ts +41 -0
- package/dist/http/mcp-sessions.js +392 -0
- package/dist/http/rate-limit.js +2 -2
- package/dist/http/server-middleware.d.ts +6 -1
- package/dist/http/server-middleware.js +3 -117
- package/dist/http/server-shutdown.js +1 -1
- package/dist/http/server-tuning.d.ts +9 -0
- package/dist/http/server-tuning.js +45 -0
- package/dist/http/server.js +206 -9
- package/dist/http/session-cleanup.js +8 -5
- package/dist/http.d.ts +78 -0
- package/dist/http.js +1437 -0
- package/dist/index.js +3 -3
- package/dist/mcp.d.ts +3 -0
- package/dist/mcp.js +94 -0
- package/dist/middleware/error-handler.d.ts +1 -1
- package/dist/middleware/error-handler.js +31 -30
- package/dist/observability.d.ts +16 -0
- package/dist/observability.js +78 -0
- package/dist/resources/cached-content-params.d.ts +5 -0
- package/dist/resources/cached-content-params.js +36 -0
- package/dist/resources/cached-content.js +33 -33
- package/dist/server.js +21 -6
- package/dist/services/cache-events.d.ts +8 -0
- package/dist/services/cache-events.js +19 -0
- package/dist/services/cache.d.ts +5 -4
- package/dist/services/cache.js +49 -45
- package/dist/services/context.d.ts +2 -0
- package/dist/services/context.js +3 -0
- package/dist/services/extractor.d.ts +1 -0
- package/dist/services/extractor.js +77 -40
- package/dist/services/fetcher/agents.js +1 -1
- package/dist/services/fetcher/dns-selection.js +1 -1
- package/dist/services/fetcher/interceptors.js +29 -60
- package/dist/services/fetcher/redirects.js +12 -4
- package/dist/services/fetcher/response.js +18 -8
- package/dist/services/fetcher.d.ts +23 -0
- package/dist/services/fetcher.js +553 -13
- package/dist/services/logger.js +4 -1
- package/dist/services/telemetry.d.ts +19 -0
- package/dist/services/telemetry.js +43 -0
- package/dist/services/transform-worker-pool.d.ts +10 -3
- package/dist/services/transform-worker-pool.js +213 -184
- package/dist/tools/handlers/fetch-single.shared.d.ts +11 -3
- package/dist/tools/handlers/fetch-single.shared.js +131 -2
- package/dist/tools/handlers/fetch-url.tool.d.ts +6 -0
- package/dist/tools/handlers/fetch-url.tool.js +56 -12
- package/dist/tools/index.d.ts +1 -0
- package/dist/tools/index.js +13 -1
- package/dist/tools/schemas.d.ts +2 -0
- package/dist/tools/schemas.js +8 -0
- package/dist/tools/utils/content-shaping.js +19 -4
- package/dist/tools/utils/content-transform-core.d.ts +5 -0
- package/dist/tools/utils/content-transform-core.js +180 -0
- package/dist/tools/utils/content-transform-workers.d.ts +1 -0
- package/dist/tools/utils/content-transform-workers.js +1 -0
- package/dist/tools/utils/content-transform.d.ts +2 -1
- package/dist/tools/utils/content-transform.js +37 -136
- package/dist/tools/utils/fetch-pipeline.js +47 -56
- package/dist/tools/utils/frontmatter.d.ts +3 -0
- package/dist/tools/utils/frontmatter.js +73 -0
- package/dist/tools/utils/markdown-heuristics.d.ts +1 -0
- package/dist/tools/utils/markdown-heuristics.js +19 -0
- package/dist/tools/utils/markdown-signals.d.ts +1 -0
- package/dist/tools/utils/markdown-signals.js +19 -0
- package/dist/tools/utils/raw-markdown-frontmatter.d.ts +3 -0
- package/dist/tools/utils/raw-markdown-frontmatter.js +73 -0
- package/dist/tools/utils/raw-markdown.d.ts +6 -0
- package/dist/tools/utils/raw-markdown.js +149 -0
- package/dist/tools.d.ts +104 -0
- package/dist/tools.js +421 -0
- package/dist/transform.d.ts +69 -0
- package/dist/transform.js +1509 -0
- package/dist/transformers/markdown/fenced-code-rule.d.ts +2 -0
- package/dist/transformers/markdown/fenced-code-rule.js +38 -0
- package/dist/transformers/markdown/frontmatter.d.ts +2 -0
- package/dist/transformers/markdown/frontmatter.js +45 -0
- package/dist/transformers/markdown/noise-rule.d.ts +2 -0
- package/dist/transformers/markdown/noise-rule.js +80 -0
- package/dist/transformers/markdown/turndown-instance.d.ts +2 -0
- package/dist/transformers/markdown/turndown-instance.js +19 -0
- package/dist/transformers/markdown.d.ts +5 -0
- package/dist/transformers/markdown.js +314 -0
- package/dist/transformers/markdown.transformer.js +2 -189
- package/dist/utils/cancellation.d.ts +1 -0
- package/dist/utils/cancellation.js +18 -0
- package/dist/utils/code-language-bash.d.ts +1 -0
- package/dist/utils/code-language-bash.js +48 -0
- package/dist/utils/code-language-core.d.ts +2 -0
- package/dist/utils/code-language-core.js +13 -0
- package/dist/utils/code-language-detectors.d.ts +5 -0
- package/dist/utils/code-language-detectors.js +142 -0
- package/dist/utils/code-language-helpers.d.ts +5 -0
- package/dist/utils/code-language-helpers.js +62 -0
- package/dist/utils/code-language-parsing.d.ts +5 -0
- package/dist/utils/code-language-parsing.js +62 -0
- package/dist/utils/code-language.js +250 -46
- package/dist/utils/error-details.d.ts +3 -0
- package/dist/utils/error-details.js +12 -0
- package/dist/utils/filename-generator.js +14 -3
- package/dist/utils/host-normalizer.d.ts +1 -0
- package/dist/utils/host-normalizer.js +37 -0
- package/dist/utils/ip-address.d.ts +4 -0
- package/dist/utils/ip-address.js +6 -0
- package/dist/utils/tool-error-handler.js +12 -17
- package/dist/utils/url-redactor.d.ts +1 -0
- package/dist/utils/url-redactor.js +13 -0
- package/dist/utils/url-validator.js +35 -20
- package/dist/workers/transform-worker.js +82 -38
- package/package.json +13 -10
package/dist/cache.js
ADDED
|
@@ -0,0 +1,565 @@
|
|
|
1
|
+
import { setInterval as setIntervalPromise } from 'node:timers/promises';
|
|
2
|
+
import { ResourceTemplate } from '@modelcontextprotocol/sdk/server/mcp.js';
|
|
3
|
+
import { ErrorCode, McpError, SubscribeRequestSchema, UnsubscribeRequestSchema, } from '@modelcontextprotocol/sdk/types.js';
|
|
4
|
+
import { config } from './config.js';
|
|
5
|
+
import { sha256Hex } from './crypto.js';
|
|
6
|
+
import { getErrorMessage } from './errors.js';
|
|
7
|
+
import { logDebug, logWarn } from './observability.js';
|
|
8
|
+
function isRecord(value) {
|
|
9
|
+
return typeof value === 'object' && value !== null;
|
|
10
|
+
}
|
|
11
|
+
export function parseCachedPayload(raw) {
|
|
12
|
+
try {
|
|
13
|
+
const parsed = JSON.parse(raw);
|
|
14
|
+
return isCachedPayload(parsed) ? parsed : null;
|
|
15
|
+
}
|
|
16
|
+
catch {
|
|
17
|
+
return null;
|
|
18
|
+
}
|
|
19
|
+
}
|
|
20
|
+
export function resolveCachedPayloadContent(payload) {
|
|
21
|
+
if (typeof payload.markdown === 'string') {
|
|
22
|
+
return payload.markdown;
|
|
23
|
+
}
|
|
24
|
+
if (typeof payload.content === 'string') {
|
|
25
|
+
return payload.content;
|
|
26
|
+
}
|
|
27
|
+
return null;
|
|
28
|
+
}
|
|
29
|
+
function hasOptionalStringProperty(value, key) {
|
|
30
|
+
const prop = value[key];
|
|
31
|
+
if (prop === undefined)
|
|
32
|
+
return true;
|
|
33
|
+
return typeof prop === 'string';
|
|
34
|
+
}
|
|
35
|
+
function isCachedPayload(value) {
|
|
36
|
+
if (!isRecord(value))
|
|
37
|
+
return false;
|
|
38
|
+
if (!hasOptionalStringProperty(value, 'content'))
|
|
39
|
+
return false;
|
|
40
|
+
if (!hasOptionalStringProperty(value, 'markdown'))
|
|
41
|
+
return false;
|
|
42
|
+
if (!hasOptionalStringProperty(value, 'title'))
|
|
43
|
+
return false;
|
|
44
|
+
return true;
|
|
45
|
+
}
|
|
46
|
+
const CACHE_HASH = {
|
|
47
|
+
URL_HASH_LENGTH: 16,
|
|
48
|
+
VARY_HASH_LENGTH: 12,
|
|
49
|
+
};
|
|
50
|
+
function stableStringify(value) {
|
|
51
|
+
if (!isRecord(value)) {
|
|
52
|
+
if (value === null || value === undefined) {
|
|
53
|
+
return '';
|
|
54
|
+
}
|
|
55
|
+
return JSON.stringify(value);
|
|
56
|
+
}
|
|
57
|
+
if (Array.isArray(value)) {
|
|
58
|
+
return `[${value.map((item) => stableStringify(item)).join(',')}]`;
|
|
59
|
+
}
|
|
60
|
+
const entries = Object.entries(value)
|
|
61
|
+
.filter(([, entryValue]) => entryValue !== undefined)
|
|
62
|
+
.sort(([a], [b]) => a.localeCompare(b))
|
|
63
|
+
.map(([key, entryValue]) => `${JSON.stringify(key)}:${stableStringify(entryValue)}`);
|
|
64
|
+
return `{${entries.join(',')}}`;
|
|
65
|
+
}
|
|
66
|
+
function createHashFragment(input, length) {
|
|
67
|
+
return sha256Hex(input).substring(0, length);
|
|
68
|
+
}
|
|
69
|
+
function buildCacheKey(namespace, urlHash, varyHash) {
|
|
70
|
+
return varyHash
|
|
71
|
+
? `${namespace}:${urlHash}.${varyHash}`
|
|
72
|
+
: `${namespace}:${urlHash}`;
|
|
73
|
+
}
|
|
74
|
+
function getVaryHash(vary) {
|
|
75
|
+
if (!vary)
|
|
76
|
+
return undefined;
|
|
77
|
+
const varyString = typeof vary === 'string' ? vary : stableStringify(vary);
|
|
78
|
+
if (!varyString)
|
|
79
|
+
return undefined;
|
|
80
|
+
return createHashFragment(varyString, CACHE_HASH.VARY_HASH_LENGTH);
|
|
81
|
+
}
|
|
82
|
+
export function createCacheKey(namespace, url, vary) {
|
|
83
|
+
if (!namespace || !url)
|
|
84
|
+
return null;
|
|
85
|
+
const urlHash = createHashFragment(url, CACHE_HASH.URL_HASH_LENGTH);
|
|
86
|
+
const varyHash = getVaryHash(vary);
|
|
87
|
+
return buildCacheKey(namespace, urlHash, varyHash);
|
|
88
|
+
}
|
|
89
|
+
export function parseCacheKey(cacheKey) {
|
|
90
|
+
if (!cacheKey)
|
|
91
|
+
return null;
|
|
92
|
+
const [namespace, ...rest] = cacheKey.split(':');
|
|
93
|
+
const urlHash = rest.join(':');
|
|
94
|
+
if (!namespace || !urlHash)
|
|
95
|
+
return null;
|
|
96
|
+
return { namespace, urlHash };
|
|
97
|
+
}
|
|
98
|
+
export function toResourceUri(cacheKey) {
|
|
99
|
+
const parts = parseCacheKey(cacheKey);
|
|
100
|
+
if (!parts)
|
|
101
|
+
return null;
|
|
102
|
+
return `superfetch://cache/${parts.namespace}/${parts.urlHash}`;
|
|
103
|
+
}
|
|
104
|
+
const contentCache = new Map();
|
|
105
|
+
let cleanupController = null;
|
|
106
|
+
const updateListeners = new Set();
|
|
107
|
+
export function onCacheUpdate(listener) {
|
|
108
|
+
updateListeners.add(listener);
|
|
109
|
+
return () => {
|
|
110
|
+
updateListeners.delete(listener);
|
|
111
|
+
};
|
|
112
|
+
}
|
|
113
|
+
function notifyCacheUpdate(cacheKey) {
|
|
114
|
+
if (updateListeners.size === 0)
|
|
115
|
+
return;
|
|
116
|
+
const parts = parseCacheKey(cacheKey);
|
|
117
|
+
if (!parts)
|
|
118
|
+
return;
|
|
119
|
+
const event = { cacheKey, ...parts };
|
|
120
|
+
for (const listener of updateListeners) {
|
|
121
|
+
listener(event);
|
|
122
|
+
}
|
|
123
|
+
}
|
|
124
|
+
function startCleanupLoop() {
|
|
125
|
+
if (cleanupController)
|
|
126
|
+
return;
|
|
127
|
+
cleanupController = new AbortController();
|
|
128
|
+
void runCleanupLoop(cleanupController.signal).catch((error) => {
|
|
129
|
+
if (error instanceof Error && error.name !== 'AbortError') {
|
|
130
|
+
logWarn('Cache cleanup loop failed', { error: getErrorMessage(error) });
|
|
131
|
+
}
|
|
132
|
+
});
|
|
133
|
+
}
|
|
134
|
+
async function runCleanupLoop(signal) {
|
|
135
|
+
const intervalMs = Math.floor(config.cache.ttl * 1000);
|
|
136
|
+
for await (const getNow of setIntervalPromise(intervalMs, Date.now, {
|
|
137
|
+
signal,
|
|
138
|
+
ref: false,
|
|
139
|
+
})) {
|
|
140
|
+
enforceCacheLimits(getNow());
|
|
141
|
+
}
|
|
142
|
+
}
|
|
143
|
+
function enforceCacheLimits(now) {
|
|
144
|
+
for (const [key, item] of contentCache.entries()) {
|
|
145
|
+
if (now > item.expiresAt) {
|
|
146
|
+
contentCache.delete(key);
|
|
147
|
+
}
|
|
148
|
+
}
|
|
149
|
+
trimCacheToMaxKeys();
|
|
150
|
+
}
|
|
151
|
+
export function get(cacheKey) {
|
|
152
|
+
if (!isCacheReadable(cacheKey))
|
|
153
|
+
return undefined;
|
|
154
|
+
return runCacheOperation(cacheKey, 'Cache get error', () => readCacheEntry(cacheKey));
|
|
155
|
+
}
|
|
156
|
+
function isCacheReadable(cacheKey) {
|
|
157
|
+
return config.cache.enabled && Boolean(cacheKey);
|
|
158
|
+
}
|
|
159
|
+
function isCacheWritable(cacheKey, content) {
|
|
160
|
+
return config.cache.enabled && Boolean(cacheKey) && Boolean(content);
|
|
161
|
+
}
|
|
162
|
+
function runCacheOperation(cacheKey, message, operation) {
|
|
163
|
+
try {
|
|
164
|
+
return operation();
|
|
165
|
+
}
|
|
166
|
+
catch (error) {
|
|
167
|
+
logCacheError(message, cacheKey, error);
|
|
168
|
+
return undefined;
|
|
169
|
+
}
|
|
170
|
+
}
|
|
171
|
+
function readCacheEntry(cacheKey) {
|
|
172
|
+
const now = Date.now();
|
|
173
|
+
return readCacheItem(cacheKey, now)?.entry;
|
|
174
|
+
}
|
|
175
|
+
function isExpired(item, now) {
|
|
176
|
+
return now > item.expiresAt;
|
|
177
|
+
}
|
|
178
|
+
function readCacheItem(cacheKey, now) {
|
|
179
|
+
const item = contentCache.get(cacheKey);
|
|
180
|
+
if (!item)
|
|
181
|
+
return undefined;
|
|
182
|
+
if (isExpired(item, now)) {
|
|
183
|
+
contentCache.delete(cacheKey);
|
|
184
|
+
return undefined;
|
|
185
|
+
}
|
|
186
|
+
return item;
|
|
187
|
+
}
|
|
188
|
+
export function set(cacheKey, content, metadata) {
|
|
189
|
+
if (!isCacheWritable(cacheKey, content))
|
|
190
|
+
return;
|
|
191
|
+
runCacheOperation(cacheKey, 'Cache set error', () => {
|
|
192
|
+
startCleanupLoop();
|
|
193
|
+
const now = Date.now();
|
|
194
|
+
const expiresAtMs = now + config.cache.ttl * 1000;
|
|
195
|
+
const entry = buildCacheEntry({
|
|
196
|
+
content,
|
|
197
|
+
metadata,
|
|
198
|
+
fetchedAtMs: now,
|
|
199
|
+
expiresAtMs,
|
|
200
|
+
});
|
|
201
|
+
persistCacheEntry(cacheKey, entry, expiresAtMs);
|
|
202
|
+
});
|
|
203
|
+
}
|
|
204
|
+
export function keys() {
|
|
205
|
+
return Array.from(contentCache.keys());
|
|
206
|
+
}
|
|
207
|
+
export function isEnabled() {
|
|
208
|
+
return config.cache.enabled;
|
|
209
|
+
}
|
|
210
|
+
function buildCacheEntry({ content, metadata, fetchedAtMs, expiresAtMs, }) {
|
|
211
|
+
return {
|
|
212
|
+
url: metadata.url,
|
|
213
|
+
content,
|
|
214
|
+
fetchedAt: new Date(fetchedAtMs).toISOString(),
|
|
215
|
+
expiresAt: new Date(expiresAtMs).toISOString(),
|
|
216
|
+
...(metadata.title === undefined ? {} : { title: metadata.title }),
|
|
217
|
+
};
|
|
218
|
+
}
|
|
219
|
+
function persistCacheEntry(cacheKey, entry, expiresAtMs) {
|
|
220
|
+
contentCache.set(cacheKey, { entry, expiresAt: expiresAtMs });
|
|
221
|
+
trimCacheToMaxKeys();
|
|
222
|
+
notifyCacheUpdate(cacheKey);
|
|
223
|
+
}
|
|
224
|
+
function trimCacheToMaxKeys() {
|
|
225
|
+
if (contentCache.size <= config.cache.maxKeys)
|
|
226
|
+
return;
|
|
227
|
+
removeOldestEntries(contentCache.size - config.cache.maxKeys);
|
|
228
|
+
}
|
|
229
|
+
function removeOldestEntries(count) {
|
|
230
|
+
const iterator = contentCache.keys();
|
|
231
|
+
for (let removed = 0; removed < count; removed += 1) {
|
|
232
|
+
const next = iterator.next();
|
|
233
|
+
if (next.done)
|
|
234
|
+
break;
|
|
235
|
+
contentCache.delete(next.value);
|
|
236
|
+
}
|
|
237
|
+
}
|
|
238
|
+
function logCacheError(message, cacheKey, error) {
|
|
239
|
+
logWarn(message, {
|
|
240
|
+
key: cacheKey.length > 100 ? cacheKey.slice(0, 100) : cacheKey,
|
|
241
|
+
error: getErrorMessage(error),
|
|
242
|
+
});
|
|
243
|
+
}
|
|
244
|
+
const CACHE_NAMESPACE = 'markdown';
|
|
245
|
+
const HASH_PATTERN = /^[a-f0-9.]+$/i;
|
|
246
|
+
function resolveCacheParams(params) {
|
|
247
|
+
const parsed = requireRecordParams(params);
|
|
248
|
+
const namespace = requireParamString(parsed, 'namespace');
|
|
249
|
+
const urlHash = requireParamString(parsed, 'urlHash');
|
|
250
|
+
if (!isValidNamespace(namespace) || !isValidHash(urlHash)) {
|
|
251
|
+
throw new McpError(ErrorCode.InvalidParams, 'Invalid cache resource parameters');
|
|
252
|
+
}
|
|
253
|
+
return { namespace, urlHash };
|
|
254
|
+
}
|
|
255
|
+
function requireRecordParams(value) {
|
|
256
|
+
if (!isRecord(value)) {
|
|
257
|
+
throw new McpError(ErrorCode.InvalidParams, 'Invalid cache resource parameters');
|
|
258
|
+
}
|
|
259
|
+
return value;
|
|
260
|
+
}
|
|
261
|
+
function requireParamString(params, key) {
|
|
262
|
+
const raw = params[key];
|
|
263
|
+
const resolved = resolveStringParam(raw);
|
|
264
|
+
if (!resolved) {
|
|
265
|
+
throw new McpError(ErrorCode.InvalidParams, 'Both namespace and urlHash parameters are required');
|
|
266
|
+
}
|
|
267
|
+
return resolved;
|
|
268
|
+
}
|
|
269
|
+
function isValidNamespace(namespace) {
|
|
270
|
+
return namespace === CACHE_NAMESPACE;
|
|
271
|
+
}
|
|
272
|
+
function isValidHash(hash) {
|
|
273
|
+
return HASH_PATTERN.test(hash) && hash.length >= 8 && hash.length <= 64;
|
|
274
|
+
}
|
|
275
|
+
function resolveStringParam(value) {
|
|
276
|
+
return typeof value === 'string' ? value : null;
|
|
277
|
+
}
|
|
278
|
+
function buildResourceEntry(namespace, urlHash) {
|
|
279
|
+
return {
|
|
280
|
+
name: `${namespace}:${urlHash}`,
|
|
281
|
+
uri: `superfetch://cache/${namespace}/${urlHash}`,
|
|
282
|
+
description: `Cached content entry for ${namespace}`,
|
|
283
|
+
mimeType: 'text/markdown',
|
|
284
|
+
};
|
|
285
|
+
}
|
|
286
|
+
function listCachedResources() {
|
|
287
|
+
const resources = keys()
|
|
288
|
+
.map((key) => {
|
|
289
|
+
const parts = parseCacheKey(key);
|
|
290
|
+
if (parts?.namespace !== CACHE_NAMESPACE)
|
|
291
|
+
return null;
|
|
292
|
+
return buildResourceEntry(parts.namespace, parts.urlHash);
|
|
293
|
+
})
|
|
294
|
+
.filter((entry) => entry !== null);
|
|
295
|
+
return { resources };
|
|
296
|
+
}
|
|
297
|
+
function appendServerOnClose(server, handler) {
|
|
298
|
+
const previousOnClose = server.server.onclose;
|
|
299
|
+
server.server.onclose = () => {
|
|
300
|
+
previousOnClose?.();
|
|
301
|
+
handler();
|
|
302
|
+
};
|
|
303
|
+
}
|
|
304
|
+
function registerResourceSubscriptionHandlers(server) {
|
|
305
|
+
const subscriptions = new Set();
|
|
306
|
+
server.server.setRequestHandler(SubscribeRequestSchema, (request) => {
|
|
307
|
+
subscriptions.add(request.params.uri);
|
|
308
|
+
return {};
|
|
309
|
+
});
|
|
310
|
+
server.server.setRequestHandler(UnsubscribeRequestSchema, (request) => {
|
|
311
|
+
subscriptions.delete(request.params.uri);
|
|
312
|
+
return {};
|
|
313
|
+
});
|
|
314
|
+
appendServerOnClose(server, () => {
|
|
315
|
+
subscriptions.clear();
|
|
316
|
+
});
|
|
317
|
+
return subscriptions;
|
|
318
|
+
}
|
|
319
|
+
function notifyResourceUpdate(server, uri, subscriptions) {
|
|
320
|
+
if (!server.isConnected())
|
|
321
|
+
return;
|
|
322
|
+
if (!subscriptions.has(uri))
|
|
323
|
+
return;
|
|
324
|
+
void server.server.sendResourceUpdated({ uri }).catch((error) => {
|
|
325
|
+
logWarn('Failed to send resource update notification', {
|
|
326
|
+
uri,
|
|
327
|
+
error: getErrorMessage(error),
|
|
328
|
+
});
|
|
329
|
+
});
|
|
330
|
+
}
|
|
331
|
+
export function registerCachedContentResource(server) {
|
|
332
|
+
const subscriptions = registerResourceSubscriptionHandlers(server);
|
|
333
|
+
registerCacheContentResource(server);
|
|
334
|
+
registerCacheUpdateSubscription(server, subscriptions);
|
|
335
|
+
}
|
|
336
|
+
function buildCachedContentResponse(uri, cacheKey) {
|
|
337
|
+
const cached = requireCacheEntry(cacheKey);
|
|
338
|
+
return buildMarkdownContentResponse(uri, cached.content);
|
|
339
|
+
}
|
|
340
|
+
function registerCacheContentResource(server) {
|
|
341
|
+
server.registerResource('cached-content', new ResourceTemplate('superfetch://cache/{namespace}/{urlHash}', {
|
|
342
|
+
list: listCachedResources,
|
|
343
|
+
}), {
|
|
344
|
+
title: 'Cached Content',
|
|
345
|
+
description: 'Access previously fetched web content from cache. Namespace: markdown. UrlHash: SHA-256 hash of the URL.',
|
|
346
|
+
mimeType: 'text/plain',
|
|
347
|
+
}, (uri, params) => {
|
|
348
|
+
const { namespace, urlHash } = resolveCacheParams(params);
|
|
349
|
+
const cacheKey = `${namespace}:${urlHash}`;
|
|
350
|
+
return buildCachedContentResponse(uri, cacheKey);
|
|
351
|
+
});
|
|
352
|
+
}
|
|
353
|
+
function registerCacheUpdateSubscription(server, subscriptions) {
|
|
354
|
+
const unsubscribe = onCacheUpdate(({ cacheKey }) => {
|
|
355
|
+
const resourceUri = toResourceUri(cacheKey);
|
|
356
|
+
if (!resourceUri)
|
|
357
|
+
return;
|
|
358
|
+
notifyResourceUpdate(server, resourceUri, subscriptions);
|
|
359
|
+
if (server.isConnected()) {
|
|
360
|
+
server.sendResourceListChanged();
|
|
361
|
+
}
|
|
362
|
+
});
|
|
363
|
+
appendServerOnClose(server, unsubscribe);
|
|
364
|
+
}
|
|
365
|
+
function requireCacheEntry(cacheKey) {
|
|
366
|
+
const cached = get(cacheKey);
|
|
367
|
+
if (!cached) {
|
|
368
|
+
throw new McpError(-32002, `Content not found in cache for key: ${cacheKey}`);
|
|
369
|
+
}
|
|
370
|
+
return cached;
|
|
371
|
+
}
|
|
372
|
+
function buildMarkdownContentResponse(uri, content) {
|
|
373
|
+
const payload = parseCachedPayload(content);
|
|
374
|
+
const resolvedContent = payload ? resolveCachedPayloadContent(payload) : null;
|
|
375
|
+
if (!resolvedContent) {
|
|
376
|
+
throw new McpError(ErrorCode.InternalError, 'Cached markdown content is missing');
|
|
377
|
+
}
|
|
378
|
+
return {
|
|
379
|
+
contents: [
|
|
380
|
+
{
|
|
381
|
+
uri: uri.href,
|
|
382
|
+
mimeType: 'text/markdown',
|
|
383
|
+
text: resolvedContent,
|
|
384
|
+
},
|
|
385
|
+
],
|
|
386
|
+
};
|
|
387
|
+
}
|
|
388
|
+
function validateNamespace(namespace) {
|
|
389
|
+
return namespace === 'markdown';
|
|
390
|
+
}
|
|
391
|
+
function validateHash(hash) {
|
|
392
|
+
return HASH_PATTERN.test(hash) && hash.length >= 8 && hash.length <= 64;
|
|
393
|
+
}
|
|
394
|
+
function isSingleParam(value) {
|
|
395
|
+
return typeof value === 'string';
|
|
396
|
+
}
|
|
397
|
+
function parseDownloadParams(req) {
|
|
398
|
+
const { namespace, hash } = req.params;
|
|
399
|
+
if (!isSingleParam(namespace) || !isSingleParam(hash))
|
|
400
|
+
return null;
|
|
401
|
+
if (!namespace || !hash)
|
|
402
|
+
return null;
|
|
403
|
+
if (!validateNamespace(namespace))
|
|
404
|
+
return null;
|
|
405
|
+
if (!validateHash(hash))
|
|
406
|
+
return null;
|
|
407
|
+
return { namespace, hash };
|
|
408
|
+
}
|
|
409
|
+
function buildCacheKeyFromParams(params) {
|
|
410
|
+
return `${params.namespace}:${params.hash}`;
|
|
411
|
+
}
|
|
412
|
+
function respondBadRequest(res, message) {
|
|
413
|
+
res.status(400).json({
|
|
414
|
+
error: message,
|
|
415
|
+
code: 'BAD_REQUEST',
|
|
416
|
+
});
|
|
417
|
+
}
|
|
418
|
+
function respondNotFound(res) {
|
|
419
|
+
res.status(404).json({
|
|
420
|
+
error: 'Content not found or expired',
|
|
421
|
+
code: 'NOT_FOUND',
|
|
422
|
+
});
|
|
423
|
+
}
|
|
424
|
+
function respondServiceUnavailable(res) {
|
|
425
|
+
res.status(503).json({
|
|
426
|
+
error: 'Download service is disabled',
|
|
427
|
+
code: 'SERVICE_UNAVAILABLE',
|
|
428
|
+
});
|
|
429
|
+
}
|
|
430
|
+
export function generateSafeFilename(url, title, hashFallback, extension = '.md') {
|
|
431
|
+
const fromUrl = extractFilenameFromUrl(url);
|
|
432
|
+
if (fromUrl)
|
|
433
|
+
return sanitizeFilename(fromUrl, extension);
|
|
434
|
+
if (title) {
|
|
435
|
+
const fromTitle = slugifyTitle(title);
|
|
436
|
+
if (fromTitle)
|
|
437
|
+
return sanitizeFilename(fromTitle, extension);
|
|
438
|
+
}
|
|
439
|
+
if (hashFallback) {
|
|
440
|
+
return `${hashFallback.substring(0, 16)}${extension}`;
|
|
441
|
+
}
|
|
442
|
+
return `download-${Date.now()}${extension}`;
|
|
443
|
+
}
|
|
444
|
+
function getLastPathSegment(url) {
|
|
445
|
+
const segments = url.pathname.split('/').filter(Boolean);
|
|
446
|
+
if (segments.length === 0)
|
|
447
|
+
return null;
|
|
448
|
+
const lastSegment = segments[segments.length - 1];
|
|
449
|
+
return lastSegment ?? null;
|
|
450
|
+
}
|
|
451
|
+
function stripCommonPageExtension(segment) {
|
|
452
|
+
return segment.replace(/\.(html?|php|aspx?|jsp)$/i, '');
|
|
453
|
+
}
|
|
454
|
+
function normalizeUrlFilenameSegment(segment) {
|
|
455
|
+
const cleaned = stripCommonPageExtension(segment);
|
|
456
|
+
if (!cleaned)
|
|
457
|
+
return null;
|
|
458
|
+
if (cleaned === 'index')
|
|
459
|
+
return null;
|
|
460
|
+
return cleaned;
|
|
461
|
+
}
|
|
462
|
+
function extractFilenameFromUrl(url) {
|
|
463
|
+
try {
|
|
464
|
+
const urlObj = new URL(url);
|
|
465
|
+
const lastSegment = getLastPathSegment(urlObj);
|
|
466
|
+
if (!lastSegment)
|
|
467
|
+
return null;
|
|
468
|
+
return normalizeUrlFilenameSegment(lastSegment);
|
|
469
|
+
}
|
|
470
|
+
catch {
|
|
471
|
+
return null;
|
|
472
|
+
}
|
|
473
|
+
}
|
|
474
|
+
const MAX_FILENAME_LENGTH = 200;
|
|
475
|
+
const UNSAFE_CHARS_REGEX = /[<>:"/\\|?*]|\p{C}/gu;
|
|
476
|
+
const WHITESPACE_REGEX = /\s+/g;
|
|
477
|
+
function trimHyphens(value) {
|
|
478
|
+
let start = 0;
|
|
479
|
+
let end = value.length;
|
|
480
|
+
while (start < end && value[start] === '-') {
|
|
481
|
+
start += 1;
|
|
482
|
+
}
|
|
483
|
+
while (end > start && value[end - 1] === '-') {
|
|
484
|
+
end -= 1;
|
|
485
|
+
}
|
|
486
|
+
return value.slice(start, end);
|
|
487
|
+
}
|
|
488
|
+
function slugifyTitle(title) {
|
|
489
|
+
const slug = title
|
|
490
|
+
.toLowerCase()
|
|
491
|
+
.trim()
|
|
492
|
+
.replace(UNSAFE_CHARS_REGEX, '')
|
|
493
|
+
.replace(WHITESPACE_REGEX, '-')
|
|
494
|
+
.replace(/-+/g, '-');
|
|
495
|
+
const trimmed = trimHyphens(slug);
|
|
496
|
+
return trimmed || null;
|
|
497
|
+
}
|
|
498
|
+
function sanitizeFilename(name, extension) {
|
|
499
|
+
let sanitized = name
|
|
500
|
+
.replace(UNSAFE_CHARS_REGEX, '')
|
|
501
|
+
.replace(WHITESPACE_REGEX, '-')
|
|
502
|
+
.trim();
|
|
503
|
+
// Truncate if too long
|
|
504
|
+
const maxBase = MAX_FILENAME_LENGTH - extension.length;
|
|
505
|
+
if (sanitized.length > maxBase) {
|
|
506
|
+
sanitized = sanitized.substring(0, maxBase);
|
|
507
|
+
}
|
|
508
|
+
return `${sanitized}${extension}`;
|
|
509
|
+
}
|
|
510
|
+
function resolveDownloadPayload(params, cacheEntry) {
|
|
511
|
+
const payload = parseCachedPayload(cacheEntry.content);
|
|
512
|
+
if (!payload)
|
|
513
|
+
return null;
|
|
514
|
+
const content = resolveCachedPayloadContent(payload);
|
|
515
|
+
if (!content)
|
|
516
|
+
return null;
|
|
517
|
+
const safeTitle = typeof payload.title === 'string' ? payload.title : undefined;
|
|
518
|
+
const fileName = generateSafeFilename(cacheEntry.url, cacheEntry.title ?? safeTitle, params.hash, '.md');
|
|
519
|
+
return {
|
|
520
|
+
content,
|
|
521
|
+
contentType: 'text/markdown; charset=utf-8',
|
|
522
|
+
fileName,
|
|
523
|
+
};
|
|
524
|
+
}
|
|
525
|
+
function buildContentDisposition(fileName) {
|
|
526
|
+
const encodedName = encodeURIComponent(fileName).replace(/'/g, '%27');
|
|
527
|
+
return `attachment; filename="${fileName}"; filename*=UTF-8''${encodedName}`;
|
|
528
|
+
}
|
|
529
|
+
function sendDownloadPayload(res, payload) {
|
|
530
|
+
const disposition = buildContentDisposition(payload.fileName);
|
|
531
|
+
res.setHeader('Content-Type', payload.contentType);
|
|
532
|
+
res.setHeader('Content-Disposition', disposition);
|
|
533
|
+
res.setHeader('Cache-Control', `private, max-age=${config.cache.ttl}`);
|
|
534
|
+
res.setHeader('X-Content-Type-Options', 'nosniff');
|
|
535
|
+
res.send(payload.content);
|
|
536
|
+
}
|
|
537
|
+
function handleDownload(req, res) {
|
|
538
|
+
if (!config.cache.enabled) {
|
|
539
|
+
respondServiceUnavailable(res);
|
|
540
|
+
return;
|
|
541
|
+
}
|
|
542
|
+
const params = parseDownloadParams(req);
|
|
543
|
+
if (!params) {
|
|
544
|
+
respondBadRequest(res, 'Invalid namespace or hash format');
|
|
545
|
+
return;
|
|
546
|
+
}
|
|
547
|
+
const cacheKey = buildCacheKeyFromParams(params);
|
|
548
|
+
const cacheEntry = get(cacheKey);
|
|
549
|
+
if (!cacheEntry) {
|
|
550
|
+
logDebug('Download request for missing cache key', { cacheKey });
|
|
551
|
+
respondNotFound(res);
|
|
552
|
+
return;
|
|
553
|
+
}
|
|
554
|
+
const payload = resolveDownloadPayload(params, cacheEntry);
|
|
555
|
+
if (!payload) {
|
|
556
|
+
logDebug('Download payload unavailable', { cacheKey });
|
|
557
|
+
respondNotFound(res);
|
|
558
|
+
return;
|
|
559
|
+
}
|
|
560
|
+
logDebug('Serving download', { cacheKey, fileName: payload.fileName });
|
|
561
|
+
sendDownloadPayload(res, payload);
|
|
562
|
+
}
|
|
563
|
+
export function registerDownloadRoutes(app) {
|
|
564
|
+
app.get('/mcp/downloads/:namespace/:hash', handleDownload);
|
|
565
|
+
}
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import type { LogLevel } from './types/runtime.js';
|
|
2
2
|
export declare function parseInteger(envValue: string | undefined, defaultValue: number, min?: number, max?: number): number;
|
|
3
|
+
export declare function parseOptionalInteger(envValue: string | undefined, min?: number, max?: number): number | undefined;
|
|
3
4
|
export declare function parseBoolean(envValue: string | undefined, defaultValue: boolean): boolean;
|
|
4
5
|
export declare function parseList(envValue: string | undefined): string[];
|
|
5
6
|
export declare function parseUrlEnv(value: string | undefined, name: string): URL | undefined;
|
|
@@ -45,6 +45,18 @@ export function parseInteger(envValue, defaultValue, min, max) {
|
|
|
45
45
|
return defaultValue;
|
|
46
46
|
return parsed;
|
|
47
47
|
}
|
|
48
|
+
export function parseOptionalInteger(envValue, min, max) {
|
|
49
|
+
if (!envValue)
|
|
50
|
+
return undefined;
|
|
51
|
+
const parsed = parseInt(envValue, 10);
|
|
52
|
+
if (Number.isNaN(parsed))
|
|
53
|
+
return undefined;
|
|
54
|
+
if (isBelowMin(parsed, min))
|
|
55
|
+
return undefined;
|
|
56
|
+
if (isAboveMax(parsed, max))
|
|
57
|
+
return undefined;
|
|
58
|
+
return parsed;
|
|
59
|
+
}
|
|
48
60
|
export function parseBoolean(envValue, defaultValue) {
|
|
49
61
|
if (!envValue)
|
|
50
62
|
return defaultValue;
|
package/dist/config/index.d.ts
CHANGED
|
@@ -10,6 +10,13 @@ export declare const config: {
|
|
|
10
10
|
sessionTtlMs: number;
|
|
11
11
|
sessionInitTimeoutMs: number;
|
|
12
12
|
maxSessions: number;
|
|
13
|
+
http: {
|
|
14
|
+
headersTimeoutMs: number | undefined;
|
|
15
|
+
requestTimeoutMs: number | undefined;
|
|
16
|
+
keepAliveTimeoutMs: number | undefined;
|
|
17
|
+
shutdownCloseIdleConnections: boolean;
|
|
18
|
+
shutdownCloseAllConnections: boolean;
|
|
19
|
+
};
|
|
13
20
|
};
|
|
14
21
|
fetcher: {
|
|
15
22
|
timeout: number;
|
package/dist/config/index.js
CHANGED
|
@@ -1,17 +1,22 @@
|
|
|
1
|
+
import { buildIpv4 } from '../utils/ip-address.js';
|
|
1
2
|
import packageJson from '../../package.json' with { type: 'json' };
|
|
2
3
|
import { buildAuthConfig } from './auth-config.js';
|
|
3
4
|
import { SIZE_LIMITS, TIMEOUT } from './constants.js';
|
|
4
|
-
import { parseAllowedHosts, parseBoolean, parseInteger, parseLogLevel, } from './env-parsers.js';
|
|
5
|
+
import { parseAllowedHosts, parseBoolean, parseInteger, parseLogLevel, parseOptionalInteger, } from './env-parsers.js';
|
|
5
6
|
function formatHostForUrl(hostname) {
|
|
6
7
|
if (hostname.includes(':') && !hostname.startsWith('[')) {
|
|
7
8
|
return `[${hostname}]`;
|
|
8
9
|
}
|
|
9
10
|
return hostname;
|
|
10
11
|
}
|
|
11
|
-
const
|
|
12
|
+
const LOOPBACK_V4 = buildIpv4([127, 0, 0, 1]);
|
|
13
|
+
const ANY_V4 = buildIpv4([0, 0, 0, 0]);
|
|
14
|
+
const METADATA_V4_AWS = buildIpv4([169, 254, 169, 254]);
|
|
15
|
+
const METADATA_V4_AZURE = buildIpv4([100, 100, 100, 200]);
|
|
16
|
+
const host = process.env.HOST ?? LOOPBACK_V4;
|
|
12
17
|
const port = parseInteger(process.env.PORT, 3000, 1024, 65535);
|
|
13
18
|
const baseUrl = new URL(`http://${formatHostForUrl(host)}:${port}`);
|
|
14
|
-
const
|
|
19
|
+
const allowRemote = parseBoolean(process.env.ALLOW_REMOTE, false);
|
|
15
20
|
const runtimeState = {
|
|
16
21
|
httpMode: false,
|
|
17
22
|
};
|
|
@@ -24,6 +29,13 @@ export const config = {
|
|
|
24
29
|
sessionTtlMs: TIMEOUT.DEFAULT_SESSION_TTL_MS,
|
|
25
30
|
sessionInitTimeoutMs: 10000,
|
|
26
31
|
maxSessions: 200,
|
|
32
|
+
http: {
|
|
33
|
+
headersTimeoutMs: parseOptionalInteger(process.env.SERVER_HEADERS_TIMEOUT_MS, 1000, 600000),
|
|
34
|
+
requestTimeoutMs: parseOptionalInteger(process.env.SERVER_REQUEST_TIMEOUT_MS, 1000, 600000),
|
|
35
|
+
keepAliveTimeoutMs: parseOptionalInteger(process.env.SERVER_KEEP_ALIVE_TIMEOUT_MS, 1000, 600000),
|
|
36
|
+
shutdownCloseIdleConnections: parseBoolean(process.env.SERVER_SHUTDOWN_CLOSE_IDLE, false),
|
|
37
|
+
shutdownCloseAllConnections: parseBoolean(process.env.SERVER_SHUTDOWN_CLOSE_ALL, false),
|
|
38
|
+
},
|
|
27
39
|
},
|
|
28
40
|
fetcher: {
|
|
29
41
|
timeout: TIMEOUT.DEFAULT_FETCH_TIMEOUT_MS,
|
|
@@ -51,13 +63,13 @@ export const config = {
|
|
|
51
63
|
security: {
|
|
52
64
|
blockedHosts: new Set([
|
|
53
65
|
'localhost',
|
|
54
|
-
|
|
55
|
-
|
|
66
|
+
LOOPBACK_V4,
|
|
67
|
+
ANY_V4,
|
|
56
68
|
'::1',
|
|
57
|
-
|
|
69
|
+
METADATA_V4_AWS,
|
|
58
70
|
'metadata.google.internal',
|
|
59
71
|
'metadata.azure.com',
|
|
60
|
-
|
|
72
|
+
METADATA_V4_AZURE,
|
|
61
73
|
'instance-data',
|
|
62
74
|
]),
|
|
63
75
|
blockedIpPatterns: [
|
|
@@ -79,7 +91,7 @@ export const config = {
|
|
|
79
91
|
],
|
|
80
92
|
allowedHosts: parseAllowedHosts(process.env.ALLOWED_HOSTS),
|
|
81
93
|
apiKey: process.env.API_KEY,
|
|
82
|
-
allowRemote
|
|
94
|
+
allowRemote,
|
|
83
95
|
},
|
|
84
96
|
auth: buildAuthConfig(baseUrl),
|
|
85
97
|
rateLimit: {
|