@j0hanz/superfetch 2.3.0 → 2.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +4 -9
- package/dist/assets/logo.svg +24835 -0
- package/dist/cache.js +58 -4
- package/dist/config.d.ts +2 -0
- package/dist/config.js +2 -0
- package/dist/dom-noise-removal.js +15 -13
- package/dist/fetch.js +16 -25
- package/dist/http-native.js +19 -3
- package/dist/markdown-cleanup.d.ts +6 -12
- package/dist/markdown-cleanup.js +243 -25
- package/dist/mcp.js +20 -9
- package/dist/observability.d.ts +2 -0
- package/dist/observability.js +25 -0
- package/dist/tools.d.ts +5 -3
- package/dist/tools.js +27 -12
- package/dist/transform-types.d.ts +38 -0
- package/dist/transform.d.ts +12 -6
- package/dist/transform.js +120 -265
- package/package.json +1 -2
package/dist/cache.js
CHANGED
|
@@ -1,4 +1,3 @@
|
|
|
1
|
-
import { LRUCache } from 'lru-cache';
|
|
2
1
|
import { ResourceTemplate } from '@modelcontextprotocol/sdk/server/mcp.js';
|
|
3
2
|
import { ErrorCode, McpError, SubscribeRequestSchema, UnsubscribeRequestSchema, } from '@modelcontextprotocol/sdk/types.js';
|
|
4
3
|
import { config } from './config.js';
|
|
@@ -105,10 +104,65 @@ export function toResourceUri(cacheKey) {
|
|
|
105
104
|
return null;
|
|
106
105
|
return buildCacheResourceUri(parts.namespace, parts.urlHash);
|
|
107
106
|
}
|
|
108
|
-
|
|
107
|
+
// Cache behavior contract (native implementation):
|
|
108
|
+
// - Max entries: config.cache.maxKeys
|
|
109
|
+
// - TTL in ms: config.cache.ttl * 1000
|
|
110
|
+
// - Access does NOT extend TTL
|
|
111
|
+
class NativeLruCache {
|
|
112
|
+
max;
|
|
113
|
+
ttlMs;
|
|
114
|
+
entries = new Map();
|
|
115
|
+
constructor({ max, ttlMs }) {
|
|
116
|
+
this.max = max;
|
|
117
|
+
this.ttlMs = ttlMs;
|
|
118
|
+
}
|
|
119
|
+
get(key) {
|
|
120
|
+
const entry = this.entries.get(key);
|
|
121
|
+
if (!entry)
|
|
122
|
+
return undefined;
|
|
123
|
+
if (this.isExpired(entry, Date.now())) {
|
|
124
|
+
this.entries.delete(key);
|
|
125
|
+
return undefined;
|
|
126
|
+
}
|
|
127
|
+
// Refresh LRU order without extending TTL.
|
|
128
|
+
this.entries.delete(key);
|
|
129
|
+
this.entries.set(key, entry);
|
|
130
|
+
return entry.value;
|
|
131
|
+
}
|
|
132
|
+
set(key, value) {
|
|
133
|
+
if (this.max <= 0 || this.ttlMs <= 0)
|
|
134
|
+
return;
|
|
135
|
+
this.entries.delete(key);
|
|
136
|
+
this.entries.set(key, {
|
|
137
|
+
value,
|
|
138
|
+
expiresAtMs: Date.now() + this.ttlMs,
|
|
139
|
+
});
|
|
140
|
+
this.purgeExpired(Date.now());
|
|
141
|
+
while (this.entries.size > this.max) {
|
|
142
|
+
const oldestKey = this.entries.keys().next().value;
|
|
143
|
+
if (oldestKey === undefined)
|
|
144
|
+
break;
|
|
145
|
+
this.entries.delete(oldestKey);
|
|
146
|
+
}
|
|
147
|
+
}
|
|
148
|
+
keys() {
|
|
149
|
+
this.purgeExpired(Date.now());
|
|
150
|
+
return [...this.entries.keys()];
|
|
151
|
+
}
|
|
152
|
+
purgeExpired(now) {
|
|
153
|
+
for (const [key, entry] of this.entries) {
|
|
154
|
+
if (this.isExpired(entry, now)) {
|
|
155
|
+
this.entries.delete(key);
|
|
156
|
+
}
|
|
157
|
+
}
|
|
158
|
+
}
|
|
159
|
+
isExpired(entry, now) {
|
|
160
|
+
return entry.expiresAtMs <= now;
|
|
161
|
+
}
|
|
162
|
+
}
|
|
163
|
+
const contentCache = new NativeLruCache({
|
|
109
164
|
max: config.cache.maxKeys,
|
|
110
|
-
|
|
111
|
-
updateAgeOnGet: false,
|
|
165
|
+
ttlMs: config.cache.ttl * 1000,
|
|
112
166
|
});
|
|
113
167
|
const updateListeners = new Set();
|
|
114
168
|
export function onCacheUpdate(listener) {
|
package/dist/config.d.ts
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
export declare const serverVersion: string;
|
|
1
2
|
export type LogLevel = 'debug' | 'info' | 'warn' | 'error';
|
|
2
3
|
export type TransformMetadataFormat = 'markdown' | 'frontmatter';
|
|
3
4
|
interface AuthConfig {
|
|
@@ -43,6 +44,7 @@ export declare const config: {
|
|
|
43
44
|
};
|
|
44
45
|
transform: {
|
|
45
46
|
timeoutMs: number;
|
|
47
|
+
stageWarnRatio: number;
|
|
46
48
|
metadataFormat: TransformMetadataFormat;
|
|
47
49
|
};
|
|
48
50
|
tools: {
|
package/dist/config.js
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import packageJson from '../package.json' with { type: 'json' };
|
|
2
|
+
export const serverVersion = packageJson.version;
|
|
2
3
|
function buildIpv4(parts) {
|
|
3
4
|
return parts.join('.');
|
|
4
5
|
}
|
|
@@ -199,6 +200,7 @@ export const config = {
|
|
|
199
200
|
},
|
|
200
201
|
transform: {
|
|
201
202
|
timeoutMs: TIMEOUT.DEFAULT_TRANSFORM_TIMEOUT_MS,
|
|
203
|
+
stageWarnRatio: parseFloat(process.env.TRANSFORM_STAGE_WARN_RATIO ?? '0.5'),
|
|
202
204
|
metadataFormat: parseTransformMetadataFormat(process.env.TRANSFORM_METADATA_FORMAT),
|
|
203
205
|
},
|
|
204
206
|
tools: {
|
|
@@ -157,6 +157,16 @@ function getPromoTokens() {
|
|
|
157
157
|
promoTokensCache = tokens;
|
|
158
158
|
return tokens;
|
|
159
159
|
}
|
|
160
|
+
let promoRegexCache = null;
|
|
161
|
+
function getPromoRegex() {
|
|
162
|
+
if (promoRegexCache)
|
|
163
|
+
return promoRegexCache;
|
|
164
|
+
const tokens = Array.from(getPromoTokens());
|
|
165
|
+
const escaped = tokens.map((t) => t.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'));
|
|
166
|
+
const pattern = `(?:^|[^a-z0-9])(?:${escaped.join('|')})(?:$|[^a-z0-9])`;
|
|
167
|
+
promoRegexCache = new RegExp(pattern, 'i');
|
|
168
|
+
return promoRegexCache;
|
|
169
|
+
}
|
|
160
170
|
const HEADER_NOISE_PATTERN = /\b(site-header|masthead|topbar|navbar|nav(?:bar)?|menu|header-nav)\b/i;
|
|
161
171
|
const FIXED_PATTERN = /\b(fixed|sticky)\b/;
|
|
162
172
|
const HIGH_Z_PATTERN = /\bz-(?:4\d|50)\b/;
|
|
@@ -261,18 +271,9 @@ function isElementHidden(element) {
|
|
|
261
271
|
function hasNoiseRole(role) {
|
|
262
272
|
return role !== null && NAVIGATION_ROLES.has(role);
|
|
263
273
|
}
|
|
264
|
-
function tokenizeIdentifierLikeText(value) {
|
|
265
|
-
return value
|
|
266
|
-
.toLowerCase()
|
|
267
|
-
.replace(/[^a-z0-9]+/g, ' ')
|
|
268
|
-
.trim()
|
|
269
|
-
.split(' ')
|
|
270
|
-
.filter(Boolean);
|
|
271
|
-
}
|
|
272
274
|
function matchesPromoIdOrClass(className, id) {
|
|
273
|
-
const
|
|
274
|
-
|
|
275
|
-
return tokens.some((token) => promoTokens.has(token));
|
|
275
|
+
const regex = getPromoRegex();
|
|
276
|
+
return regex.test(className) || regex.test(id);
|
|
276
277
|
}
|
|
277
278
|
function matchesFixedOrHighZIsolate(className) {
|
|
278
279
|
return (FIXED_PATTERN.test(className) ||
|
|
@@ -310,9 +311,10 @@ function isNodeListLike(value) {
|
|
|
310
311
|
return isObject(value) && typeof value.length === 'number';
|
|
311
312
|
}
|
|
312
313
|
function tryGetNodeListItem(nodes, index) {
|
|
313
|
-
if (typeof nodes.item === 'function')
|
|
314
|
+
if ('item' in nodes && typeof nodes.item === 'function') {
|
|
314
315
|
return nodes.item(index);
|
|
315
|
-
|
|
316
|
+
}
|
|
317
|
+
return nodes[index] ?? null;
|
|
316
318
|
}
|
|
317
319
|
function removeNoiseFromNodeListLike(nodes, shouldCheckNoise) {
|
|
318
320
|
for (let index = nodes.length - 1; index >= 0; index -= 1) {
|
package/dist/fetch.js
CHANGED
|
@@ -300,11 +300,11 @@ export function isRawTextContentUrl(url) {
|
|
|
300
300
|
return hasKnownRawTextExtension(lowerBase);
|
|
301
301
|
}
|
|
302
302
|
function hasKnownRawTextExtension(urlBaseLower) {
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
return
|
|
303
|
+
const lastDot = urlBaseLower.lastIndexOf('.');
|
|
304
|
+
if (lastDot === -1)
|
|
305
|
+
return false;
|
|
306
|
+
const ext = urlBaseLower.slice(lastDot);
|
|
307
|
+
return RAW_TEXT_EXTENSIONS.has(ext);
|
|
308
308
|
}
|
|
309
309
|
const DNS_LOOKUP_TIMEOUT_MS = 5000;
|
|
310
310
|
const SLOW_REQUEST_THRESHOLD_MS = 5000;
|
|
@@ -314,25 +314,6 @@ function normalizeLookupResults(addresses, family) {
|
|
|
314
314
|
}
|
|
315
315
|
return [{ address: addresses, family: family ?? 4 }];
|
|
316
316
|
}
|
|
317
|
-
function findBlockedIpError(list, hostname) {
|
|
318
|
-
for (const addr of list) {
|
|
319
|
-
const ip = typeof addr === 'string' ? addr : addr.address;
|
|
320
|
-
if (!isBlockedIp(ip)) {
|
|
321
|
-
continue;
|
|
322
|
-
}
|
|
323
|
-
return createErrorWithCode(`Blocked IP detected for ${hostname}`, 'EBLOCKED');
|
|
324
|
-
}
|
|
325
|
-
return null;
|
|
326
|
-
}
|
|
327
|
-
function findInvalidFamilyError(list, hostname) {
|
|
328
|
-
for (const addr of list) {
|
|
329
|
-
const family = typeof addr === 'string' ? 0 : addr.family;
|
|
330
|
-
if (family === 4 || family === 6)
|
|
331
|
-
continue;
|
|
332
|
-
return createErrorWithCode(`Invalid address family returned for ${hostname}`, 'EINVAL');
|
|
333
|
-
}
|
|
334
|
-
return null;
|
|
335
|
-
}
|
|
336
317
|
function createNoDnsResultsError(hostname) {
|
|
337
318
|
return createErrorWithCode(`No DNS results returned for ${hostname}`, 'ENODATA');
|
|
338
319
|
}
|
|
@@ -358,7 +339,17 @@ function selectLookupResult(list, useAll, hostname) {
|
|
|
358
339
|
};
|
|
359
340
|
}
|
|
360
341
|
function findLookupError(list, hostname) {
|
|
361
|
-
|
|
342
|
+
for (const addr of list) {
|
|
343
|
+
const family = typeof addr === 'string' ? 0 : addr.family;
|
|
344
|
+
if (family !== 4 && family !== 6) {
|
|
345
|
+
return createErrorWithCode(`Invalid address family returned for ${hostname}`, 'EINVAL');
|
|
346
|
+
}
|
|
347
|
+
const ip = typeof addr === 'string' ? addr : addr.address;
|
|
348
|
+
if (isBlockedIp(ip)) {
|
|
349
|
+
return createErrorWithCode(`Blocked IP detected for ${hostname}`, 'EBLOCKED');
|
|
350
|
+
}
|
|
351
|
+
}
|
|
352
|
+
return null;
|
|
362
353
|
}
|
|
363
354
|
function normalizeAndValidateLookupResults(addresses, resolvedFamily, hostname) {
|
|
364
355
|
const list = normalizeLookupResults(addresses, resolvedFamily);
|
package/dist/http-native.js
CHANGED
|
@@ -5,8 +5,8 @@ import { URL, URLSearchParams } from 'node:url';
|
|
|
5
5
|
import { InvalidTokenError, ServerError, } from '@modelcontextprotocol/sdk/server/auth/errors.js';
|
|
6
6
|
import { StreamableHTTPServerTransport } from '@modelcontextprotocol/sdk/server/streamableHttp.js';
|
|
7
7
|
import { isInitializeRequest } from '@modelcontextprotocol/sdk/types.js';
|
|
8
|
-
import { handleDownload } from './cache.js';
|
|
9
|
-
import { config, enableHttpMode } from './config.js';
|
|
8
|
+
import { keys as cacheKeys, handleDownload } from './cache.js';
|
|
9
|
+
import { config, enableHttpMode, serverVersion } from './config.js';
|
|
10
10
|
import { timingSafeEqualUtf8 } from './crypto.js';
|
|
11
11
|
import { normalizeHost } from './host-normalization.js';
|
|
12
12
|
import { acceptsEventStream, isJsonRpcBatchRequest, isMcpRequestBody, } from './mcp-validator.js';
|
|
@@ -14,6 +14,7 @@ import { createMcpServer } from './mcp.js';
|
|
|
14
14
|
import { logError, logInfo, logWarn } from './observability.js';
|
|
15
15
|
import { applyHttpServerTuning, drainConnectionsOnShutdown, } from './server-tuning.js';
|
|
16
16
|
import { composeCloseHandlers, createSessionStore, createSlotTracker, ensureSessionCapacity, reserveSessionSlot, startSessionCleanupLoop, } from './session.js';
|
|
17
|
+
import { getTransformPoolStats } from './transform.js';
|
|
17
18
|
import { isObject } from './type-guards.js';
|
|
18
19
|
function createTransportAdapter(transportImpl) {
|
|
19
20
|
const noopOnClose = () => { };
|
|
@@ -580,7 +581,22 @@ async function dispatchRequest(req, res, url, ctx) {
|
|
|
580
581
|
const { method } = req;
|
|
581
582
|
try {
|
|
582
583
|
if (method === 'GET' && path === '/health') {
|
|
583
|
-
|
|
584
|
+
const poolStats = getTransformPoolStats();
|
|
585
|
+
res.status(200).json({
|
|
586
|
+
status: 'ok',
|
|
587
|
+
version: serverVersion,
|
|
588
|
+
uptime: Math.floor(process.uptime()),
|
|
589
|
+
timestamp: new Date().toISOString(),
|
|
590
|
+
stats: {
|
|
591
|
+
activeSessions: ctx.store.size(),
|
|
592
|
+
cacheKeys: cacheKeys().length,
|
|
593
|
+
workerPool: poolStats ?? {
|
|
594
|
+
queueDepth: 0,
|
|
595
|
+
activeWorkers: 0,
|
|
596
|
+
capacity: 0,
|
|
597
|
+
},
|
|
598
|
+
},
|
|
599
|
+
});
|
|
584
600
|
return;
|
|
585
601
|
}
|
|
586
602
|
if (!(await authenticateRequest(req, res))) {
|
|
@@ -1,16 +1,10 @@
|
|
|
1
|
-
|
|
2
|
-
* Markdown cleanup utilities for post-processing converted content.
|
|
3
|
-
*
|
|
4
|
-
* Goals:
|
|
5
|
-
* - Never mutate fenced code blocks (``` / ~~~) content.
|
|
6
|
-
* - Keep rules localized and readable.
|
|
7
|
-
* - Avoid multi-pass regexes that accidentally hit code blocks.
|
|
8
|
-
*/
|
|
9
|
-
/**
|
|
10
|
-
* Clean up common markdown artifacts and formatting issues.
|
|
11
|
-
* IMPORTANT: All rules are applied ONLY outside fenced code blocks.
|
|
12
|
-
*/
|
|
1
|
+
import type { MetadataBlock } from './transform-types.js';
|
|
13
2
|
export declare function cleanupMarkdownArtifacts(content: string): string;
|
|
3
|
+
export declare function extractTitleFromRawMarkdown(content: string): string | undefined;
|
|
4
|
+
export declare function addSourceToMarkdown(content: string, url: string): string;
|
|
5
|
+
export declare function isRawTextContent(content: string): boolean;
|
|
6
|
+
export declare function isLikelyHtmlContent(content: string): boolean;
|
|
7
|
+
export declare function buildMetadataFooter(metadata?: MetadataBlock, fallbackUrl?: string): string;
|
|
14
8
|
/**
|
|
15
9
|
* Promote standalone lines that look like headings to proper markdown headings.
|
|
16
10
|
* Fence-aware: never modifies content inside fenced code blocks.
|
package/dist/markdown-cleanup.js
CHANGED
|
@@ -1,11 +1,4 @@
|
|
|
1
|
-
|
|
2
|
-
* Markdown cleanup utilities for post-processing converted content.
|
|
3
|
-
*
|
|
4
|
-
* Goals:
|
|
5
|
-
* - Never mutate fenced code blocks (``` / ~~~) content.
|
|
6
|
-
* - Keep rules localized and readable.
|
|
7
|
-
* - Avoid multi-pass regexes that accidentally hit code blocks.
|
|
8
|
-
*/
|
|
1
|
+
import { config } from './config.js';
|
|
9
2
|
// ─────────────────────────────────────────────────────────────────────────────
|
|
10
3
|
// Fence state helpers
|
|
11
4
|
// ─────────────────────────────────────────────────────────────────────────────
|
|
@@ -76,15 +69,6 @@ function splitByFences(content) {
|
|
|
76
69
|
}
|
|
77
70
|
return segments;
|
|
78
71
|
}
|
|
79
|
-
/**
|
|
80
|
-
* Apply a transformation function only to non-fenced content.
|
|
81
|
-
*/
|
|
82
|
-
function mapOutsideFences(content, transform) {
|
|
83
|
-
const segments = splitByFences(content);
|
|
84
|
-
return segments
|
|
85
|
-
.map((seg) => (seg.inFence ? seg.content : transform(seg.content)))
|
|
86
|
-
.join('\n');
|
|
87
|
-
}
|
|
88
72
|
// ─────────────────────────────────────────────────────────────────────────────
|
|
89
73
|
// Cleanup rules (OUTSIDE fences only)
|
|
90
74
|
// ─────────────────────────────────────────────────────────────────────────────
|
|
@@ -177,17 +161,251 @@ const CLEANUP_STEPS = [
|
|
|
177
161
|
// ─────────────────────────────────────────────────────────────────────────────
|
|
178
162
|
// Public API
|
|
179
163
|
// ─────────────────────────────────────────────────────────────────────────────
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
164
|
+
function getLastLine(text) {
|
|
165
|
+
const index = text.lastIndexOf('\n');
|
|
166
|
+
return index === -1 ? text : text.slice(index + 1);
|
|
167
|
+
}
|
|
184
168
|
export function cleanupMarkdownArtifacts(content) {
|
|
185
169
|
if (!content)
|
|
186
170
|
return '';
|
|
187
|
-
const
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
171
|
+
const segments = splitByFences(content);
|
|
172
|
+
return segments
|
|
173
|
+
.map((seg, index) => {
|
|
174
|
+
if (seg.inFence)
|
|
175
|
+
return seg.content;
|
|
176
|
+
const prevSeg = segments[index - 1];
|
|
177
|
+
const prevLineContext = prevSeg ? getLastLine(prevSeg.content) : '';
|
|
178
|
+
const lines = seg.content.split('\n');
|
|
179
|
+
const promotedLines = [];
|
|
180
|
+
for (let i = 0; i < lines.length; i += 1) {
|
|
181
|
+
const line = lines[i] ?? '';
|
|
182
|
+
const prevLine = i > 0 ? (lines[i - 1] ?? '') : prevLineContext;
|
|
183
|
+
promotedLines.push(processNonFencedLine(line, prevLine));
|
|
184
|
+
}
|
|
185
|
+
const promoted = promotedLines.join('\n');
|
|
186
|
+
return CLEANUP_STEPS.reduce((text, step) => step(text), promoted);
|
|
187
|
+
})
|
|
188
|
+
.join('\n')
|
|
189
|
+
.trim();
|
|
190
|
+
}
|
|
191
|
+
// ─────────────────────────────────────────────────────────────────────────────
|
|
192
|
+
// Raw markdown handling + metadata footer
|
|
193
|
+
// ─────────────────────────────────────────────────────────────────────────────
|
|
194
|
+
const HEADING_PATTERN = /^#{1,6}\s/m;
|
|
195
|
+
const LIST_PATTERN = /^(?:[-*+])\s/m;
|
|
196
|
+
const HTML_DOCUMENT_PATTERN = /^(<!doctype|<html)/i;
|
|
197
|
+
function containsMarkdownHeading(content) {
|
|
198
|
+
return HEADING_PATTERN.test(content);
|
|
199
|
+
}
|
|
200
|
+
function containsMarkdownList(content) {
|
|
201
|
+
return LIST_PATTERN.test(content);
|
|
202
|
+
}
|
|
203
|
+
function containsFencedCodeBlock(content) {
|
|
204
|
+
const first = content.indexOf('```');
|
|
205
|
+
if (first === -1)
|
|
206
|
+
return false;
|
|
207
|
+
return content.includes('```', first + 3);
|
|
208
|
+
}
|
|
209
|
+
function looksLikeMarkdown(content) {
|
|
210
|
+
return (containsMarkdownHeading(content) ||
|
|
211
|
+
containsMarkdownList(content) ||
|
|
212
|
+
containsFencedCodeBlock(content));
|
|
213
|
+
}
|
|
214
|
+
function detectLineEnding(content) {
|
|
215
|
+
return content.includes('\r\n') ? '\r\n' : '\n';
|
|
216
|
+
}
|
|
217
|
+
const FRONTMATTER_DELIMITER = '---';
|
|
218
|
+
function findFrontmatterLines(content) {
|
|
219
|
+
const lineEnding = detectLineEnding(content);
|
|
220
|
+
const lines = content.split(lineEnding);
|
|
221
|
+
if (lines[0] !== FRONTMATTER_DELIMITER)
|
|
222
|
+
return null;
|
|
223
|
+
const endIndex = lines.indexOf(FRONTMATTER_DELIMITER, 1);
|
|
224
|
+
if (endIndex === -1)
|
|
225
|
+
return null;
|
|
226
|
+
return { lineEnding, lines, endIndex };
|
|
227
|
+
}
|
|
228
|
+
function stripOptionalQuotes(value) {
|
|
229
|
+
const trimmed = value.trim();
|
|
230
|
+
if (trimmed.length < 2)
|
|
231
|
+
return trimmed;
|
|
232
|
+
const first = trimmed[0];
|
|
233
|
+
const last = trimmed[trimmed.length - 1];
|
|
234
|
+
if ((first === '"' && last === '"') || (first === "'" && last === "'")) {
|
|
235
|
+
return trimmed.slice(1, -1).trim();
|
|
236
|
+
}
|
|
237
|
+
return trimmed;
|
|
238
|
+
}
|
|
239
|
+
function parseFrontmatterEntry(line) {
|
|
240
|
+
const trimmed = line.trim();
|
|
241
|
+
if (!trimmed)
|
|
242
|
+
return null;
|
|
243
|
+
const separatorIndex = trimmed.indexOf(':');
|
|
244
|
+
if (separatorIndex <= 0)
|
|
245
|
+
return null;
|
|
246
|
+
const key = trimmed.slice(0, separatorIndex).trim().toLowerCase();
|
|
247
|
+
const value = trimmed.slice(separatorIndex + 1);
|
|
248
|
+
return { key, value };
|
|
249
|
+
}
|
|
250
|
+
function isTitleKey(key) {
|
|
251
|
+
return key === 'title' || key === 'name';
|
|
252
|
+
}
|
|
253
|
+
function extractTitleFromHeading(content) {
|
|
254
|
+
const lineEnding = detectLineEnding(content);
|
|
255
|
+
const lines = content.split(lineEnding);
|
|
256
|
+
for (const line of lines) {
|
|
257
|
+
const trimmed = line.trim();
|
|
258
|
+
if (!trimmed)
|
|
259
|
+
continue;
|
|
260
|
+
let index = 0;
|
|
261
|
+
while (index < trimmed.length && trimmed[index] === '#') {
|
|
262
|
+
index += 1;
|
|
263
|
+
}
|
|
264
|
+
if (index === 0 || index > 6)
|
|
265
|
+
return undefined;
|
|
266
|
+
const nextChar = trimmed[index];
|
|
267
|
+
if (nextChar !== ' ' && nextChar !== '\t')
|
|
268
|
+
return undefined;
|
|
269
|
+
const heading = trimmed.slice(index).trim();
|
|
270
|
+
return heading.length > 0 ? heading : undefined;
|
|
271
|
+
}
|
|
272
|
+
return undefined;
|
|
273
|
+
}
|
|
274
|
+
export function extractTitleFromRawMarkdown(content) {
|
|
275
|
+
const frontmatter = findFrontmatterLines(content);
|
|
276
|
+
if (!frontmatter) {
|
|
277
|
+
return extractTitleFromHeading(content);
|
|
278
|
+
}
|
|
279
|
+
const { lines, endIndex } = frontmatter;
|
|
280
|
+
const entry = lines
|
|
281
|
+
.slice(1, endIndex)
|
|
282
|
+
.map((line) => parseFrontmatterEntry(line))
|
|
283
|
+
.find((parsed) => parsed !== null && isTitleKey(parsed.key));
|
|
284
|
+
if (!entry)
|
|
285
|
+
return undefined;
|
|
286
|
+
const value = stripOptionalQuotes(entry.value);
|
|
287
|
+
return value || undefined;
|
|
288
|
+
}
|
|
289
|
+
function hasMarkdownSourceLine(content) {
|
|
290
|
+
const lineEnding = detectLineEnding(content);
|
|
291
|
+
const lines = content.split(lineEnding);
|
|
292
|
+
const limit = Math.min(lines.length, 50);
|
|
293
|
+
for (let index = 0; index < limit; index += 1) {
|
|
294
|
+
const line = lines[index];
|
|
295
|
+
if (!line)
|
|
296
|
+
continue;
|
|
297
|
+
if (line.trimStart().toLowerCase().startsWith('source:')) {
|
|
298
|
+
return true;
|
|
299
|
+
}
|
|
300
|
+
}
|
|
301
|
+
return false;
|
|
302
|
+
}
|
|
303
|
+
function addSourceToMarkdownMarkdownFormat(content, url) {
|
|
304
|
+
if (hasMarkdownSourceLine(content))
|
|
305
|
+
return content;
|
|
306
|
+
const lineEnding = detectLineEnding(content);
|
|
307
|
+
const lines = content.split(lineEnding);
|
|
308
|
+
const firstNonEmptyIndex = lines.findIndex((line) => line.trim().length > 0);
|
|
309
|
+
if (firstNonEmptyIndex !== -1) {
|
|
310
|
+
const firstLine = lines[firstNonEmptyIndex];
|
|
311
|
+
if (firstLine && /^#{1,6}\s+/.test(firstLine.trim())) {
|
|
312
|
+
const insertAt = firstNonEmptyIndex + 1;
|
|
313
|
+
const updated = [
|
|
314
|
+
...lines.slice(0, insertAt),
|
|
315
|
+
'',
|
|
316
|
+
`Source: ${url}`,
|
|
317
|
+
'',
|
|
318
|
+
...lines.slice(insertAt),
|
|
319
|
+
];
|
|
320
|
+
return updated.join(lineEnding);
|
|
321
|
+
}
|
|
322
|
+
}
|
|
323
|
+
return [`Source: ${url}`, '', content].join(lineEnding);
|
|
324
|
+
}
|
|
325
|
+
export function addSourceToMarkdown(content, url) {
|
|
326
|
+
const frontmatter = findFrontmatterLines(content);
|
|
327
|
+
if (config.transform.metadataFormat === 'markdown' && !frontmatter) {
|
|
328
|
+
return addSourceToMarkdownMarkdownFormat(content, url);
|
|
329
|
+
}
|
|
330
|
+
if (!frontmatter) {
|
|
331
|
+
return `---\nsource: "${url}"\n---\n\n${content}`;
|
|
332
|
+
}
|
|
333
|
+
const { lineEnding, lines, endIndex } = frontmatter;
|
|
334
|
+
const bodyLines = lines.slice(1, endIndex);
|
|
335
|
+
const hasSource = bodyLines.some((line) => line.trimStart().toLowerCase().startsWith('source:'));
|
|
336
|
+
if (hasSource)
|
|
337
|
+
return content;
|
|
338
|
+
const updatedLines = [
|
|
339
|
+
lines[0],
|
|
340
|
+
...bodyLines,
|
|
341
|
+
`source: "${url}"`,
|
|
342
|
+
...lines.slice(endIndex),
|
|
343
|
+
];
|
|
344
|
+
return updatedLines.join(lineEnding);
|
|
345
|
+
}
|
|
346
|
+
function hasFrontmatter(trimmed) {
|
|
347
|
+
return trimmed.startsWith('---\n') || trimmed.startsWith('---\r\n');
|
|
348
|
+
}
|
|
349
|
+
function looksLikeHtmlDocument(trimmed) {
|
|
350
|
+
return HTML_DOCUMENT_PATTERN.test(trimmed);
|
|
351
|
+
}
|
|
352
|
+
function countCommonHtmlTags(content) {
|
|
353
|
+
const matches = content.match(/<(html|head|body|div|span|script|style|meta|link)\b/gi) ??
|
|
354
|
+
[];
|
|
355
|
+
return matches.length;
|
|
356
|
+
}
|
|
357
|
+
export function isRawTextContent(content) {
|
|
358
|
+
const trimmed = content.trim();
|
|
359
|
+
const isHtmlDocument = looksLikeHtmlDocument(trimmed);
|
|
360
|
+
const hasMarkdownFrontmatter = hasFrontmatter(trimmed);
|
|
361
|
+
const hasTooManyHtmlTags = countCommonHtmlTags(content) > 2;
|
|
362
|
+
const isMarkdown = looksLikeMarkdown(content);
|
|
363
|
+
return (!isHtmlDocument &&
|
|
364
|
+
(hasMarkdownFrontmatter || (!hasTooManyHtmlTags && isMarkdown)));
|
|
365
|
+
}
|
|
366
|
+
export function isLikelyHtmlContent(content) {
|
|
367
|
+
const trimmed = content.trim();
|
|
368
|
+
if (!trimmed)
|
|
369
|
+
return false;
|
|
370
|
+
if (looksLikeHtmlDocument(trimmed))
|
|
371
|
+
return true;
|
|
372
|
+
return countCommonHtmlTags(content) > 2;
|
|
373
|
+
}
|
|
374
|
+
function formatFetchedDate(isoString) {
|
|
375
|
+
try {
|
|
376
|
+
const date = new Date(isoString);
|
|
377
|
+
const day = String(date.getDate()).padStart(2, '0');
|
|
378
|
+
const month = String(date.getMonth() + 1).padStart(2, '0');
|
|
379
|
+
const year = date.getFullYear();
|
|
380
|
+
return `${day}-${month}-${year}`;
|
|
381
|
+
}
|
|
382
|
+
catch {
|
|
383
|
+
return isoString;
|
|
384
|
+
}
|
|
385
|
+
}
|
|
386
|
+
export function buildMetadataFooter(metadata, fallbackUrl) {
|
|
387
|
+
if (!metadata)
|
|
388
|
+
return '';
|
|
389
|
+
const lines = ['---', ''];
|
|
390
|
+
const url = metadata.url || fallbackUrl;
|
|
391
|
+
const parts = [];
|
|
392
|
+
if (metadata.title)
|
|
393
|
+
parts.push(`_${metadata.title}_`);
|
|
394
|
+
if (metadata.author)
|
|
395
|
+
parts.push(`_${metadata.author}_`);
|
|
396
|
+
if (url)
|
|
397
|
+
parts.push(`[_Original Source_](${url})`);
|
|
398
|
+
if (metadata.fetchedAt) {
|
|
399
|
+
const formattedDate = formatFetchedDate(metadata.fetchedAt);
|
|
400
|
+
parts.push(`_${formattedDate}_`);
|
|
401
|
+
}
|
|
402
|
+
if (parts.length > 0) {
|
|
403
|
+
lines.push(` ${parts.join(' | ')}`);
|
|
404
|
+
}
|
|
405
|
+
if (metadata.description) {
|
|
406
|
+
lines.push(` <sub>${metadata.description}</sub>`);
|
|
407
|
+
}
|
|
408
|
+
return lines.join('\n');
|
|
191
409
|
}
|
|
192
410
|
// ─────────────────────────────────────────────────────────────────────────────
|
|
193
411
|
// Heading Promotion (fence-aware)
|
package/dist/mcp.js
CHANGED
|
@@ -4,37 +4,47 @@ import { StdioServerTransport } from '@modelcontextprotocol/sdk/server/stdio.js'
|
|
|
4
4
|
import { registerCachedContentResource } from './cache.js';
|
|
5
5
|
import { config } from './config.js';
|
|
6
6
|
import { destroyAgents } from './fetch.js';
|
|
7
|
-
import { logError, logInfo } from './observability.js';
|
|
7
|
+
import { logError, logInfo, setMcpServer } from './observability.js';
|
|
8
8
|
import { registerTools } from './tools.js';
|
|
9
9
|
import { shutdownTransformWorkerPool } from './transform.js';
|
|
10
|
+
function getLocalIconData() {
|
|
11
|
+
try {
|
|
12
|
+
const iconPath = new URL('../assets/logo.svg', import.meta.url);
|
|
13
|
+
const buffer = readFileSync(iconPath);
|
|
14
|
+
return `data:image/svg+xml;base64,${buffer.toString('base64')}`;
|
|
15
|
+
}
|
|
16
|
+
catch {
|
|
17
|
+
return undefined;
|
|
18
|
+
}
|
|
19
|
+
}
|
|
10
20
|
function createServerInfo() {
|
|
21
|
+
const localIcon = getLocalIconData();
|
|
11
22
|
return {
|
|
12
23
|
name: config.server.name,
|
|
13
24
|
version: config.server.version,
|
|
25
|
+
...(localIcon ? { icons: [{ src: localIcon, sizes: ['any'] }] } : {}),
|
|
14
26
|
};
|
|
15
27
|
}
|
|
16
28
|
function createServerCapabilities() {
|
|
17
29
|
return {
|
|
18
30
|
tools: { listChanged: true },
|
|
19
31
|
resources: { listChanged: true, subscribe: true },
|
|
32
|
+
logging: {},
|
|
20
33
|
};
|
|
21
34
|
}
|
|
22
35
|
function createServerInstructions(serverVersion) {
|
|
23
36
|
try {
|
|
24
|
-
const raw = readFileSync(new URL('./instructions.md', import.meta.url),
|
|
25
|
-
|
|
26
|
-
});
|
|
27
|
-
const resolved = raw.replaceAll('{{SERVER_VERSION}}', serverVersion);
|
|
28
|
-
return resolved.trim();
|
|
37
|
+
const raw = readFileSync(new URL('./instructions.md', import.meta.url), 'utf8');
|
|
38
|
+
return raw.replaceAll('{{SERVER_VERSION}}', serverVersion).trim();
|
|
29
39
|
}
|
|
30
40
|
catch {
|
|
31
|
-
return `
|
|
41
|
+
return `Instructions unavailable | ${serverVersion}`;
|
|
32
42
|
}
|
|
33
43
|
}
|
|
34
44
|
function registerInstructionsResource(server, instructions) {
|
|
35
45
|
server.registerResource('instructions', new ResourceTemplate('internal://instructions', { list: undefined }), {
|
|
36
|
-
title:
|
|
37
|
-
description: '
|
|
46
|
+
title: `SuperFetch MCP | ${config.server.version}`,
|
|
47
|
+
description: 'Guidance for using the superFetch MCP server.',
|
|
38
48
|
mimeType: 'text/markdown',
|
|
39
49
|
}, (uri) => ({
|
|
40
50
|
contents: [
|
|
@@ -52,6 +62,7 @@ export function createMcpServer() {
|
|
|
52
62
|
capabilities: createServerCapabilities(),
|
|
53
63
|
instructions,
|
|
54
64
|
});
|
|
65
|
+
setMcpServer(server);
|
|
55
66
|
registerTools(server);
|
|
56
67
|
registerCachedContentResource(server);
|
|
57
68
|
registerInstructionsResource(server, instructions);
|
package/dist/observability.d.ts
CHANGED
|
@@ -1,9 +1,11 @@
|
|
|
1
|
+
import type { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
|
|
1
2
|
export type LogMetadata = Record<string, unknown>;
|
|
2
3
|
interface RequestContext {
|
|
3
4
|
readonly requestId: string;
|
|
4
5
|
readonly sessionId?: string;
|
|
5
6
|
readonly operationId?: string;
|
|
6
7
|
}
|
|
8
|
+
export declare function setMcpServer(server: McpServer): void;
|
|
7
9
|
export declare function runWithRequestContext<T>(context: RequestContext, fn: () => T): T;
|
|
8
10
|
export declare function getRequestId(): string | undefined;
|
|
9
11
|
export declare function getSessionId(): string | undefined;
|