@j0hanz/fetch-url-mcp 1.5.1 → 1.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/http/auth.js +4 -1
- package/dist/http/native.js +14 -2
- package/dist/lib/content.js +1 -1
- package/dist/lib/core.d.ts +1 -0
- package/dist/lib/core.js +6 -24
- package/dist/lib/http.d.ts +2 -0
- package/dist/lib/http.js +23 -7
- package/dist/lib/net-utils.d.ts +3 -0
- package/dist/lib/net-utils.js +21 -0
- package/dist/lib/task-handlers.js +3 -0
- package/dist/lib/types.d.ts +4 -0
- package/dist/lib/types.js +1 -0
- package/dist/lib/url.js +1 -20
- package/dist/prompts/index.d.ts +1 -5
- package/dist/resources/index.d.ts +1 -5
- package/dist/resources/index.js +1 -0
- package/dist/resources/instructions.js +3 -3
- package/dist/schemas/outputs.d.ts +0 -1
- package/dist/schemas/outputs.js +0 -5
- package/dist/server.js +5 -7
- package/dist/tasks/execution.js +2 -1
- package/dist/tasks/manager.js +2 -0
- package/dist/tools/fetch-url.js +8 -73
- package/package.json +2 -2
package/dist/http/auth.js
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { Buffer } from 'node:buffer';
|
|
2
2
|
import { randomBytes } from 'node:crypto';
|
|
3
3
|
import { InvalidTokenError, ServerError, } from '@modelcontextprotocol/sdk/server/auth/errors.js';
|
|
4
|
-
import { config } from '../lib/core.js';
|
|
4
|
+
import { config, logWarn } from '../lib/core.js';
|
|
5
5
|
import { normalizeHost } from '../lib/url.js';
|
|
6
6
|
import { hmacSha256Hex, timingSafeEqualUtf8 } from '../lib/utils.js';
|
|
7
7
|
import { isObject } from '../lib/utils.js';
|
|
@@ -186,6 +186,9 @@ export function ensureMcpProtocolVersion(req, res, options) {
|
|
|
186
186
|
// Permissive backward-compat fallback: clients predating MCP 2025-03-26 do not
|
|
187
187
|
// send MCP-Protocol-Version. Accepting requests without the header keeps older
|
|
188
188
|
// integrations working. Pass requireHeader: true to enforce strict version checking.
|
|
189
|
+
logWarn('MCP-Protocol-Version header missing; defaulting to permissive fallback', {
|
|
190
|
+
remoteAddress: req.socket.remoteAddress,
|
|
191
|
+
});
|
|
189
192
|
return true;
|
|
190
193
|
}
|
|
191
194
|
sendError(res, -32600, 'Missing MCP-Protocol-Version header');
|
package/dist/http/native.js
CHANGED
|
@@ -139,6 +139,11 @@ class McpSessionGateway {
|
|
|
139
139
|
sendError(ctx.res, -32600, 'Session not found', 404);
|
|
140
140
|
return;
|
|
141
141
|
}
|
|
142
|
+
const fingerprint = buildAuthFingerprint(ctx.auth);
|
|
143
|
+
if (!fingerprint || session.authFingerprint !== fingerprint) {
|
|
144
|
+
sendError(ctx.res, -32600, 'Session not found', 404);
|
|
145
|
+
return;
|
|
146
|
+
}
|
|
142
147
|
if (!ensureMcpProtocolVersion(ctx.req, ctx.res, {
|
|
143
148
|
requireHeader: true,
|
|
144
149
|
expectedVersion: session.negotiatedProtocolVersion,
|
|
@@ -147,7 +152,9 @@ class McpSessionGateway {
|
|
|
147
152
|
}
|
|
148
153
|
const acceptHeader = getHeaderValue(ctx.req, 'accept');
|
|
149
154
|
if (!acceptsEventStream(acceptHeader)) {
|
|
150
|
-
sendJson(ctx.res,
|
|
155
|
+
sendJson(ctx.res, 406, {
|
|
156
|
+
error: 'Not Acceptable: expected text/event-stream',
|
|
157
|
+
});
|
|
151
158
|
return;
|
|
152
159
|
}
|
|
153
160
|
this.store.touch(sessionId);
|
|
@@ -164,6 +171,11 @@ class McpSessionGateway {
|
|
|
164
171
|
sendError(ctx.res, -32600, 'Session not found', 404);
|
|
165
172
|
return;
|
|
166
173
|
}
|
|
174
|
+
const fingerprint = buildAuthFingerprint(ctx.auth);
|
|
175
|
+
if (!fingerprint || session.authFingerprint !== fingerprint) {
|
|
176
|
+
sendError(ctx.res, -32600, 'Session not found', 404);
|
|
177
|
+
return;
|
|
178
|
+
}
|
|
167
179
|
if (!ensureMcpProtocolVersion(ctx.req, ctx.res, {
|
|
168
180
|
requireHeader: true,
|
|
169
181
|
expectedVersion: session.negotiatedProtocolVersion,
|
|
@@ -172,7 +184,7 @@ class McpSessionGateway {
|
|
|
172
184
|
}
|
|
173
185
|
await session.transport.close();
|
|
174
186
|
this.cleanupSessionRecord(sessionId, 'session-delete');
|
|
175
|
-
|
|
187
|
+
sendJson(ctx.res, 200, { status: 'closed' });
|
|
176
188
|
}
|
|
177
189
|
async getOrCreateTransport(ctx, requestId) {
|
|
178
190
|
const sessionId = getMcpSessionId(ctx.req);
|
package/dist/lib/content.js
CHANGED
|
@@ -19,7 +19,7 @@ const HEADER_NOISE_PATTERN = /\b(site-header|masthead|topbar|navbar|nav(?:bar)?|
|
|
|
19
19
|
const FIXED_OR_HIGH_Z_PATTERN = /\b(?:fixed|sticky|z-(?:4\d|50)|isolate)\b/;
|
|
20
20
|
const SKIP_URL_PREFIXES = [
|
|
21
21
|
'#',
|
|
22
|
-
'
|
|
22
|
+
'javascript:',
|
|
23
23
|
'mailto:',
|
|
24
24
|
'tel:',
|
|
25
25
|
'data:',
|
package/dist/lib/core.d.ts
CHANGED
|
@@ -188,6 +188,7 @@ export declare function keys(): readonly string[];
|
|
|
188
188
|
export declare function getEntryMeta(cacheKey: string): {
|
|
189
189
|
url: string;
|
|
190
190
|
title?: string;
|
|
191
|
+
fetchedAt?: string;
|
|
191
192
|
} | undefined;
|
|
192
193
|
export declare function isEnabled(): boolean;
|
|
193
194
|
type LogMetadata = Record<string, unknown>;
|
package/dist/lib/core.js
CHANGED
|
@@ -4,11 +4,11 @@ import { accessSync, constants as fsConstants, readFileSync } from 'node:fs';
|
|
|
4
4
|
import { findPackageJSON } from 'node:module';
|
|
5
5
|
import { isIP } from 'node:net';
|
|
6
6
|
import process from 'node:process';
|
|
7
|
-
import { domainToASCII } from 'node:url';
|
|
8
7
|
import { inspect, stripVTControlCharacters } from 'node:util';
|
|
9
8
|
import {} from '@modelcontextprotocol/sdk/server/mcp.js';
|
|
10
9
|
import {} from '@modelcontextprotocol/sdk/server/streamableHttp.js';
|
|
11
10
|
import { z } from 'zod';
|
|
11
|
+
import { buildIpv4, normalizeHostname, stripTrailingDots, } from './net-utils.js';
|
|
12
12
|
import { getErrorMessage, isAbortError, sha256Hex, stableStringify as stableJsonStringify, startAbortableIntervalLoop, } from './utils.js';
|
|
13
13
|
export const serverVersion = readServerVersion(import.meta.url);
|
|
14
14
|
const LOG_LEVELS = ['debug', 'info', 'warn', 'error'];
|
|
@@ -57,31 +57,11 @@ function loadEnvFileIfAvailable() {
|
|
|
57
57
|
}
|
|
58
58
|
loadEnvFileIfAvailable();
|
|
59
59
|
const { env } = process;
|
|
60
|
-
function buildIpv4(parts) {
|
|
61
|
-
return parts.join('.');
|
|
62
|
-
}
|
|
63
|
-
function stripTrailingDots(value) {
|
|
64
|
-
let result = value;
|
|
65
|
-
while (result.endsWith('.'))
|
|
66
|
-
result = result.slice(0, -1);
|
|
67
|
-
return result;
|
|
68
|
-
}
|
|
69
60
|
function formatHostForUrl(hostname) {
|
|
70
61
|
if (hostname.includes(':') && !hostname.startsWith('['))
|
|
71
62
|
return `[${hostname}]`;
|
|
72
63
|
return hostname;
|
|
73
64
|
}
|
|
74
|
-
function normalizeHostname(value) {
|
|
75
|
-
const trimmed = value.trim();
|
|
76
|
-
if (!trimmed)
|
|
77
|
-
return null;
|
|
78
|
-
const lowered = trimmed.toLowerCase();
|
|
79
|
-
const ipType = isIP(lowered);
|
|
80
|
-
if (ipType)
|
|
81
|
-
return stripTrailingDots(lowered);
|
|
82
|
-
const ascii = domainToASCII(lowered);
|
|
83
|
-
return ascii ? stripTrailingDots(ascii) : null;
|
|
84
|
-
}
|
|
85
65
|
function normalizeHostValue(value) {
|
|
86
66
|
const raw = value.trim();
|
|
87
67
|
if (!raw)
|
|
@@ -706,9 +686,11 @@ export function getEntryMeta(cacheKey) {
|
|
|
706
686
|
const entry = store.peek(cacheKey);
|
|
707
687
|
if (!entry)
|
|
708
688
|
return undefined;
|
|
709
|
-
return
|
|
710
|
-
|
|
711
|
-
|
|
689
|
+
return {
|
|
690
|
+
url: entry.url,
|
|
691
|
+
...(entry.title !== undefined ? { title: entry.title } : {}),
|
|
692
|
+
...(entry.fetchedAt ? { fetchedAt: entry.fetchedAt } : {}),
|
|
693
|
+
};
|
|
712
694
|
}
|
|
713
695
|
export function isEnabled() {
|
|
714
696
|
return store.isEnabled();
|
package/dist/lib/http.d.ts
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { type ServerResponse } from 'node:http';
|
|
2
|
+
import { Agent } from 'undici';
|
|
2
3
|
import { type TransformResult } from './url.js';
|
|
3
4
|
export declare function generateSafeFilename(url: string, title?: string, hashFallback?: string, extension?: string): string;
|
|
4
5
|
export declare function handleDownload(res: ServerResponse, namespace: string, hash: string): void;
|
|
@@ -27,6 +28,7 @@ export declare function recordFetchError(context: FetchTelemetryContext, error:
|
|
|
27
28
|
export declare function fetchWithRedirects(url: string, init: RequestInit, maxRedirects: number): Promise<{
|
|
28
29
|
response: Response;
|
|
29
30
|
url: string;
|
|
31
|
+
agent?: Agent;
|
|
30
32
|
}>;
|
|
31
33
|
export declare function readResponseText(response: Response, url: string, maxBytes: number, signal?: AbortSignal, encoding?: string): Promise<{
|
|
32
34
|
text: string;
|
package/dist/lib/http.js
CHANGED
|
@@ -388,15 +388,20 @@ class RedirectFollower {
|
|
|
388
388
|
throw createFetchError({ kind: 'too-many-redirects' }, currentUrl);
|
|
389
389
|
}
|
|
390
390
|
visited.add(currentUrl);
|
|
391
|
-
const { response, nextUrl } = await this.withRedirectErrorContext(currentUrl, async () => {
|
|
391
|
+
const { response, nextUrl, agent: returnedAgent, } = await this.withRedirectErrorContext(currentUrl, async () => {
|
|
392
392
|
let ipAddress;
|
|
393
393
|
if (this.preflight) {
|
|
394
394
|
ipAddress = await this.preflight(currentUrl, init.signal ?? undefined);
|
|
395
395
|
}
|
|
396
396
|
return this.performFetchCycle(currentUrl, init, redirectLimit, redirectCount, ipAddress);
|
|
397
397
|
});
|
|
398
|
-
if (!nextUrl)
|
|
399
|
-
return {
|
|
398
|
+
if (!nextUrl) {
|
|
399
|
+
return {
|
|
400
|
+
response,
|
|
401
|
+
url: currentUrl,
|
|
402
|
+
...(returnedAgent ? { agent: returnedAgent } : {}),
|
|
403
|
+
};
|
|
404
|
+
}
|
|
400
405
|
currentUrl = nextUrl;
|
|
401
406
|
}
|
|
402
407
|
throw createFetchError({ kind: 'too-many-redirects' }, currentUrl);
|
|
@@ -430,10 +435,14 @@ class RedirectFollower {
|
|
|
430
435
|
});
|
|
431
436
|
fetchInit.dispatcher = agent;
|
|
432
437
|
}
|
|
438
|
+
let closeAgent = true;
|
|
433
439
|
try {
|
|
434
440
|
const response = await this.fetchFn(currentUrl, fetchInit);
|
|
435
|
-
if
|
|
436
|
-
|
|
441
|
+
// Only follow redirects if the status code indicates a redirect and there's a Location header.
|
|
442
|
+
if (!isRedirectStatus(response.status)) {
|
|
443
|
+
closeAgent = false;
|
|
444
|
+
return { response, ...(agent ? { agent } : {}) };
|
|
445
|
+
}
|
|
437
446
|
if (redirectCount >= redirectLimit) {
|
|
438
447
|
cancelResponseBody(response);
|
|
439
448
|
throw createFetchError({ kind: 'too-many-redirects' }, currentUrl);
|
|
@@ -452,7 +461,9 @@ class RedirectFollower {
|
|
|
452
461
|
};
|
|
453
462
|
}
|
|
454
463
|
finally {
|
|
455
|
-
|
|
464
|
+
if (closeAgent) {
|
|
465
|
+
await agent?.close();
|
|
466
|
+
}
|
|
456
467
|
}
|
|
457
468
|
}
|
|
458
469
|
getRedirectLocation(response, currentUrl) {
|
|
@@ -1051,8 +1062,10 @@ class HttpFetcher {
|
|
|
1051
1062
|
const signal = buildRequestSignal(timeoutMs, options?.signal);
|
|
1052
1063
|
const init = buildRequestInit(headers, signal);
|
|
1053
1064
|
const ctx = this.telemetry.start(normalizedUrl, 'GET');
|
|
1065
|
+
let agent;
|
|
1054
1066
|
try {
|
|
1055
|
-
const { response, url: finalUrl } = await this.redirectFollower.fetchWithRedirects(normalizedUrl, init, this.fetcherConfig.maxRedirects);
|
|
1067
|
+
const { response, url: finalUrl, agent: returnedAgent, } = await this.redirectFollower.fetchWithRedirects(normalizedUrl, init, this.fetcherConfig.maxRedirects);
|
|
1068
|
+
agent = returnedAgent;
|
|
1056
1069
|
ctx.url = this.telemetry.redact(finalUrl);
|
|
1057
1070
|
return await this.readPayload(response, finalUrl, ctx, mode, init.signal ?? undefined);
|
|
1058
1071
|
}
|
|
@@ -1062,6 +1075,9 @@ class HttpFetcher {
|
|
|
1062
1075
|
this.telemetry.recordError(ctx, mapped, mapped.statusCode);
|
|
1063
1076
|
throw mapped;
|
|
1064
1077
|
}
|
|
1078
|
+
finally {
|
|
1079
|
+
await agent?.close();
|
|
1080
|
+
}
|
|
1065
1081
|
}
|
|
1066
1082
|
async readPayload(response, finalUrl, ctx, mode, signal) {
|
|
1067
1083
|
try {
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
import { isIP } from 'node:net';
|
|
2
|
+
import { domainToASCII } from 'node:url';
|
|
3
|
+
export function buildIpv4(parts) {
|
|
4
|
+
return parts.join('.');
|
|
5
|
+
}
|
|
6
|
+
export function stripTrailingDots(value) {
|
|
7
|
+
let result = value;
|
|
8
|
+
while (result.endsWith('.'))
|
|
9
|
+
result = result.slice(0, -1);
|
|
10
|
+
return result;
|
|
11
|
+
}
|
|
12
|
+
export function normalizeHostname(value) {
|
|
13
|
+
const trimmed = value.trim();
|
|
14
|
+
if (!trimmed)
|
|
15
|
+
return null;
|
|
16
|
+
const lowered = trimmed.toLowerCase();
|
|
17
|
+
if (isIP(lowered))
|
|
18
|
+
return stripTrailingDots(lowered);
|
|
19
|
+
const ascii = domainToASCII(lowered);
|
|
20
|
+
return ascii ? stripTrailingDots(ascii) : null;
|
|
21
|
+
}
|
|
@@ -136,6 +136,7 @@ function resolveOwnerScopedExtra(extra) {
|
|
|
136
136
|
};
|
|
137
137
|
}
|
|
138
138
|
function getSdkCallToolHandler(server) {
|
|
139
|
+
// S-2: see tests/sdk-compat-guard.test.ts
|
|
139
140
|
const maybeHandlers = Reflect.get(server.server, '_requestHandlers');
|
|
140
141
|
if (!(maybeHandlers instanceof Map))
|
|
141
142
|
return null;
|
|
@@ -209,6 +210,8 @@ export function registerTaskHandlers(server) {
|
|
|
209
210
|
...(task.statusMessage ? { statusMessage: task.statusMessage } : {}),
|
|
210
211
|
});
|
|
211
212
|
}
|
|
213
|
+
// Forward-compat: input_required is a valid MCP task status but not currently
|
|
214
|
+
// produced by any tool in this server. Kept for future spec support.
|
|
212
215
|
if (task.status === 'input_required') {
|
|
213
216
|
throw new McpError(ErrorCode.InvalidRequest, 'Task requires additional input', { taskId: task.taskId, status: 'input_required' });
|
|
214
217
|
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
package/dist/lib/url.js
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import dns from 'node:dns';
|
|
2
2
|
import { BlockList, isIP, SocketAddress } from 'node:net';
|
|
3
|
-
import { domainToASCII } from 'node:url';
|
|
4
3
|
import { config, logDebug } from './core.js';
|
|
4
|
+
import { buildIpv4, normalizeHostname } from './net-utils.js';
|
|
5
5
|
import { createErrorWithCode, isError, isSystemError } from './utils.js';
|
|
6
6
|
const DNS_LOOKUP_TIMEOUT_MS = 5000;
|
|
7
7
|
const CNAME_LOOKUP_MAX_DEPTH = 5;
|
|
@@ -227,15 +227,6 @@ function normalizeBracketedIpv6(value) {
|
|
|
227
227
|
return null;
|
|
228
228
|
return normalizeHostname(ipv6);
|
|
229
229
|
}
|
|
230
|
-
function normalizeHostname(value) {
|
|
231
|
-
const trimmed = trimToNull(value)?.toLowerCase();
|
|
232
|
-
if (!trimmed)
|
|
233
|
-
return null;
|
|
234
|
-
if (isIP(trimmed))
|
|
235
|
-
return stripTrailingDots(trimmed);
|
|
236
|
-
const ascii = domainToASCII(trimmed);
|
|
237
|
-
return ascii ? stripTrailingDots(ascii) : null;
|
|
238
|
-
}
|
|
239
230
|
function parseHostWithUrl(value) {
|
|
240
231
|
const candidateUrl = `http://${value}`;
|
|
241
232
|
if (!URL.canParse(candidateUrl))
|
|
@@ -252,16 +243,6 @@ function trimToNull(value) {
|
|
|
252
243
|
const trimmed = value.trim();
|
|
253
244
|
return trimmed ? trimmed : null;
|
|
254
245
|
}
|
|
255
|
-
function stripTrailingDots(value) {
|
|
256
|
-
// Keep loop (rather than regex) to preserve exact behavior and avoid hidden allocations.
|
|
257
|
-
let result = value;
|
|
258
|
-
while (result.endsWith('.'))
|
|
259
|
-
result = result.slice(0, -1);
|
|
260
|
-
return result;
|
|
261
|
-
}
|
|
262
|
-
function buildIpv4(parts) {
|
|
263
|
-
return parts.join('.');
|
|
264
|
-
}
|
|
265
246
|
function buildIpv6(parts) {
|
|
266
247
|
return parts.map(String).join(':');
|
|
267
248
|
}
|
package/dist/prompts/index.d.ts
CHANGED
|
@@ -1,7 +1,3 @@
|
|
|
1
1
|
import type { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
|
|
2
|
-
|
|
3
|
-
src: string;
|
|
4
|
-
mimeType: string;
|
|
5
|
-
}
|
|
2
|
+
import type { IconInfo } from '../lib/types.js';
|
|
6
3
|
export declare function registerGetHelpPrompt(server: McpServer, instructions: string, iconInfo?: IconInfo): void;
|
|
7
|
-
export {};
|
|
@@ -1,8 +1,4 @@
|
|
|
1
1
|
import { type McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
|
|
2
|
-
|
|
3
|
-
src: string;
|
|
4
|
-
mimeType: string;
|
|
5
|
-
}
|
|
2
|
+
import type { IconInfo } from '../lib/types.js';
|
|
6
3
|
export declare function registerInstructionResource(server: McpServer, instructions: string, iconInfo?: IconInfo): void;
|
|
7
4
|
export declare function registerCacheResourceTemplate(server: McpServer, iconInfo?: IconInfo): void;
|
|
8
|
-
export {};
|
package/dist/resources/index.js
CHANGED
|
@@ -9,12 +9,12 @@ export function buildServerInstructions() {
|
|
|
9
9
|
|
|
10
10
|
<capabilities>
|
|
11
11
|
- Tools: \`${FETCH_URL_TOOL_NAME}\` (READ-ONLY).
|
|
12
|
-
- Resources: \`internal://
|
|
12
|
+
- Resources: \`internal://instructions\` (server usage guidance).
|
|
13
13
|
- Prompts: \`get-help\` (returns these instructions).
|
|
14
14
|
</capabilities>
|
|
15
15
|
|
|
16
16
|
<workflows>
|
|
17
|
-
1. Standard: Call \`${FETCH_URL_TOOL_NAME}\` -> Read \`markdown\`. If \`truncated: true\`,
|
|
17
|
+
1. Standard: Call \`${FETCH_URL_TOOL_NAME}\` -> Read \`markdown\`. If \`truncated: true\`, retry with \`forceRefresh: true\`.
|
|
18
18
|
2. Fresh: Set \`forceRefresh: true\` to bypass cache.
|
|
19
19
|
3. Full-Fidelity: Set \`skipNoiseRemoval: true\` to preserve nav/footers.
|
|
20
20
|
4. Async: Add \`task: { ttl: <ms> }\` to \`tools/call\` -> Poll \`tasks/get\` -> Call \`tasks/result\`.
|
|
@@ -24,7 +24,7 @@ export function buildServerInstructions() {
|
|
|
24
24
|
- Blocked: localhost, private IPs (10.x, 172.16-31.x, 192.168.x), metadata endpoints (169.254.169.254), .local/.internal.
|
|
25
25
|
- Limits: Max HTML ${maxHtmlSizeMb}MB. Max ${config.fetcher.maxRedirects} redirects.
|
|
26
26
|
- Cache: ${config.cache.maxKeys} entries, ${cacheSizeMb}MB, ${cacheTtlHours}h TTL.
|
|
27
|
-
- Cache scope: process-local and ephemeral.
|
|
27
|
+
- Cache scope: process-local and ephemeral.
|
|
28
28
|
- No JS: Client-side rendered pages may be incomplete.
|
|
29
29
|
- Binary: Not supported.
|
|
30
30
|
- Batch JSON-RPC: Array requests (\`[{...}]\`) are rejected with HTTP 400.
|
|
@@ -4,7 +4,6 @@ export declare const fetchUrlOutputSchema: z.ZodObject<{
|
|
|
4
4
|
inputUrl: z.ZodOptional<z.ZodString>;
|
|
5
5
|
resolvedUrl: z.ZodOptional<z.ZodString>;
|
|
6
6
|
finalUrl: z.ZodOptional<z.ZodString>;
|
|
7
|
-
cacheResourceUri: z.ZodOptional<z.ZodString>;
|
|
8
7
|
title: z.ZodOptional<z.ZodString>;
|
|
9
8
|
metadata: z.ZodOptional<z.ZodObject<{
|
|
10
9
|
title: z.ZodOptional<z.ZodString>;
|
package/dist/schemas/outputs.js
CHANGED
|
@@ -21,11 +21,6 @@ export const fetchUrlOutputSchema = z.strictObject({
|
|
|
21
21
|
.max(config.constants.maxUrlLength)
|
|
22
22
|
.optional()
|
|
23
23
|
.describe('Final URL after HTTP redirects.'),
|
|
24
|
-
cacheResourceUri: z
|
|
25
|
-
.string()
|
|
26
|
-
.max(config.constants.maxUrlLength)
|
|
27
|
-
.optional()
|
|
28
|
-
.describe('URI for resources/read to get full markdown.'),
|
|
29
24
|
title: z.string().max(512).optional().describe('Page title.'),
|
|
30
25
|
metadata: z
|
|
31
26
|
.strictObject({
|
package/dist/server.js
CHANGED
|
@@ -8,10 +8,13 @@ import { logError, logInfo, setLogLevel, setMcpServer } from './lib/core.js';
|
|
|
8
8
|
import { abortAllTaskExecutions, registerTaskHandlers, } from './lib/mcp-tools.js';
|
|
9
9
|
import { toError } from './lib/utils.js';
|
|
10
10
|
import { registerGetHelpPrompt } from './prompts/index.js';
|
|
11
|
-
import {
|
|
11
|
+
import { registerInstructionResource } from './resources/index.js';
|
|
12
12
|
import { buildServerInstructions } from './resources/instructions.js';
|
|
13
13
|
import { registerAllTools } from './tools/index.js';
|
|
14
14
|
import { shutdownTransformWorkerPool } from './transform/transform.js';
|
|
15
|
+
/* -------------------------------------------------------------------------------------------------
|
|
16
|
+
* Icons + server info
|
|
17
|
+
* ------------------------------------------------------------------------------------------------- */
|
|
15
18
|
async function getLocalIconInfo() {
|
|
16
19
|
const name = 'logo.svg';
|
|
17
20
|
const mime = 'image/svg+xml';
|
|
@@ -31,13 +34,9 @@ const serverInstructions = buildServerInstructions();
|
|
|
31
34
|
function createServerCapabilities() {
|
|
32
35
|
return {
|
|
33
36
|
logging: {},
|
|
34
|
-
resources: {
|
|
35
|
-
subscribe: true,
|
|
36
|
-
listChanged: true,
|
|
37
|
-
},
|
|
37
|
+
resources: { subscribe: true, listChanged: true },
|
|
38
38
|
tools: {},
|
|
39
39
|
prompts: {},
|
|
40
|
-
completions: {},
|
|
41
40
|
tasks: {
|
|
42
41
|
list: {},
|
|
43
42
|
cancel: {},
|
|
@@ -85,7 +84,6 @@ async function createMcpServerWithOptions(options) {
|
|
|
85
84
|
registerAllTools(server);
|
|
86
85
|
registerGetHelpPrompt(server, serverInstructions, localIcon);
|
|
87
86
|
registerInstructionResource(server, serverInstructions, localIcon);
|
|
88
|
-
registerCacheResourceTemplate(server, localIcon);
|
|
89
87
|
// NOTE: Internally patches server.close and server.server.onclose for cleanup
|
|
90
88
|
// callbacks, and intercepts tools/call via Reflect.get on private SDK state.
|
|
91
89
|
// See src/lib/task-handlers.ts for risk documentation (S-2, S-3).
|
package/dist/tasks/execution.js
CHANGED
|
@@ -6,6 +6,7 @@ import { isObject } from '../lib/utils.js';
|
|
|
6
6
|
import { taskManager, } from './manager.js';
|
|
7
7
|
import { compact, tryReadToolStructuredError, } from './owner.js';
|
|
8
8
|
import { getTaskCapableTool, hasTaskCapableTool, } from './tool-registry.js';
|
|
9
|
+
const TASK_NOT_FOUND_ERROR_CODE = RESOURCE_NOT_FOUND_ERROR_CODE;
|
|
9
10
|
/* -------------------------------------------------------------------------------------------------
|
|
10
11
|
* Abort-controller management for in-flight task executions
|
|
11
12
|
* ------------------------------------------------------------------------------------------------- */
|
|
@@ -113,7 +114,7 @@ function buildTaskStatusNotificationParams(task) {
|
|
|
113
114
|
* Validation helpers
|
|
114
115
|
* ------------------------------------------------------------------------------------------------- */
|
|
115
116
|
export function throwTaskNotFound() {
|
|
116
|
-
throw new McpError(
|
|
117
|
+
throw new McpError(TASK_NOT_FOUND_ERROR_CODE, 'Task not found');
|
|
117
118
|
}
|
|
118
119
|
function resolveTaskCapableTool(name) {
|
|
119
120
|
const descriptor = getTaskCapableTool(name);
|
package/dist/tasks/manager.js
CHANGED
|
@@ -69,6 +69,8 @@ class TaskManager {
|
|
|
69
69
|
applyTaskUpdate(task, updates) {
|
|
70
70
|
Object.assign(task, updates);
|
|
71
71
|
task.lastUpdatedAt = new Date().toISOString();
|
|
72
|
+
// Slide TTL window on every activity so long-running tasks don't expire mid-flight.
|
|
73
|
+
task._createdAtMs = Date.now();
|
|
72
74
|
}
|
|
73
75
|
cancelActiveTask(task, statusMessage) {
|
|
74
76
|
this.applyTaskUpdate(task, {
|
package/dist/tools/fetch-url.js
CHANGED
|
@@ -1,12 +1,10 @@
|
|
|
1
1
|
import { randomUUID } from 'node:crypto';
|
|
2
2
|
import { ErrorCode, McpError } from '@modelcontextprotocol/sdk/types.js';
|
|
3
3
|
import { z } from 'zod';
|
|
4
|
-
import * as cache from '../lib/core.js';
|
|
5
4
|
import { config } from '../lib/core.js';
|
|
6
5
|
import { getRequestId, logDebug, logError, logWarn, runWithRequestContext, } from '../lib/core.js';
|
|
7
|
-
import { generateSafeFilename } from '../lib/http.js';
|
|
8
6
|
import { handleToolError } from '../lib/mcp-tools.js';
|
|
9
|
-
import { appendTruncationMarker, markdownTransform, parseCachedMarkdownResult, performSharedFetch, readNestedRecord,
|
|
7
|
+
import { appendTruncationMarker, markdownTransform, parseCachedMarkdownResult, performSharedFetch, readNestedRecord, serializeMarkdownResult, TRUNCATION_MARKER, withSignal, } from '../lib/mcp-tools.js';
|
|
10
8
|
import { createProgressReporter, } from '../lib/mcp-tools.js';
|
|
11
9
|
import { isAbortError, isObject, toError } from '../lib/utils.js';
|
|
12
10
|
import { fetchUrlInputSchema } from '../schemas/inputs.js';
|
|
@@ -19,7 +17,7 @@ const FETCH_URL_TOOL_DESCRIPTION = `
|
|
|
19
17
|
<constraints>
|
|
20
18
|
- READ-ONLY. No JavaScript execution.
|
|
21
19
|
- GitHub/GitLab/Bitbucket URLs auto-transform to raw endpoints (check resolvedUrl).
|
|
22
|
-
- If truncated=true,
|
|
20
|
+
- If truncated=true, full content is available in the next fetch with forceRefresh.
|
|
23
21
|
- For large pages/timeouts, use task mode (task: {}).
|
|
24
22
|
- If error queue_full, retry with task mode.
|
|
25
23
|
</constraints>
|
|
@@ -38,47 +36,6 @@ function buildTextBlock(structuredContent) {
|
|
|
38
36
|
text: JSON.stringify(structuredContent),
|
|
39
37
|
};
|
|
40
38
|
}
|
|
41
|
-
function buildEmbeddedResource(content, url, title) {
|
|
42
|
-
if (!content)
|
|
43
|
-
return null;
|
|
44
|
-
const filename = generateSafeFilename(url, title, undefined, '.md');
|
|
45
|
-
const uri = `internal://inline/${encodeURIComponent(filename)}`;
|
|
46
|
-
const resource = {
|
|
47
|
-
uri,
|
|
48
|
-
mimeType: 'text/markdown',
|
|
49
|
-
text: content,
|
|
50
|
-
};
|
|
51
|
-
return {
|
|
52
|
-
type: 'resource',
|
|
53
|
-
resource,
|
|
54
|
-
};
|
|
55
|
-
}
|
|
56
|
-
function buildCacheResourceLink(cacheResourceUri, contentSize, fetchedAt) {
|
|
57
|
-
return {
|
|
58
|
-
type: 'resource_link',
|
|
59
|
-
uri: cacheResourceUri,
|
|
60
|
-
name: 'cached-markdown',
|
|
61
|
-
title: 'Cached Fetch Output',
|
|
62
|
-
description: 'Read full markdown via resources/read.',
|
|
63
|
-
mimeType: 'text/markdown',
|
|
64
|
-
...(contentSize > 0 ? { size: contentSize } : {}),
|
|
65
|
-
annotations: {
|
|
66
|
-
audience: ['assistant'],
|
|
67
|
-
priority: 0.8,
|
|
68
|
-
lastModified: fetchedAt,
|
|
69
|
-
},
|
|
70
|
-
};
|
|
71
|
-
}
|
|
72
|
-
function buildToolContentBlocks(structuredContent, resourceLink, embeddedResource) {
|
|
73
|
-
const blocks = [buildTextBlock(structuredContent)];
|
|
74
|
-
appendIfPresent(blocks, resourceLink);
|
|
75
|
-
appendIfPresent(blocks, embeddedResource);
|
|
76
|
-
return blocks;
|
|
77
|
-
}
|
|
78
|
-
function appendIfPresent(items, value) {
|
|
79
|
-
if (value !== null && value !== undefined)
|
|
80
|
-
items.push(value);
|
|
81
|
-
}
|
|
82
39
|
/* -------------------------------------------------------------------------------------------------
|
|
83
40
|
* Tool abort signal
|
|
84
41
|
* ------------------------------------------------------------------------------------------------- */
|
|
@@ -116,15 +73,15 @@ function truncateMetadata(metadata) {
|
|
|
116
73
|
return result;
|
|
117
74
|
}
|
|
118
75
|
function buildStructuredContent(pipeline, inlineResult, inputUrl) {
|
|
119
|
-
const cacheResourceUri = resolveCacheResourceUri(pipeline.cacheKey);
|
|
120
76
|
const truncated = inlineResult.truncated ?? pipeline.data.truncated;
|
|
121
|
-
const
|
|
77
|
+
const rawMarkdown = applyTruncationMarker(inlineResult.content, pipeline.data.truncated);
|
|
78
|
+
const maxChars = config.constants.maxInlineContentChars;
|
|
79
|
+
const markdown = maxChars > 0 ? truncateStr(rawMarkdown, maxChars) : rawMarkdown;
|
|
122
80
|
const { metadata } = pipeline.data;
|
|
123
81
|
return {
|
|
124
82
|
url: pipeline.originalUrl ?? pipeline.url,
|
|
125
83
|
resolvedUrl: pipeline.url,
|
|
126
84
|
...(pipeline.finalUrl ? { finalUrl: pipeline.finalUrl } : {}),
|
|
127
|
-
...(cacheResourceUri ? { cacheResourceUri } : {}),
|
|
128
85
|
inputUrl,
|
|
129
86
|
title: truncateStr(pipeline.data.title, 512),
|
|
130
87
|
...(metadata ? { metadata: truncateMetadata(metadata) } : {}),
|
|
@@ -140,34 +97,12 @@ function applyTruncationMarker(content, truncated) {
|
|
|
140
97
|
return content;
|
|
141
98
|
return appendTruncationMarker(content, TRUNCATION_MARKER);
|
|
142
99
|
}
|
|
143
|
-
function
|
|
144
|
-
|
|
145
|
-
return undefined;
|
|
146
|
-
if (!cache.isEnabled())
|
|
147
|
-
return undefined;
|
|
148
|
-
if (!cache.get(cacheKey))
|
|
149
|
-
return undefined;
|
|
150
|
-
const parsed = cache.parseCacheKey(cacheKey);
|
|
151
|
-
if (!parsed)
|
|
152
|
-
return undefined;
|
|
153
|
-
return `internal://cache/${encodeURIComponent(parsed.namespace)}/${encodeURIComponent(parsed.urlHash)}`;
|
|
154
|
-
}
|
|
155
|
-
function buildFetchUrlContentBlocks(structuredContent, pipeline, inlineResult) {
|
|
156
|
-
const cacheResourceUri = readString(structuredContent, 'cacheResourceUri');
|
|
157
|
-
const contentToEmbed = config.runtime.httpMode
|
|
158
|
-
? inlineResult.content
|
|
159
|
-
: pipeline.data.content;
|
|
160
|
-
const resourceLink = cacheResourceUri
|
|
161
|
-
? buildCacheResourceLink(cacheResourceUri, inlineResult.contentSize, pipeline.fetchedAt)
|
|
162
|
-
: null;
|
|
163
|
-
const embedded = contentToEmbed && pipeline.url
|
|
164
|
-
? buildEmbeddedResource(contentToEmbed, pipeline.url, pipeline.data.title)
|
|
165
|
-
: null;
|
|
166
|
-
return buildToolContentBlocks(structuredContent, resourceLink, embedded);
|
|
100
|
+
function buildFetchUrlContentBlocks(structuredContent) {
|
|
101
|
+
return [buildTextBlock(structuredContent)];
|
|
167
102
|
}
|
|
168
103
|
function buildResponse(pipeline, inlineResult, inputUrl) {
|
|
169
104
|
const structuredContent = buildStructuredContent(pipeline, inlineResult, inputUrl);
|
|
170
|
-
const content = buildFetchUrlContentBlocks(structuredContent
|
|
105
|
+
const content = buildFetchUrlContentBlocks(structuredContent);
|
|
171
106
|
const validation = fetchUrlOutputSchema.safeParse(structuredContent);
|
|
172
107
|
if (!validation.success) {
|
|
173
108
|
logWarn('Tool output schema validation failed', {
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@j0hanz/fetch-url-mcp",
|
|
3
|
-
"version": "1.
|
|
3
|
+
"version": "1.6.0",
|
|
4
4
|
"mcpName": "io.github.j0hanz/fetch-url-mcp",
|
|
5
5
|
"description": "Intelligent web content fetcher MCP server that converts HTML to clean, AI-readable Markdown",
|
|
6
6
|
"type": "module",
|
|
@@ -81,7 +81,7 @@
|
|
|
81
81
|
"eslint": "^10.0.2",
|
|
82
82
|
"eslint-config-prettier": "^10.1.8",
|
|
83
83
|
"eslint-plugin-de-morgan": "^2.1.1",
|
|
84
|
-
"eslint-plugin-depend": "^1.
|
|
84
|
+
"eslint-plugin-depend": "^1.5.0",
|
|
85
85
|
"eslint-plugin-unused-imports": "^4.4.1",
|
|
86
86
|
"knip": "^5.85.0",
|
|
87
87
|
"prettier": "^3.8.1",
|