@j0hanz/fetch-url-mcp 1.5.1 → 1.6.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/http/auth.js CHANGED
@@ -1,7 +1,7 @@
1
1
  import { Buffer } from 'node:buffer';
2
2
  import { randomBytes } from 'node:crypto';
3
3
  import { InvalidTokenError, ServerError, } from '@modelcontextprotocol/sdk/server/auth/errors.js';
4
- import { config } from '../lib/core.js';
4
+ import { config, logWarn } from '../lib/core.js';
5
5
  import { normalizeHost } from '../lib/url.js';
6
6
  import { hmacSha256Hex, timingSafeEqualUtf8 } from '../lib/utils.js';
7
7
  import { isObject } from '../lib/utils.js';
@@ -186,6 +186,9 @@ export function ensureMcpProtocolVersion(req, res, options) {
186
186
  // Permissive backward-compat fallback: clients predating MCP 2025-03-26 do not
187
187
  // send MCP-Protocol-Version. Accepting requests without the header keeps older
188
188
  // integrations working. Pass requireHeader: true to enforce strict version checking.
189
+ logWarn('MCP-Protocol-Version header missing; defaulting to permissive fallback', {
190
+ remoteAddress: req.socket.remoteAddress,
191
+ });
189
192
  return true;
190
193
  }
191
194
  sendError(res, -32600, 'Missing MCP-Protocol-Version header');
@@ -139,6 +139,11 @@ class McpSessionGateway {
139
139
  sendError(ctx.res, -32600, 'Session not found', 404);
140
140
  return;
141
141
  }
142
+ const fingerprint = buildAuthFingerprint(ctx.auth);
143
+ if (!fingerprint || session.authFingerprint !== fingerprint) {
144
+ sendError(ctx.res, -32600, 'Session not found', 404);
145
+ return;
146
+ }
142
147
  if (!ensureMcpProtocolVersion(ctx.req, ctx.res, {
143
148
  requireHeader: true,
144
149
  expectedVersion: session.negotiatedProtocolVersion,
@@ -147,7 +152,9 @@ class McpSessionGateway {
147
152
  }
148
153
  const acceptHeader = getHeaderValue(ctx.req, 'accept');
149
154
  if (!acceptsEventStream(acceptHeader)) {
150
- sendJson(ctx.res, 405, { error: 'Method Not Allowed' });
155
+ sendJson(ctx.res, 406, {
156
+ error: 'Not Acceptable: expected text/event-stream',
157
+ });
151
158
  return;
152
159
  }
153
160
  this.store.touch(sessionId);
@@ -164,6 +171,11 @@ class McpSessionGateway {
164
171
  sendError(ctx.res, -32600, 'Session not found', 404);
165
172
  return;
166
173
  }
174
+ const fingerprint = buildAuthFingerprint(ctx.auth);
175
+ if (!fingerprint || session.authFingerprint !== fingerprint) {
176
+ sendError(ctx.res, -32600, 'Session not found', 404);
177
+ return;
178
+ }
167
179
  if (!ensureMcpProtocolVersion(ctx.req, ctx.res, {
168
180
  requireHeader: true,
169
181
  expectedVersion: session.negotiatedProtocolVersion,
@@ -172,7 +184,7 @@ class McpSessionGateway {
172
184
  }
173
185
  await session.transport.close();
174
186
  this.cleanupSessionRecord(sessionId, 'session-delete');
175
- sendText(ctx.res, 200, 'Session closed');
187
+ sendJson(ctx.res, 200, { status: 'closed' });
176
188
  }
177
189
  async getOrCreateTransport(ctx, requestId) {
178
190
  const sessionId = getMcpSessionId(ctx.req);
@@ -19,7 +19,7 @@ const HEADER_NOISE_PATTERN = /\b(site-header|masthead|topbar|navbar|nav(?:bar)?|
19
19
  const FIXED_OR_HIGH_Z_PATTERN = /\b(?:fixed|sticky|z-(?:4\d|50)|isolate)\b/;
20
20
  const SKIP_URL_PREFIXES = [
21
21
  '#',
22
- 'java' + 'script:',
22
+ 'javascript:',
23
23
  'mailto:',
24
24
  'tel:',
25
25
  'data:',
@@ -188,6 +188,7 @@ export declare function keys(): readonly string[];
188
188
  export declare function getEntryMeta(cacheKey: string): {
189
189
  url: string;
190
190
  title?: string;
191
+ fetchedAt?: string;
191
192
  } | undefined;
192
193
  export declare function isEnabled(): boolean;
193
194
  type LogMetadata = Record<string, unknown>;
package/dist/lib/core.js CHANGED
@@ -4,11 +4,11 @@ import { accessSync, constants as fsConstants, readFileSync } from 'node:fs';
4
4
  import { findPackageJSON } from 'node:module';
5
5
  import { isIP } from 'node:net';
6
6
  import process from 'node:process';
7
- import { domainToASCII } from 'node:url';
8
7
  import { inspect, stripVTControlCharacters } from 'node:util';
9
8
  import {} from '@modelcontextprotocol/sdk/server/mcp.js';
10
9
  import {} from '@modelcontextprotocol/sdk/server/streamableHttp.js';
11
10
  import { z } from 'zod';
11
+ import { buildIpv4, normalizeHostname, stripTrailingDots, } from './net-utils.js';
12
12
  import { getErrorMessage, isAbortError, sha256Hex, stableStringify as stableJsonStringify, startAbortableIntervalLoop, } from './utils.js';
13
13
  export const serverVersion = readServerVersion(import.meta.url);
14
14
  const LOG_LEVELS = ['debug', 'info', 'warn', 'error'];
@@ -57,31 +57,11 @@ function loadEnvFileIfAvailable() {
57
57
  }
58
58
  loadEnvFileIfAvailable();
59
59
  const { env } = process;
60
- function buildIpv4(parts) {
61
- return parts.join('.');
62
- }
63
- function stripTrailingDots(value) {
64
- let result = value;
65
- while (result.endsWith('.'))
66
- result = result.slice(0, -1);
67
- return result;
68
- }
69
60
  function formatHostForUrl(hostname) {
70
61
  if (hostname.includes(':') && !hostname.startsWith('['))
71
62
  return `[${hostname}]`;
72
63
  return hostname;
73
64
  }
74
- function normalizeHostname(value) {
75
- const trimmed = value.trim();
76
- if (!trimmed)
77
- return null;
78
- const lowered = trimmed.toLowerCase();
79
- const ipType = isIP(lowered);
80
- if (ipType)
81
- return stripTrailingDots(lowered);
82
- const ascii = domainToASCII(lowered);
83
- return ascii ? stripTrailingDots(ascii) : null;
84
- }
85
65
  function normalizeHostValue(value) {
86
66
  const raw = value.trim();
87
67
  if (!raw)
@@ -706,9 +686,11 @@ export function getEntryMeta(cacheKey) {
706
686
  const entry = store.peek(cacheKey);
707
687
  if (!entry)
708
688
  return undefined;
709
- return entry.title !== undefined
710
- ? { url: entry.url, title: entry.title }
711
- : { url: entry.url };
689
+ return {
690
+ url: entry.url,
691
+ ...(entry.title !== undefined ? { title: entry.title } : {}),
692
+ ...(entry.fetchedAt ? { fetchedAt: entry.fetchedAt } : {}),
693
+ };
712
694
  }
713
695
  export function isEnabled() {
714
696
  return store.isEnabled();
@@ -1,4 +1,5 @@
1
1
  import { type ServerResponse } from 'node:http';
2
+ import { Agent } from 'undici';
2
3
  import { type TransformResult } from './url.js';
3
4
  export declare function generateSafeFilename(url: string, title?: string, hashFallback?: string, extension?: string): string;
4
5
  export declare function handleDownload(res: ServerResponse, namespace: string, hash: string): void;
@@ -27,6 +28,7 @@ export declare function recordFetchError(context: FetchTelemetryContext, error:
27
28
  export declare function fetchWithRedirects(url: string, init: RequestInit, maxRedirects: number): Promise<{
28
29
  response: Response;
29
30
  url: string;
31
+ agent?: Agent;
30
32
  }>;
31
33
  export declare function readResponseText(response: Response, url: string, maxBytes: number, signal?: AbortSignal, encoding?: string): Promise<{
32
34
  text: string;
package/dist/lib/http.js CHANGED
@@ -388,15 +388,20 @@ class RedirectFollower {
388
388
  throw createFetchError({ kind: 'too-many-redirects' }, currentUrl);
389
389
  }
390
390
  visited.add(currentUrl);
391
- const { response, nextUrl } = await this.withRedirectErrorContext(currentUrl, async () => {
391
+ const { response, nextUrl, agent: returnedAgent, } = await this.withRedirectErrorContext(currentUrl, async () => {
392
392
  let ipAddress;
393
393
  if (this.preflight) {
394
394
  ipAddress = await this.preflight(currentUrl, init.signal ?? undefined);
395
395
  }
396
396
  return this.performFetchCycle(currentUrl, init, redirectLimit, redirectCount, ipAddress);
397
397
  });
398
- if (!nextUrl)
399
- return { response, url: currentUrl };
398
+ if (!nextUrl) {
399
+ return {
400
+ response,
401
+ url: currentUrl,
402
+ ...(returnedAgent ? { agent: returnedAgent } : {}),
403
+ };
404
+ }
400
405
  currentUrl = nextUrl;
401
406
  }
402
407
  throw createFetchError({ kind: 'too-many-redirects' }, currentUrl);
@@ -430,10 +435,14 @@ class RedirectFollower {
430
435
  });
431
436
  fetchInit.dispatcher = agent;
432
437
  }
438
+ let closeAgent = true;
433
439
  try {
434
440
  const response = await this.fetchFn(currentUrl, fetchInit);
435
- if (!isRedirectStatus(response.status))
436
- return { response };
441
+ // Only follow redirects if the status code indicates a redirect and there's a Location header.
442
+ if (!isRedirectStatus(response.status)) {
443
+ closeAgent = false;
444
+ return { response, ...(agent ? { agent } : {}) };
445
+ }
437
446
  if (redirectCount >= redirectLimit) {
438
447
  cancelResponseBody(response);
439
448
  throw createFetchError({ kind: 'too-many-redirects' }, currentUrl);
@@ -452,7 +461,9 @@ class RedirectFollower {
452
461
  };
453
462
  }
454
463
  finally {
455
- await agent?.close();
464
+ if (closeAgent) {
465
+ await agent?.close();
466
+ }
456
467
  }
457
468
  }
458
469
  getRedirectLocation(response, currentUrl) {
@@ -1051,8 +1062,10 @@ class HttpFetcher {
1051
1062
  const signal = buildRequestSignal(timeoutMs, options?.signal);
1052
1063
  const init = buildRequestInit(headers, signal);
1053
1064
  const ctx = this.telemetry.start(normalizedUrl, 'GET');
1065
+ let agent;
1054
1066
  try {
1055
- const { response, url: finalUrl } = await this.redirectFollower.fetchWithRedirects(normalizedUrl, init, this.fetcherConfig.maxRedirects);
1067
+ const { response, url: finalUrl, agent: returnedAgent, } = await this.redirectFollower.fetchWithRedirects(normalizedUrl, init, this.fetcherConfig.maxRedirects);
1068
+ agent = returnedAgent;
1056
1069
  ctx.url = this.telemetry.redact(finalUrl);
1057
1070
  return await this.readPayload(response, finalUrl, ctx, mode, init.signal ?? undefined);
1058
1071
  }
@@ -1062,6 +1075,9 @@ class HttpFetcher {
1062
1075
  this.telemetry.recordError(ctx, mapped, mapped.statusCode);
1063
1076
  throw mapped;
1064
1077
  }
1078
+ finally {
1079
+ await agent?.close();
1080
+ }
1065
1081
  }
1066
1082
  async readPayload(response, finalUrl, ctx, mode, signal) {
1067
1083
  try {
@@ -0,0 +1,3 @@
1
+ export declare function buildIpv4(parts: readonly [number, number, number, number]): string;
2
+ export declare function stripTrailingDots(value: string): string;
3
+ export declare function normalizeHostname(value: string): string | null;
@@ -0,0 +1,21 @@
1
+ import { isIP } from 'node:net';
2
+ import { domainToASCII } from 'node:url';
3
+ export function buildIpv4(parts) {
4
+ return parts.join('.');
5
+ }
6
+ export function stripTrailingDots(value) {
7
+ let result = value;
8
+ while (result.endsWith('.'))
9
+ result = result.slice(0, -1);
10
+ return result;
11
+ }
12
+ export function normalizeHostname(value) {
13
+ const trimmed = value.trim();
14
+ if (!trimmed)
15
+ return null;
16
+ const lowered = trimmed.toLowerCase();
17
+ if (isIP(lowered))
18
+ return stripTrailingDots(lowered);
19
+ const ascii = domainToASCII(lowered);
20
+ return ascii ? stripTrailingDots(ascii) : null;
21
+ }
@@ -136,6 +136,7 @@ function resolveOwnerScopedExtra(extra) {
136
136
  };
137
137
  }
138
138
  function getSdkCallToolHandler(server) {
139
+ // S-2: see tests/sdk-compat-guard.test.ts
139
140
  const maybeHandlers = Reflect.get(server.server, '_requestHandlers');
140
141
  if (!(maybeHandlers instanceof Map))
141
142
  return null;
@@ -209,6 +210,8 @@ export function registerTaskHandlers(server) {
209
210
  ...(task.statusMessage ? { statusMessage: task.statusMessage } : {}),
210
211
  });
211
212
  }
213
+ // Forward-compat: input_required is a valid MCP task status but not currently
214
+ // produced by any tool in this server. Kept for future spec support.
212
215
  if (task.status === 'input_required') {
213
216
  throw new McpError(ErrorCode.InvalidRequest, 'Task requires additional input', { taskId: task.taskId, status: 'input_required' });
214
217
  }
@@ -0,0 +1,4 @@
1
+ export interface IconInfo {
2
+ src: string;
3
+ mimeType: string;
4
+ }
@@ -0,0 +1 @@
1
+ export {};
package/dist/lib/url.js CHANGED
@@ -1,7 +1,7 @@
1
1
  import dns from 'node:dns';
2
2
  import { BlockList, isIP, SocketAddress } from 'node:net';
3
- import { domainToASCII } from 'node:url';
4
3
  import { config, logDebug } from './core.js';
4
+ import { buildIpv4, normalizeHostname } from './net-utils.js';
5
5
  import { createErrorWithCode, isError, isSystemError } from './utils.js';
6
6
  const DNS_LOOKUP_TIMEOUT_MS = 5000;
7
7
  const CNAME_LOOKUP_MAX_DEPTH = 5;
@@ -227,15 +227,6 @@ function normalizeBracketedIpv6(value) {
227
227
  return null;
228
228
  return normalizeHostname(ipv6);
229
229
  }
230
- function normalizeHostname(value) {
231
- const trimmed = trimToNull(value)?.toLowerCase();
232
- if (!trimmed)
233
- return null;
234
- if (isIP(trimmed))
235
- return stripTrailingDots(trimmed);
236
- const ascii = domainToASCII(trimmed);
237
- return ascii ? stripTrailingDots(ascii) : null;
238
- }
239
230
  function parseHostWithUrl(value) {
240
231
  const candidateUrl = `http://${value}`;
241
232
  if (!URL.canParse(candidateUrl))
@@ -252,16 +243,6 @@ function trimToNull(value) {
252
243
  const trimmed = value.trim();
253
244
  return trimmed ? trimmed : null;
254
245
  }
255
- function stripTrailingDots(value) {
256
- // Keep loop (rather than regex) to preserve exact behavior and avoid hidden allocations.
257
- let result = value;
258
- while (result.endsWith('.'))
259
- result = result.slice(0, -1);
260
- return result;
261
- }
262
- function buildIpv4(parts) {
263
- return parts.join('.');
264
- }
265
246
  function buildIpv6(parts) {
266
247
  return parts.map(String).join(':');
267
248
  }
@@ -1,7 +1,3 @@
1
1
  import type { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
2
- interface IconInfo {
3
- src: string;
4
- mimeType: string;
5
- }
2
+ import type { IconInfo } from '../lib/types.js';
6
3
  export declare function registerGetHelpPrompt(server: McpServer, instructions: string, iconInfo?: IconInfo): void;
7
- export {};
@@ -1,8 +1,4 @@
1
1
  import { type McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
2
- interface IconInfo {
3
- src: string;
4
- mimeType: string;
5
- }
2
+ import type { IconInfo } from '../lib/types.js';
6
3
  export declare function registerInstructionResource(server: McpServer, instructions: string, iconInfo?: IconInfo): void;
7
4
  export declare function registerCacheResourceTemplate(server: McpServer, iconInfo?: IconInfo): void;
8
- export {};
@@ -177,6 +177,7 @@ function listCacheResources() {
177
177
  annotations: {
178
178
  audience: ['assistant'],
179
179
  priority: 0.6,
180
+ ...(meta.fetchedAt ? { lastModified: meta.fetchedAt } : {}),
180
181
  },
181
182
  };
182
183
  })
@@ -9,12 +9,12 @@ export function buildServerInstructions() {
9
9
 
10
10
  <capabilities>
11
11
  - Tools: \`${FETCH_URL_TOOL_NAME}\` (READ-ONLY).
12
- - Resources: \`internal://cache/{namespace}/{hash}\` (ephemeral cached Markdown).
12
+ - Resources: \`internal://instructions\` (server usage guidance).
13
13
  - Prompts: \`get-help\` (returns these instructions).
14
14
  </capabilities>
15
15
 
16
16
  <workflows>
17
- 1. Standard: Call \`${FETCH_URL_TOOL_NAME}\` -> Read \`markdown\`. If \`truncated: true\`, use \`cacheResourceUri\` with \`resources/read\` for full content.
17
+ 1. Standard: Call \`${FETCH_URL_TOOL_NAME}\` -> Read \`markdown\`. If \`truncated: true\`, retry with \`forceRefresh: true\`.
18
18
  2. Fresh: Set \`forceRefresh: true\` to bypass cache.
19
19
  3. Full-Fidelity: Set \`skipNoiseRemoval: true\` to preserve nav/footers.
20
20
  4. Async: Add \`task: { ttl: <ms> }\` to \`tools/call\` -> Poll \`tasks/get\` -> Call \`tasks/result\`.
@@ -24,7 +24,7 @@ export function buildServerInstructions() {
24
24
  - Blocked: localhost, private IPs (10.x, 172.16-31.x, 192.168.x), metadata endpoints (169.254.169.254), .local/.internal.
25
25
  - Limits: Max HTML ${maxHtmlSizeMb}MB. Max ${config.fetcher.maxRedirects} redirects.
26
26
  - Cache: ${config.cache.maxKeys} entries, ${cacheSizeMb}MB, ${cacheTtlHours}h TTL.
27
- - Cache scope: process-local and ephemeral. Cache URIs are invalid across server restarts or separate CLI invocations (-32002).
27
+ - Cache scope: process-local and ephemeral.
28
28
  - No JS: Client-side rendered pages may be incomplete.
29
29
  - Binary: Not supported.
30
30
  - Batch JSON-RPC: Array requests (\`[{...}]\`) are rejected with HTTP 400.
@@ -4,7 +4,6 @@ export declare const fetchUrlOutputSchema: z.ZodObject<{
4
4
  inputUrl: z.ZodOptional<z.ZodString>;
5
5
  resolvedUrl: z.ZodOptional<z.ZodString>;
6
6
  finalUrl: z.ZodOptional<z.ZodString>;
7
- cacheResourceUri: z.ZodOptional<z.ZodString>;
8
7
  title: z.ZodOptional<z.ZodString>;
9
8
  metadata: z.ZodOptional<z.ZodObject<{
10
9
  title: z.ZodOptional<z.ZodString>;
@@ -21,11 +21,6 @@ export const fetchUrlOutputSchema = z.strictObject({
21
21
  .max(config.constants.maxUrlLength)
22
22
  .optional()
23
23
  .describe('Final URL after HTTP redirects.'),
24
- cacheResourceUri: z
25
- .string()
26
- .max(config.constants.maxUrlLength)
27
- .optional()
28
- .describe('URI for resources/read to get full markdown.'),
29
24
  title: z.string().max(512).optional().describe('Page title.'),
30
25
  metadata: z
31
26
  .strictObject({
package/dist/server.js CHANGED
@@ -8,10 +8,13 @@ import { logError, logInfo, setLogLevel, setMcpServer } from './lib/core.js';
8
8
  import { abortAllTaskExecutions, registerTaskHandlers, } from './lib/mcp-tools.js';
9
9
  import { toError } from './lib/utils.js';
10
10
  import { registerGetHelpPrompt } from './prompts/index.js';
11
- import { registerCacheResourceTemplate, registerInstructionResource, } from './resources/index.js';
11
+ import { registerInstructionResource } from './resources/index.js';
12
12
  import { buildServerInstructions } from './resources/instructions.js';
13
13
  import { registerAllTools } from './tools/index.js';
14
14
  import { shutdownTransformWorkerPool } from './transform/transform.js';
15
+ /* -------------------------------------------------------------------------------------------------
16
+ * Icons + server info
17
+ * ------------------------------------------------------------------------------------------------- */
15
18
  async function getLocalIconInfo() {
16
19
  const name = 'logo.svg';
17
20
  const mime = 'image/svg+xml';
@@ -31,13 +34,9 @@ const serverInstructions = buildServerInstructions();
31
34
  function createServerCapabilities() {
32
35
  return {
33
36
  logging: {},
34
- resources: {
35
- subscribe: true,
36
- listChanged: true,
37
- },
37
+ resources: { subscribe: true, listChanged: true },
38
38
  tools: {},
39
39
  prompts: {},
40
- completions: {},
41
40
  tasks: {
42
41
  list: {},
43
42
  cancel: {},
@@ -85,7 +84,6 @@ async function createMcpServerWithOptions(options) {
85
84
  registerAllTools(server);
86
85
  registerGetHelpPrompt(server, serverInstructions, localIcon);
87
86
  registerInstructionResource(server, serverInstructions, localIcon);
88
- registerCacheResourceTemplate(server, localIcon);
89
87
  // NOTE: Internally patches server.close and server.server.onclose for cleanup
90
88
  // callbacks, and intercepts tools/call via Reflect.get on private SDK state.
91
89
  // See src/lib/task-handlers.ts for risk documentation (S-2, S-3).
@@ -6,6 +6,7 @@ import { isObject } from '../lib/utils.js';
6
6
  import { taskManager, } from './manager.js';
7
7
  import { compact, tryReadToolStructuredError, } from './owner.js';
8
8
  import { getTaskCapableTool, hasTaskCapableTool, } from './tool-registry.js';
9
+ const TASK_NOT_FOUND_ERROR_CODE = RESOURCE_NOT_FOUND_ERROR_CODE;
9
10
  /* -------------------------------------------------------------------------------------------------
10
11
  * Abort-controller management for in-flight task executions
11
12
  * ------------------------------------------------------------------------------------------------- */
@@ -113,7 +114,7 @@ function buildTaskStatusNotificationParams(task) {
113
114
  * Validation helpers
114
115
  * ------------------------------------------------------------------------------------------------- */
115
116
  export function throwTaskNotFound() {
116
- throw new McpError(RESOURCE_NOT_FOUND_ERROR_CODE, 'Task not found');
117
+ throw new McpError(TASK_NOT_FOUND_ERROR_CODE, 'Task not found');
117
118
  }
118
119
  function resolveTaskCapableTool(name) {
119
120
  const descriptor = getTaskCapableTool(name);
@@ -69,6 +69,8 @@ class TaskManager {
69
69
  applyTaskUpdate(task, updates) {
70
70
  Object.assign(task, updates);
71
71
  task.lastUpdatedAt = new Date().toISOString();
72
+ // Slide TTL window on every activity so long-running tasks don't expire mid-flight.
73
+ task._createdAtMs = Date.now();
72
74
  }
73
75
  cancelActiveTask(task, statusMessage) {
74
76
  this.applyTaskUpdate(task, {
@@ -1,12 +1,10 @@
1
1
  import { randomUUID } from 'node:crypto';
2
2
  import { ErrorCode, McpError } from '@modelcontextprotocol/sdk/types.js';
3
3
  import { z } from 'zod';
4
- import * as cache from '../lib/core.js';
5
4
  import { config } from '../lib/core.js';
6
5
  import { getRequestId, logDebug, logError, logWarn, runWithRequestContext, } from '../lib/core.js';
7
- import { generateSafeFilename } from '../lib/http.js';
8
6
  import { handleToolError } from '../lib/mcp-tools.js';
9
- import { appendTruncationMarker, markdownTransform, parseCachedMarkdownResult, performSharedFetch, readNestedRecord, readString, serializeMarkdownResult, TRUNCATION_MARKER, withSignal, } from '../lib/mcp-tools.js';
7
+ import { appendTruncationMarker, markdownTransform, parseCachedMarkdownResult, performSharedFetch, readNestedRecord, serializeMarkdownResult, TRUNCATION_MARKER, withSignal, } from '../lib/mcp-tools.js';
10
8
  import { createProgressReporter, } from '../lib/mcp-tools.js';
11
9
  import { isAbortError, isObject, toError } from '../lib/utils.js';
12
10
  import { fetchUrlInputSchema } from '../schemas/inputs.js';
@@ -19,7 +17,7 @@ const FETCH_URL_TOOL_DESCRIPTION = `
19
17
  <constraints>
20
18
  - READ-ONLY. No JavaScript execution.
21
19
  - GitHub/GitLab/Bitbucket URLs auto-transform to raw endpoints (check resolvedUrl).
22
- - If truncated=true, use cacheResourceUri with resources/read for full content.
20
+ - If truncated=true, full content is available in the next fetch with forceRefresh.
23
21
  - For large pages/timeouts, use task mode (task: {}).
24
22
  - If error queue_full, retry with task mode.
25
23
  </constraints>
@@ -38,47 +36,6 @@ function buildTextBlock(structuredContent) {
38
36
  text: JSON.stringify(structuredContent),
39
37
  };
40
38
  }
41
- function buildEmbeddedResource(content, url, title) {
42
- if (!content)
43
- return null;
44
- const filename = generateSafeFilename(url, title, undefined, '.md');
45
- const uri = `internal://inline/${encodeURIComponent(filename)}`;
46
- const resource = {
47
- uri,
48
- mimeType: 'text/markdown',
49
- text: content,
50
- };
51
- return {
52
- type: 'resource',
53
- resource,
54
- };
55
- }
56
- function buildCacheResourceLink(cacheResourceUri, contentSize, fetchedAt) {
57
- return {
58
- type: 'resource_link',
59
- uri: cacheResourceUri,
60
- name: 'cached-markdown',
61
- title: 'Cached Fetch Output',
62
- description: 'Read full markdown via resources/read.',
63
- mimeType: 'text/markdown',
64
- ...(contentSize > 0 ? { size: contentSize } : {}),
65
- annotations: {
66
- audience: ['assistant'],
67
- priority: 0.8,
68
- lastModified: fetchedAt,
69
- },
70
- };
71
- }
72
- function buildToolContentBlocks(structuredContent, resourceLink, embeddedResource) {
73
- const blocks = [buildTextBlock(structuredContent)];
74
- appendIfPresent(blocks, resourceLink);
75
- appendIfPresent(blocks, embeddedResource);
76
- return blocks;
77
- }
78
- function appendIfPresent(items, value) {
79
- if (value !== null && value !== undefined)
80
- items.push(value);
81
- }
82
39
  /* -------------------------------------------------------------------------------------------------
83
40
  * Tool abort signal
84
41
  * ------------------------------------------------------------------------------------------------- */
@@ -116,15 +73,15 @@ function truncateMetadata(metadata) {
116
73
  return result;
117
74
  }
118
75
  function buildStructuredContent(pipeline, inlineResult, inputUrl) {
119
- const cacheResourceUri = resolveCacheResourceUri(pipeline.cacheKey);
120
76
  const truncated = inlineResult.truncated ?? pipeline.data.truncated;
121
- const markdown = applyTruncationMarker(inlineResult.content, pipeline.data.truncated);
77
+ const rawMarkdown = applyTruncationMarker(inlineResult.content, pipeline.data.truncated);
78
+ const maxChars = config.constants.maxInlineContentChars;
79
+ const markdown = maxChars > 0 ? truncateStr(rawMarkdown, maxChars) : rawMarkdown;
122
80
  const { metadata } = pipeline.data;
123
81
  return {
124
82
  url: pipeline.originalUrl ?? pipeline.url,
125
83
  resolvedUrl: pipeline.url,
126
84
  ...(pipeline.finalUrl ? { finalUrl: pipeline.finalUrl } : {}),
127
- ...(cacheResourceUri ? { cacheResourceUri } : {}),
128
85
  inputUrl,
129
86
  title: truncateStr(pipeline.data.title, 512),
130
87
  ...(metadata ? { metadata: truncateMetadata(metadata) } : {}),
@@ -140,34 +97,12 @@ function applyTruncationMarker(content, truncated) {
140
97
  return content;
141
98
  return appendTruncationMarker(content, TRUNCATION_MARKER);
142
99
  }
143
- function resolveCacheResourceUri(cacheKey) {
144
- if (!cacheKey)
145
- return undefined;
146
- if (!cache.isEnabled())
147
- return undefined;
148
- if (!cache.get(cacheKey))
149
- return undefined;
150
- const parsed = cache.parseCacheKey(cacheKey);
151
- if (!parsed)
152
- return undefined;
153
- return `internal://cache/${encodeURIComponent(parsed.namespace)}/${encodeURIComponent(parsed.urlHash)}`;
154
- }
155
- function buildFetchUrlContentBlocks(structuredContent, pipeline, inlineResult) {
156
- const cacheResourceUri = readString(structuredContent, 'cacheResourceUri');
157
- const contentToEmbed = config.runtime.httpMode
158
- ? inlineResult.content
159
- : pipeline.data.content;
160
- const resourceLink = cacheResourceUri
161
- ? buildCacheResourceLink(cacheResourceUri, inlineResult.contentSize, pipeline.fetchedAt)
162
- : null;
163
- const embedded = contentToEmbed && pipeline.url
164
- ? buildEmbeddedResource(contentToEmbed, pipeline.url, pipeline.data.title)
165
- : null;
166
- return buildToolContentBlocks(structuredContent, resourceLink, embedded);
100
+ function buildFetchUrlContentBlocks(structuredContent) {
101
+ return [buildTextBlock(structuredContent)];
167
102
  }
168
103
  function buildResponse(pipeline, inlineResult, inputUrl) {
169
104
  const structuredContent = buildStructuredContent(pipeline, inlineResult, inputUrl);
170
- const content = buildFetchUrlContentBlocks(structuredContent, pipeline, inlineResult);
105
+ const content = buildFetchUrlContentBlocks(structuredContent);
171
106
  const validation = fetchUrlOutputSchema.safeParse(structuredContent);
172
107
  if (!validation.success) {
173
108
  logWarn('Tool output schema validation failed', {
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@j0hanz/fetch-url-mcp",
3
- "version": "1.5.1",
3
+ "version": "1.6.0",
4
4
  "mcpName": "io.github.j0hanz/fetch-url-mcp",
5
5
  "description": "Intelligent web content fetcher MCP server that converts HTML to clean, AI-readable Markdown",
6
6
  "type": "module",
@@ -81,7 +81,7 @@
81
81
  "eslint": "^10.0.2",
82
82
  "eslint-config-prettier": "^10.1.8",
83
83
  "eslint-plugin-de-morgan": "^2.1.1",
84
- "eslint-plugin-depend": "^1.4.0",
84
+ "eslint-plugin-depend": "^1.5.0",
85
85
  "eslint-plugin-unused-imports": "^4.4.1",
86
86
  "knip": "^5.85.0",
87
87
  "prettier": "^3.8.1",