@j0hanz/superfetch 2.2.2 → 2.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (86) hide show
  1. package/README.md +358 -363
  2. package/dist/assets/logo.svg +24835 -0
  3. package/dist/cache.d.ts +0 -1
  4. package/dist/cache.js +71 -29
  5. package/dist/config.d.ts +2 -1
  6. package/dist/config.js +11 -7
  7. package/dist/crypto.d.ts +0 -1
  8. package/dist/crypto.js +0 -1
  9. package/dist/dom-noise-removal.d.ts +0 -1
  10. package/dist/dom-noise-removal.js +50 -45
  11. package/dist/errors.d.ts +0 -1
  12. package/dist/errors.js +0 -1
  13. package/dist/fetch.d.ts +0 -1
  14. package/dist/fetch.js +61 -54
  15. package/dist/host-normalization.d.ts +1 -0
  16. package/dist/host-normalization.js +47 -0
  17. package/dist/http-native.d.ts +0 -1
  18. package/dist/http-native.js +92 -28
  19. package/dist/index.d.ts +0 -1
  20. package/dist/index.js +0 -1
  21. package/dist/instructions.md +41 -41
  22. package/dist/json.d.ts +0 -1
  23. package/dist/json.js +0 -1
  24. package/dist/language-detection.d.ts +0 -1
  25. package/dist/language-detection.js +10 -2
  26. package/dist/markdown-cleanup.d.ts +6 -13
  27. package/dist/markdown-cleanup.js +252 -34
  28. package/dist/mcp-validator.d.ts +14 -0
  29. package/dist/mcp-validator.js +22 -0
  30. package/dist/mcp.d.ts +0 -1
  31. package/dist/mcp.js +20 -10
  32. package/dist/observability.d.ts +2 -1
  33. package/dist/observability.js +30 -3
  34. package/dist/server-tuning.d.ts +9 -0
  35. package/dist/server-tuning.js +30 -0
  36. package/dist/{http-utils.d.ts → session.d.ts} +0 -25
  37. package/dist/{http-utils.js → session.js} +11 -104
  38. package/dist/tools.d.ts +5 -4
  39. package/dist/tools.js +46 -41
  40. package/dist/transform-types.d.ts +38 -1
  41. package/dist/transform-types.js +0 -1
  42. package/dist/transform.d.ts +12 -7
  43. package/dist/transform.js +205 -344
  44. package/dist/type-guards.d.ts +0 -1
  45. package/dist/type-guards.js +0 -1
  46. package/dist/workers/transform-worker.d.ts +0 -1
  47. package/dist/workers/transform-worker.js +29 -19
  48. package/package.json +84 -85
  49. package/dist/cache.d.ts.map +0 -1
  50. package/dist/cache.js.map +0 -1
  51. package/dist/config.d.ts.map +0 -1
  52. package/dist/config.js.map +0 -1
  53. package/dist/crypto.d.ts.map +0 -1
  54. package/dist/crypto.js.map +0 -1
  55. package/dist/dom-noise-removal.d.ts.map +0 -1
  56. package/dist/dom-noise-removal.js.map +0 -1
  57. package/dist/errors.d.ts.map +0 -1
  58. package/dist/errors.js.map +0 -1
  59. package/dist/fetch.d.ts.map +0 -1
  60. package/dist/fetch.js.map +0 -1
  61. package/dist/http-native.d.ts.map +0 -1
  62. package/dist/http-native.js.map +0 -1
  63. package/dist/http-utils.d.ts.map +0 -1
  64. package/dist/http-utils.js.map +0 -1
  65. package/dist/index.d.ts.map +0 -1
  66. package/dist/index.js.map +0 -1
  67. package/dist/json.d.ts.map +0 -1
  68. package/dist/json.js.map +0 -1
  69. package/dist/language-detection.d.ts.map +0 -1
  70. package/dist/language-detection.js.map +0 -1
  71. package/dist/markdown-cleanup.d.ts.map +0 -1
  72. package/dist/markdown-cleanup.js.map +0 -1
  73. package/dist/mcp.d.ts.map +0 -1
  74. package/dist/mcp.js.map +0 -1
  75. package/dist/observability.d.ts.map +0 -1
  76. package/dist/observability.js.map +0 -1
  77. package/dist/tools.d.ts.map +0 -1
  78. package/dist/tools.js.map +0 -1
  79. package/dist/transform-types.d.ts.map +0 -1
  80. package/dist/transform-types.js.map +0 -1
  81. package/dist/transform.d.ts.map +0 -1
  82. package/dist/transform.js.map +0 -1
  83. package/dist/type-guards.d.ts.map +0 -1
  84. package/dist/type-guards.js.map +0 -1
  85. package/dist/workers/transform-worker.d.ts.map +0 -1
  86. package/dist/workers/transform-worker.js.map +0 -1
@@ -0,0 +1,22 @@
1
+ import { z } from 'zod';
2
+ // --- Validation ---
3
+ const paramsSchema = z.looseObject({});
4
+ const mcpRequestSchema = z.looseObject({
5
+ jsonrpc: z.literal('2.0'),
6
+ method: z.string().min(1),
7
+ id: z.union([z.string(), z.number(), z.null()]).optional(),
8
+ params: paramsSchema.optional(),
9
+ });
10
+ export function isJsonRpcBatchRequest(body) {
11
+ return Array.isArray(body);
12
+ }
13
+ export function isMcpRequestBody(body) {
14
+ return mcpRequestSchema.safeParse(body).success;
15
+ }
16
+ export function acceptsEventStream(header) {
17
+ if (!header)
18
+ return false;
19
+ return header
20
+ .split(',')
21
+ .some((value) => value.trim().toLowerCase().startsWith('text/event-stream'));
22
+ }
package/dist/mcp.d.ts CHANGED
@@ -1,4 +1,3 @@
1
1
  import { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
2
2
  export declare function createMcpServer(): McpServer;
3
3
  export declare function startStdioServer(): Promise<void>;
4
- //# sourceMappingURL=mcp.d.ts.map
package/dist/mcp.js CHANGED
@@ -4,37 +4,47 @@ import { StdioServerTransport } from '@modelcontextprotocol/sdk/server/stdio.js'
4
4
  import { registerCachedContentResource } from './cache.js';
5
5
  import { config } from './config.js';
6
6
  import { destroyAgents } from './fetch.js';
7
- import { logError, logInfo } from './observability.js';
7
+ import { logError, logInfo, setMcpServer } from './observability.js';
8
8
  import { registerTools } from './tools.js';
9
9
  import { shutdownTransformWorkerPool } from './transform.js';
10
+ function getLocalIconData() {
11
+ try {
12
+ const iconPath = new URL('../assets/logo.svg', import.meta.url);
13
+ const buffer = readFileSync(iconPath);
14
+ return `data:image/svg+xml;base64,${buffer.toString('base64')}`;
15
+ }
16
+ catch {
17
+ return undefined;
18
+ }
19
+ }
10
20
  function createServerInfo() {
21
+ const localIcon = getLocalIconData();
11
22
  return {
12
23
  name: config.server.name,
13
24
  version: config.server.version,
25
+ ...(localIcon ? { icons: [{ src: localIcon, sizes: ['any'] }] } : {}),
14
26
  };
15
27
  }
16
28
  function createServerCapabilities() {
17
29
  return {
18
30
  tools: { listChanged: true },
19
31
  resources: { listChanged: true, subscribe: true },
32
+ logging: {},
20
33
  };
21
34
  }
22
35
  function createServerInstructions(serverVersion) {
23
36
  try {
24
- const raw = readFileSync(new URL('./instructions.md', import.meta.url), {
25
- encoding: 'utf8',
26
- });
27
- const resolved = raw.replaceAll('{{SERVER_VERSION}}', serverVersion);
28
- return resolved.trim();
37
+ const raw = readFileSync(new URL('./instructions.md', import.meta.url), 'utf8');
38
+ return raw.replaceAll('{{SERVER_VERSION}}', serverVersion).trim();
29
39
  }
30
40
  catch {
31
- return `superFetch MCP server |${serverVersion}| A high-performance web content fetching and processing server.`;
41
+ return `Instructions unavailable | ${serverVersion}`;
32
42
  }
33
43
  }
34
44
  function registerInstructionsResource(server, instructions) {
35
45
  server.registerResource('instructions', new ResourceTemplate('internal://instructions', { list: undefined }), {
36
- title: 'Server Instructions',
37
- description: 'Usage guidance for the superFetch MCP server.',
46
+ title: `SuperFetch MCP | ${config.server.version}`,
47
+ description: 'Guidance for using the superFetch MCP server.',
38
48
  mimeType: 'text/markdown',
39
49
  }, (uri) => ({
40
50
  contents: [
@@ -52,6 +62,7 @@ export function createMcpServer() {
52
62
  capabilities: createServerCapabilities(),
53
63
  instructions,
54
64
  });
65
+ setMcpServer(server);
55
66
  registerTools(server);
56
67
  registerCachedContentResource(server);
57
68
  registerInstructionsResource(server, instructions);
@@ -118,4 +129,3 @@ export async function startStdioServer() {
118
129
  registerSignalHandlers(createShutdownHandler(server));
119
130
  await connectStdioServer(server, transport);
120
131
  }
121
- //# sourceMappingURL=mcp.js.map
@@ -1,9 +1,11 @@
1
+ import type { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
1
2
  export type LogMetadata = Record<string, unknown>;
2
3
  interface RequestContext {
3
4
  readonly requestId: string;
4
5
  readonly sessionId?: string;
5
6
  readonly operationId?: string;
6
7
  }
8
+ export declare function setMcpServer(server: McpServer): void;
7
9
  export declare function runWithRequestContext<T>(context: RequestContext, fn: () => T): T;
8
10
  export declare function getRequestId(): string | undefined;
9
11
  export declare function getSessionId(): string | undefined;
@@ -15,4 +17,3 @@ export declare function logError(message: string, error?: Error | LogMetadata):
15
17
  export declare function redactUrl(rawUrl: string): string;
16
18
  export declare function redactHeaders(headers: Record<string, unknown>): Record<string, unknown>;
17
19
  export {};
18
- //# sourceMappingURL=observability.d.ts.map
@@ -1,6 +1,10 @@
1
1
  import { AsyncLocalStorage } from 'node:async_hooks';
2
2
  import { config } from './config.js';
3
3
  const requestContext = new AsyncLocalStorage();
4
+ let mcpServer;
5
+ export function setMcpServer(server) {
6
+ mcpServer = server;
7
+ }
4
8
  export function runWithRequestContext(context, fn) {
5
9
  return requestContext.run(context, fn);
6
10
  }
@@ -13,7 +17,7 @@ export function getSessionId() {
13
17
  export function getOperationId() {
14
18
  return requestContext.getStore()?.operationId;
15
19
  }
16
- function formatMetadata(meta) {
20
+ function buildContextMetadata() {
17
21
  const requestId = getRequestId();
18
22
  const sessionId = getSessionId();
19
23
  const operationId = getOperationId();
@@ -24,7 +28,10 @@ function formatMetadata(meta) {
24
28
  contextMeta.sessionId = sessionId;
25
29
  if (operationId)
26
30
  contextMeta.operationId = operationId;
27
- const merged = { ...contextMeta, ...meta };
31
+ return contextMeta;
32
+ }
33
+ function formatMetadata(meta) {
34
+ const merged = { ...buildContextMetadata(), ...meta };
28
35
  return Object.keys(merged).length > 0 ? ` ${JSON.stringify(merged)}` : '';
29
36
  }
30
37
  function createTimestamp() {
@@ -40,10 +47,31 @@ function shouldLog(level) {
40
47
  // All other levels always log
41
48
  return true;
42
49
  }
50
+ function mapToMcpLevel(level) {
51
+ switch (level) {
52
+ case 'warn':
53
+ return 'warning';
54
+ case 'error':
55
+ return 'error';
56
+ case 'debug':
57
+ return 'debug';
58
+ case 'info':
59
+ default:
60
+ return 'info';
61
+ }
62
+ }
43
63
  function writeLog(level, message, meta) {
44
64
  if (!shouldLog(level))
45
65
  return;
46
66
  process.stderr.write(`${formatLogEntry(level, message, meta)}\n`);
67
+ if (mcpServer) {
68
+ mcpServer.server
69
+ .sendLoggingMessage({
70
+ level: mapToMcpLevel(level),
71
+ data: meta ? { message, ...meta } : message,
72
+ })
73
+ .catch(() => { });
74
+ }
47
75
  }
48
76
  export function logInfo(message, meta) {
49
77
  writeLog('info', message, meta);
@@ -83,4 +111,3 @@ export function redactHeaders(headers) {
83
111
  }
84
112
  return redacted;
85
113
  }
86
- //# sourceMappingURL=observability.js.map
@@ -0,0 +1,9 @@
1
+ export interface HttpServerTuningTarget {
2
+ headersTimeout?: number;
3
+ requestTimeout?: number;
4
+ keepAliveTimeout?: number;
5
+ closeIdleConnections?: () => void;
6
+ closeAllConnections?: () => void;
7
+ }
8
+ export declare function applyHttpServerTuning(server: HttpServerTuningTarget): void;
9
+ export declare function drainConnectionsOnShutdown(server: HttpServerTuningTarget): void;
@@ -0,0 +1,30 @@
1
+ import { config } from './config.js';
2
+ import { logDebug } from './observability.js';
3
+ export function applyHttpServerTuning(server) {
4
+ const { headersTimeoutMs, requestTimeoutMs, keepAliveTimeoutMs } = config.server.http;
5
+ if (headersTimeoutMs !== undefined) {
6
+ server.headersTimeout = headersTimeoutMs;
7
+ }
8
+ if (requestTimeoutMs !== undefined) {
9
+ server.requestTimeout = requestTimeoutMs;
10
+ }
11
+ if (keepAliveTimeoutMs !== undefined) {
12
+ server.keepAliveTimeout = keepAliveTimeoutMs;
13
+ }
14
+ }
15
+ export function drainConnectionsOnShutdown(server) {
16
+ const { shutdownCloseAllConnections, shutdownCloseIdleConnections } = config.server.http;
17
+ if (shutdownCloseAllConnections) {
18
+ if (typeof server.closeAllConnections === 'function') {
19
+ server.closeAllConnections();
20
+ logDebug('Closed all HTTP connections during shutdown');
21
+ }
22
+ return;
23
+ }
24
+ if (shutdownCloseIdleConnections) {
25
+ if (typeof server.closeIdleConnections === 'function') {
26
+ server.closeIdleConnections();
27
+ logDebug('Closed idle HTTP connections during shutdown');
28
+ }
29
+ }
30
+ }
@@ -1,15 +1,4 @@
1
1
  import type { StreamableHTTPServerTransport } from '@modelcontextprotocol/sdk/server/streamableHttp.js';
2
- export type JsonRpcId = string | number | null;
3
- export interface McpRequestParams {
4
- _meta?: Record<string, unknown>;
5
- [key: string]: unknown;
6
- }
7
- export interface McpRequestBody {
8
- jsonrpc: '2.0';
9
- method: string;
10
- id?: JsonRpcId;
11
- params?: McpRequestParams;
12
- }
13
2
  export interface SessionEntry {
14
3
  readonly transport: StreamableHTTPServerTransport;
15
4
  createdAt: number;
@@ -34,14 +23,10 @@ export interface SlotTracker {
34
23
  readonly markInitialized: () => void;
35
24
  readonly isInitialized: () => boolean;
36
25
  }
37
- export declare function normalizeHost(value: string): string | null;
38
26
  export type CloseHandler = (() => void) | undefined;
39
27
  export declare function composeCloseHandlers(first: CloseHandler, second: CloseHandler): CloseHandler;
40
28
  export declare function startSessionCleanupLoop(store: SessionStore, sessionTtlMs: number): AbortController;
41
29
  export declare function createSessionStore(sessionTtlMs: number): SessionStore;
42
- export declare function isJsonRpcBatchRequest(body: unknown): boolean;
43
- export declare function isMcpRequestBody(body: unknown): body is McpRequestBody;
44
- export declare function acceptsEventStream(header: string | null | undefined): boolean;
45
30
  export declare function createSlotTracker(store: SessionStore): SlotTracker;
46
31
  export declare function reserveSessionSlot(store: SessionStore, maxSessions: number): boolean;
47
32
  export declare function ensureSessionCapacity({ store, maxSessions, evictOldest, }: {
@@ -49,13 +34,3 @@ export declare function ensureSessionCapacity({ store, maxSessions, evictOldest,
49
34
  maxSessions: number;
50
35
  evictOldest: (store: SessionStore) => boolean;
51
36
  }): boolean;
52
- export interface HttpServerTuningTarget {
53
- headersTimeout?: number;
54
- requestTimeout?: number;
55
- keepAliveTimeout?: number;
56
- closeIdleConnections?: () => void;
57
- closeAllConnections?: () => void;
58
- }
59
- export declare function applyHttpServerTuning(server: HttpServerTuningTarget): void;
60
- export declare function drainConnectionsOnShutdown(server: HttpServerTuningTarget): void;
61
- //# sourceMappingURL=http-utils.d.ts.map
@@ -1,55 +1,5 @@
1
- import { isIP } from 'node:net';
2
1
  import { setInterval as setIntervalPromise } from 'node:timers/promises';
3
- import { z } from 'zod';
4
- import { config } from './config.js';
5
- import { logDebug, logInfo, logWarn } from './observability.js';
6
- // --- Host Normalization ---
7
- export function normalizeHost(value) {
8
- const trimmed = value.trim().toLowerCase();
9
- if (!trimmed)
10
- return null;
11
- const first = takeFirstHostValue(trimmed);
12
- if (!first)
13
- return null;
14
- const ipv6 = stripIpv6Brackets(first);
15
- if (ipv6)
16
- return stripTrailingDots(ipv6);
17
- if (isIpV6Literal(first)) {
18
- return stripTrailingDots(first);
19
- }
20
- return stripTrailingDots(stripPortIfPresent(first));
21
- }
22
- function takeFirstHostValue(value) {
23
- const first = value.split(',')[0];
24
- if (!first)
25
- return null;
26
- const trimmed = first.trim();
27
- return trimmed ? trimmed : null;
28
- }
29
- function stripIpv6Brackets(value) {
30
- if (!value.startsWith('['))
31
- return null;
32
- const end = value.indexOf(']');
33
- if (end === -1)
34
- return null;
35
- return value.slice(1, end);
36
- }
37
- function stripPortIfPresent(value) {
38
- const colonIndex = value.indexOf(':');
39
- if (colonIndex === -1)
40
- return value;
41
- return value.slice(0, colonIndex);
42
- }
43
- function isIpV6Literal(value) {
44
- return isIP(value) === 6;
45
- }
46
- function stripTrailingDots(value) {
47
- let result = value;
48
- while (result.endsWith('.')) {
49
- result = result.slice(0, -1);
50
- }
51
- return result;
52
- }
2
+ import { logInfo, logWarn } from './observability.js';
53
3
  export function composeCloseHandlers(first, second) {
54
4
  if (!first)
55
5
  return second;
@@ -79,6 +29,13 @@ function handleSessionCleanupError(error) {
79
29
  error: error instanceof Error ? error.message : 'Unknown error',
80
30
  });
81
31
  }
32
+ function moveSessionToEnd(sessions, sessionId, session) {
33
+ sessions.delete(sessionId);
34
+ sessions.set(sessionId, session);
35
+ }
36
+ function isSessionExpired(session, now, sessionTtlMs) {
37
+ return now - session.lastSeen > sessionTtlMs;
38
+ }
82
39
  async function runSessionCleanupLoop(store, sessionTtlMs, signal) {
83
40
  const intervalMs = getCleanupIntervalMs(sessionTtlMs);
84
41
  for await (const getNow of setIntervalPromise(intervalMs, Date.now, {
@@ -115,8 +72,8 @@ export function createSessionStore(sessionTtlMs) {
115
72
  const session = sessions.get(sessionId);
116
73
  if (session) {
117
74
  session.lastSeen = Date.now();
118
- sessions.delete(sessionId); // Move to end (LRU behavior if needed, but Map insertion order)
119
- sessions.set(sessionId, session);
75
+ // Move to end (LRU behavior if needed, but Map insertion order)
76
+ moveSessionToEnd(sessions, sessionId, session);
120
77
  }
121
78
  },
122
79
  set: (sessionId, entry) => {
@@ -145,7 +102,7 @@ export function createSessionStore(sessionTtlMs) {
145
102
  const now = Date.now();
146
103
  const evicted = [];
147
104
  for (const [id, session] of sessions.entries()) {
148
- if (now - session.lastSeen > sessionTtlMs) {
105
+ if (isSessionExpired(session, now, sessionTtlMs)) {
149
106
  sessions.delete(id);
150
107
  evicted.push(session);
151
108
  }
@@ -163,27 +120,6 @@ export function createSessionStore(sessionTtlMs) {
163
120
  },
164
121
  };
165
122
  }
166
- // --- Validation ---
167
- const paramsSchema = z.looseObject({});
168
- const mcpRequestSchema = z.looseObject({
169
- jsonrpc: z.literal('2.0'),
170
- method: z.string().min(1),
171
- id: z.union([z.string(), z.number(), z.null()]).optional(),
172
- params: paramsSchema.optional(),
173
- });
174
- export function isJsonRpcBatchRequest(body) {
175
- return Array.isArray(body);
176
- }
177
- export function isMcpRequestBody(body) {
178
- return mcpRequestSchema.safeParse(body).success;
179
- }
180
- export function acceptsEventStream(header) {
181
- if (!header)
182
- return false;
183
- return header
184
- .split(',')
185
- .some((value) => value.trim().toLowerCase().startsWith('text/event-stream'));
186
- }
187
123
  // --- Slot Tracker ---
188
124
  export function createSlotTracker(store) {
189
125
  let slotReleased = false;
@@ -221,32 +157,3 @@ export function ensureSessionCapacity({ store, maxSessions, evictOldest, }) {
221
157
  }
222
158
  return false;
223
159
  }
224
- export function applyHttpServerTuning(server) {
225
- const { headersTimeoutMs, requestTimeoutMs, keepAliveTimeoutMs } = config.server.http;
226
- if (headersTimeoutMs !== undefined) {
227
- server.headersTimeout = headersTimeoutMs;
228
- }
229
- if (requestTimeoutMs !== undefined) {
230
- server.requestTimeout = requestTimeoutMs;
231
- }
232
- if (keepAliveTimeoutMs !== undefined) {
233
- server.keepAliveTimeout = keepAliveTimeoutMs;
234
- }
235
- }
236
- export function drainConnectionsOnShutdown(server) {
237
- const { shutdownCloseAllConnections, shutdownCloseIdleConnections } = config.server.http;
238
- if (shutdownCloseAllConnections) {
239
- if (typeof server.closeAllConnections === 'function') {
240
- server.closeAllConnections();
241
- logDebug('Closed all HTTP connections during shutdown');
242
- }
243
- return;
244
- }
245
- if (shutdownCloseIdleConnections) {
246
- if (typeof server.closeIdleConnections === 'function') {
247
- server.closeIdleConnections();
248
- logDebug('Closed idle HTTP connections during shutdown');
249
- }
250
- }
251
- }
252
- //# sourceMappingURL=http-utils.js.map
package/dist/tools.d.ts CHANGED
@@ -1,5 +1,5 @@
1
1
  import type { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
2
- import { type MarkdownTransformResult } from './transform.js';
2
+ import type { MarkdownTransformResult } from './transform-types.js';
3
3
  export interface FetchUrlInput {
4
4
  url: string;
5
5
  }
@@ -25,7 +25,6 @@ export interface ToolContentResourceBlock {
25
25
  }
26
26
  export type ToolContentBlockUnion = ToolContentBlock | ToolContentResourceLinkBlock | ToolContentResourceBlock;
27
27
  export interface ToolErrorResponse {
28
- [x: string]: unknown;
29
28
  content: ToolContentBlockUnion[];
30
29
  structuredContent: {
31
30
  error: string;
@@ -34,7 +33,6 @@ export interface ToolErrorResponse {
34
33
  isError: true;
35
34
  }
36
35
  export interface ToolResponseBase {
37
- [x: string]: unknown;
38
36
  content: ToolContentBlockUnion[];
39
37
  structuredContent?: Record<string, unknown>;
40
38
  isError?: boolean;
@@ -86,6 +84,10 @@ export interface ToolHandlerExtra {
86
84
  }
87
85
  export declare const FETCH_URL_TOOL_NAME = "fetch-url";
88
86
  export declare const FETCH_URL_TOOL_DESCRIPTION = "Fetches a webpage and converts it to clean Markdown format";
87
+ interface ProgressReporter {
88
+ report: (progress: number, message: string) => Promise<void>;
89
+ }
90
+ export declare function createProgressReporter(extra?: ToolHandlerExtra): ProgressReporter;
89
91
  interface InlineContentResult {
90
92
  content?: string;
91
93
  contentSize: number;
@@ -125,4 +127,3 @@ export declare function fetchUrlToolHandler(input: FetchUrlInput, extra?: ToolHa
125
127
  export declare function withRequestContextIfMissing<TParams, TResult, TExtra = unknown>(handler: (params: TParams, extra?: TExtra) => Promise<TResult>): (params: TParams, extra?: TExtra) => Promise<TResult>;
126
128
  export declare function registerTools(server: McpServer): void;
127
129
  export {};
128
- //# sourceMappingURL=tools.d.ts.map
package/dist/tools.js CHANGED
@@ -5,10 +5,11 @@ import { config } from './config.js';
5
5
  import { FetchError, getErrorMessage, isSystemError } from './errors.js';
6
6
  import { fetchNormalizedUrl, normalizeUrl, transformToRawUrl, } from './fetch.js';
7
7
  import { getRequestId, logDebug, logError, logWarn, runWithRequestContext, } from './observability.js';
8
- import { transformHtmlToMarkdown, } from './transform.js';
8
+ import { transformHtmlToMarkdown } from './transform.js';
9
9
  import { isObject } from './type-guards.js';
10
10
  const TRUNCATION_MARKER = '...[truncated]';
11
11
  const FETCH_PROGRESS_TOTAL = 4;
12
+ const PROGRESS_NOTIFICATION_TIMEOUT_MS = 5000;
12
13
  const fetchUrlInputSchema = z.strictObject({
13
14
  url: z
14
15
  .url({ protocol: /^https?$/i })
@@ -46,7 +47,7 @@ const fetchUrlOutputSchema = z.strictObject({
46
47
  });
47
48
  export const FETCH_URL_TOOL_NAME = 'fetch-url';
48
49
  export const FETCH_URL_TOOL_DESCRIPTION = 'Fetches a webpage and converts it to clean Markdown format';
49
- function createProgressReporter(extra) {
50
+ export function createProgressReporter(extra) {
50
51
  const token = extra?._meta?.progressToken ?? null;
51
52
  const sendNotification = extra?.sendNotification;
52
53
  if (token === null || !sendNotification) {
@@ -55,19 +56,33 @@ function createProgressReporter(extra) {
55
56
  return {
56
57
  report: async (progress, message) => {
57
58
  try {
58
- await sendNotification({
59
- method: 'notifications/progress',
60
- params: {
61
- progressToken: token,
62
- progress,
63
- total: FETCH_PROGRESS_TOTAL,
64
- message,
65
- },
66
- });
59
+ await Promise.race([
60
+ sendNotification({
61
+ method: 'notifications/progress',
62
+ params: {
63
+ progressToken: token,
64
+ progress,
65
+ total: FETCH_PROGRESS_TOTAL,
66
+ message,
67
+ },
68
+ }),
69
+ new Promise((_, reject) => {
70
+ setTimeout(() => {
71
+ reject(new Error('Progress notification timeout'));
72
+ }, PROGRESS_NOTIFICATION_TIMEOUT_MS);
73
+ }),
74
+ ]);
67
75
  }
68
76
  catch (error) {
69
- logWarn('Failed to send progress notification', {
77
+ const isTimeout = error instanceof Error &&
78
+ error.message === 'Progress notification timeout';
79
+ const logMessage = isTimeout
80
+ ? 'Progress notification timed out'
81
+ : 'Failed to send progress notification';
82
+ logWarn(logMessage, {
70
83
  error: getErrorMessage(error),
84
+ progress,
85
+ message,
71
86
  });
72
87
  }
73
88
  },
@@ -106,23 +121,16 @@ function buildEmbeddedResource(content, url, title) {
106
121
  },
107
122
  };
108
123
  }
109
- function resolveContentToEmbed(inlineResult, fullContent, useInlineInHttpMode) {
110
- if (useInlineInHttpMode) {
111
- return inlineResult.content;
112
- }
113
- return fullContent ?? inlineResult.content;
114
- }
115
- function maybeAppendEmbeddedResource(blocks, contentToEmbed, url, title) {
116
- if (!contentToEmbed)
117
- return;
118
- if (!url)
119
- return;
120
- const embeddedResource = buildEmbeddedResource(contentToEmbed, url, title);
121
- if (embeddedResource) {
122
- blocks.push(embeddedResource);
124
+ function appendResourceBlocks({ blocks, inlineResult, resourceName, url, title, fullContent, }) {
125
+ const contentToEmbed = config.runtime.httpMode
126
+ ? inlineResult.content
127
+ : (fullContent ?? inlineResult.content);
128
+ if (contentToEmbed && url) {
129
+ const embeddedResource = buildEmbeddedResource(contentToEmbed, url, title);
130
+ if (embeddedResource) {
131
+ blocks.push(embeddedResource);
132
+ }
123
133
  }
124
- }
125
- function maybeAppendResourceLink(blocks, inlineResult, resourceName) {
126
134
  const resourceLink = buildResourceLink(inlineResult, resourceName);
127
135
  if (resourceLink) {
128
136
  blocks.push(resourceLink);
@@ -136,9 +144,14 @@ function buildTextBlock(structuredContent) {
136
144
  }
137
145
  function buildToolContentBlocks(structuredContent, fromCache, inlineResult, resourceName, cacheKey, fullContent, url, title) {
138
146
  const blocks = [buildTextBlock(structuredContent)];
139
- const contentToEmbed = resolveContentToEmbed(inlineResult, fullContent, config.runtime.httpMode);
140
- maybeAppendEmbeddedResource(blocks, contentToEmbed, url, title);
141
- maybeAppendResourceLink(blocks, inlineResult, resourceName);
147
+ appendResourceBlocks({
148
+ blocks,
149
+ inlineResult,
150
+ resourceName,
151
+ url,
152
+ title,
153
+ fullContent,
154
+ });
142
155
  return blocks;
143
156
  }
144
157
  function applyInlineContentLimit(content, cacheKey) {
@@ -266,14 +279,6 @@ function logRawUrlTransformation(resolvedUrl) {
266
279
  original: resolvedUrl.originalUrl,
267
280
  });
268
281
  }
269
- function applyOptionalPipelineSerialization(pipelineOptions, options) {
270
- if (options.serialize !== undefined) {
271
- pipelineOptions.serialize = options.serialize;
272
- }
273
- if (options.deserialize !== undefined) {
274
- pipelineOptions.deserialize = options.deserialize;
275
- }
276
- }
277
282
  export async function performSharedFetch(options, deps = {}) {
278
283
  const executePipeline = deps.executeFetchPipeline ?? executeFetchPipeline;
279
284
  const pipelineOptions = {
@@ -281,8 +286,9 @@ export async function performSharedFetch(options, deps = {}) {
281
286
  cacheNamespace: 'markdown',
282
287
  ...(options.signal === undefined ? {} : { signal: options.signal }),
283
288
  transform: options.transform,
289
+ ...(options.serialize ? { serialize: options.serialize } : {}),
290
+ ...(options.deserialize ? { deserialize: options.deserialize } : {}),
284
291
  };
285
- applyOptionalPipelineSerialization(pipelineOptions, options);
286
292
  const pipeline = await executePipeline(pipelineOptions);
287
293
  const inlineResult = applyInlineContentLimit(pipeline.data.content, pipeline.cacheKey ?? null);
288
294
  return { pipeline, inlineResult };
@@ -485,4 +491,3 @@ export function registerTools(server) {
485
491
  annotations: TOOL_DEFINITION.annotations,
486
492
  }, withRequestContextIfMissing(TOOL_DEFINITION.handler));
487
493
  }
488
- //# sourceMappingURL=tools.js.map
@@ -77,5 +77,42 @@ export interface TransformStageContext {
77
77
  readonly stage: string;
78
78
  readonly startTime: number;
79
79
  readonly url: string;
80
+ readonly budgetMs?: number;
81
+ readonly totalBudgetMs?: number;
80
82
  }
81
- //# sourceMappingURL=transform-types.d.ts.map
83
+ /**
84
+ * Worker message types for transform workers.
85
+ */
86
+ export interface TransformWorkerTransformMessage {
87
+ type: 'transform';
88
+ id: string;
89
+ html: string;
90
+ url: string;
91
+ includeMetadata: boolean;
92
+ }
93
+ export interface TransformWorkerCancelMessage {
94
+ type: 'cancel';
95
+ id: string;
96
+ }
97
+ export interface TransformWorkerResultMessage {
98
+ type: 'result';
99
+ id: string;
100
+ result: {
101
+ markdown: string;
102
+ title?: string;
103
+ truncated: boolean;
104
+ };
105
+ }
106
+ export interface TransformWorkerErrorMessage {
107
+ type: 'error';
108
+ id: string;
109
+ error: {
110
+ name: string;
111
+ message: string;
112
+ url: string;
113
+ statusCode?: number;
114
+ details?: Record<string, unknown>;
115
+ };
116
+ }
117
+ export type TransformWorkerIncomingMessage = TransformWorkerTransformMessage | TransformWorkerCancelMessage;
118
+ export type TransformWorkerOutgoingMessage = TransformWorkerResultMessage | TransformWorkerErrorMessage;
@@ -3,4 +3,3 @@
3
3
  * Extracted to avoid circular dependencies between transform modules.
4
4
  */
5
5
  export {};
6
- //# sourceMappingURL=transform-types.js.map