@j0hanz/superfetch 2.0.1 → 2.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (79) hide show
  1. package/README.md +120 -38
  2. package/dist/cache.d.ts +42 -0
  3. package/dist/cache.js +565 -0
  4. package/dist/config/env-parsers.d.ts +1 -0
  5. package/dist/config/env-parsers.js +12 -0
  6. package/dist/config/index.d.ts +7 -0
  7. package/dist/config/index.js +10 -3
  8. package/dist/config/types/content.d.ts +1 -0
  9. package/dist/config.d.ts +77 -0
  10. package/dist/config.js +261 -0
  11. package/dist/crypto.d.ts +2 -0
  12. package/dist/crypto.js +32 -0
  13. package/dist/errors.d.ts +10 -0
  14. package/dist/errors.js +28 -0
  15. package/dist/fetch.d.ts +40 -0
  16. package/dist/fetch.js +910 -0
  17. package/dist/http/base-middleware.d.ts +7 -0
  18. package/dist/http/base-middleware.js +143 -0
  19. package/dist/http/cors.d.ts +0 -5
  20. package/dist/http/cors.js +0 -6
  21. package/dist/http/download-routes.js +6 -2
  22. package/dist/http/error-handler.d.ts +2 -0
  23. package/dist/http/error-handler.js +55 -0
  24. package/dist/http/mcp-routes.js +2 -2
  25. package/dist/http/mcp-sessions.d.ts +3 -5
  26. package/dist/http/mcp-sessions.js +8 -8
  27. package/dist/http/server-tuning.d.ts +9 -0
  28. package/dist/http/server-tuning.js +45 -0
  29. package/dist/http/server.d.ts +0 -10
  30. package/dist/http/server.js +33 -333
  31. package/dist/http.d.ts +78 -0
  32. package/dist/http.js +1437 -0
  33. package/dist/index.js +3 -3
  34. package/dist/mcp.d.ts +3 -0
  35. package/dist/mcp.js +94 -0
  36. package/dist/observability.d.ts +16 -0
  37. package/dist/observability.js +78 -0
  38. package/dist/server.js +20 -5
  39. package/dist/services/cache.d.ts +1 -1
  40. package/dist/services/context.d.ts +2 -0
  41. package/dist/services/context.js +3 -0
  42. package/dist/services/extractor.d.ts +1 -0
  43. package/dist/services/extractor.js +28 -2
  44. package/dist/services/fetcher.d.ts +2 -0
  45. package/dist/services/fetcher.js +35 -14
  46. package/dist/services/logger.js +4 -1
  47. package/dist/services/telemetry.d.ts +19 -0
  48. package/dist/services/telemetry.js +43 -0
  49. package/dist/services/transform-worker-pool.d.ts +10 -3
  50. package/dist/services/transform-worker-pool.js +213 -184
  51. package/dist/tools/handlers/fetch-url.tool.js +8 -6
  52. package/dist/tools/index.d.ts +1 -0
  53. package/dist/tools/index.js +13 -1
  54. package/dist/tools/schemas.d.ts +2 -0
  55. package/dist/tools/schemas.js +8 -0
  56. package/dist/tools/utils/content-transform-core.d.ts +5 -0
  57. package/dist/tools/utils/content-transform-core.js +180 -0
  58. package/dist/tools/utils/content-transform-workers.d.ts +1 -0
  59. package/dist/tools/utils/content-transform-workers.js +1 -0
  60. package/dist/tools/utils/content-transform.d.ts +3 -5
  61. package/dist/tools/utils/content-transform.js +35 -148
  62. package/dist/tools/utils/raw-markdown.js +15 -1
  63. package/dist/tools.d.ts +104 -0
  64. package/dist/tools.js +421 -0
  65. package/dist/transform.d.ts +69 -0
  66. package/dist/transform.js +1509 -0
  67. package/dist/transformers/markdown.d.ts +4 -1
  68. package/dist/transformers/markdown.js +182 -53
  69. package/dist/utils/cancellation.d.ts +1 -0
  70. package/dist/utils/cancellation.js +18 -0
  71. package/dist/utils/code-language.d.ts +0 -9
  72. package/dist/utils/code-language.js +5 -5
  73. package/dist/utils/host-normalizer.d.ts +1 -0
  74. package/dist/utils/host-normalizer.js +37 -0
  75. package/dist/utils/url-redactor.d.ts +1 -0
  76. package/dist/utils/url-redactor.js +13 -0
  77. package/dist/utils/url-validator.js +8 -5
  78. package/dist/workers/transform-worker.js +82 -38
  79. package/package.json +8 -7
@@ -0,0 +1,7 @@
1
+ import type { Express, RequestHandler } from 'express';
2
+ export declare function attachBaseMiddleware(options: {
3
+ app: Express;
4
+ jsonParser: RequestHandler;
5
+ rateLimitMiddleware: RequestHandler;
6
+ corsMiddleware: RequestHandler;
7
+ }): void;
@@ -0,0 +1,143 @@
1
+ import { randomUUID } from 'node:crypto';
2
+ import { config } from '../config/index.js';
3
+ import { runWithRequestContext } from '../services/context.js';
4
+ import { logDebug } from '../services/logger.js';
5
+ import { normalizeHost } from '../utils/host-normalizer.js';
6
+ import { getSessionId } from './mcp-sessions.js';
7
+ const LOOPBACK_HOSTS = new Set(['localhost', '127.0.0.1', '::1']);
8
+ function getNonEmptyStringHeader(value) {
9
+ if (typeof value !== 'string')
10
+ return null;
11
+ const trimmed = value.trim();
12
+ return trimmed === '' ? null : trimmed;
13
+ }
14
+ function respondHostNotAllowed(res) {
15
+ res.status(403).json({
16
+ error: 'Host not allowed',
17
+ code: 'HOST_NOT_ALLOWED',
18
+ });
19
+ }
20
+ function respondOriginNotAllowed(res) {
21
+ res.status(403).json({
22
+ error: 'Origin not allowed',
23
+ code: 'ORIGIN_NOT_ALLOWED',
24
+ });
25
+ }
26
+ function tryParseOriginHostname(originHeader) {
27
+ try {
28
+ return new URL(originHeader).hostname.toLowerCase();
29
+ }
30
+ catch {
31
+ return null;
32
+ }
33
+ }
34
+ function isWildcardHost(host) {
35
+ return host === '0.0.0.0' || host === '::';
36
+ }
37
+ function addLoopbackHosts(allowedHosts) {
38
+ for (const host of LOOPBACK_HOSTS) {
39
+ allowedHosts.add(host);
40
+ }
41
+ }
42
+ function addConfiguredHost(allowedHosts) {
43
+ const configuredHost = normalizeHost(config.server.host);
44
+ if (!configuredHost)
45
+ return;
46
+ if (isWildcardHost(configuredHost))
47
+ return;
48
+ allowedHosts.add(configuredHost);
49
+ }
50
+ function addExplicitAllowedHosts(allowedHosts) {
51
+ for (const host of config.security.allowedHosts) {
52
+ const normalized = normalizeHost(host);
53
+ if (!normalized) {
54
+ logDebug('Ignoring invalid allowed host entry', { host });
55
+ continue;
56
+ }
57
+ allowedHosts.add(normalized);
58
+ }
59
+ }
60
+ function buildAllowedHosts() {
61
+ const allowedHosts = new Set();
62
+ addLoopbackHosts(allowedHosts);
63
+ addConfiguredHost(allowedHosts);
64
+ addExplicitAllowedHosts(allowedHosts);
65
+ return allowedHosts;
66
+ }
67
+ function createHostValidationMiddleware() {
68
+ const allowedHosts = buildAllowedHosts();
69
+ return (req, res, next) => {
70
+ const hostHeader = typeof req.headers.host === 'string' ? req.headers.host : '';
71
+ const normalized = normalizeHost(hostHeader);
72
+ if (!normalized || !allowedHosts.has(normalized)) {
73
+ respondHostNotAllowed(res);
74
+ return;
75
+ }
76
+ next();
77
+ };
78
+ }
79
+ function createOriginValidationMiddleware() {
80
+ const allowedHosts = buildAllowedHosts();
81
+ return (req, res, next) => {
82
+ const originHeader = getNonEmptyStringHeader(req.headers.origin);
83
+ if (!originHeader) {
84
+ next();
85
+ return;
86
+ }
87
+ const originHostname = tryParseOriginHostname(originHeader);
88
+ if (!originHostname || !allowedHosts.has(originHostname)) {
89
+ respondOriginNotAllowed(res);
90
+ return;
91
+ }
92
+ next();
93
+ };
94
+ }
95
+ function createJsonParseErrorHandler() {
96
+ return (err, _req, res, next) => {
97
+ if (err instanceof SyntaxError && 'body' in err) {
98
+ res.status(400).json({
99
+ jsonrpc: '2.0',
100
+ error: {
101
+ code: -32700,
102
+ message: 'Parse error: Invalid JSON',
103
+ },
104
+ id: null,
105
+ });
106
+ return;
107
+ }
108
+ next();
109
+ };
110
+ }
111
+ function createContextMiddleware() {
112
+ return (req, _res, next) => {
113
+ const requestId = randomUUID();
114
+ const sessionId = getSessionId(req);
115
+ const context = sessionId === undefined
116
+ ? { requestId, operationId: requestId }
117
+ : { requestId, operationId: requestId, sessionId };
118
+ runWithRequestContext(context, () => {
119
+ next();
120
+ });
121
+ };
122
+ }
123
+ function registerHealthRoute(app) {
124
+ app.get('/health', (_req, res) => {
125
+ res.json({
126
+ status: 'healthy',
127
+ name: config.server.name,
128
+ version: config.server.version,
129
+ uptime: process.uptime(),
130
+ });
131
+ });
132
+ }
133
+ export function attachBaseMiddleware(options) {
134
+ const { app, jsonParser, rateLimitMiddleware, corsMiddleware } = options;
135
+ app.use(createHostValidationMiddleware());
136
+ app.use(createOriginValidationMiddleware());
137
+ app.use(jsonParser);
138
+ app.use(createContextMiddleware());
139
+ app.use(createJsonParseErrorHandler());
140
+ app.use(corsMiddleware);
141
+ app.use('/mcp', rateLimitMiddleware);
142
+ registerHealthRoute(app);
143
+ }
@@ -1,7 +1,2 @@
1
1
  import type { NextFunction, Request, Response } from 'express';
2
- /**
3
- * Creates a minimal CORS middleware.
4
- * MCP clients are not browser-based, so CORS is not needed.
5
- * This just handles OPTIONS preflight requests.
6
- */
7
2
  export declare function createCorsMiddleware(): (req: Request, res: Response, next: NextFunction) => void;
package/dist/http/cors.js CHANGED
@@ -1,11 +1,5 @@
1
- /**
2
- * Creates a minimal CORS middleware.
3
- * MCP clients are not browser-based, so CORS is not needed.
4
- * This just handles OPTIONS preflight requests.
5
- */
6
1
  export function createCorsMiddleware() {
7
2
  return (req, res, next) => {
8
- // Handle OPTIONS preflight
9
3
  if (req.method === 'OPTIONS') {
10
4
  res.sendStatus(200);
11
5
  return;
@@ -3,7 +3,6 @@ import * as cache from '../services/cache.js';
3
3
  import { logDebug } from '../services/logger.js';
4
4
  import { parseCachedPayload, resolveCachedPayloadContent, } from '../utils/cached-payload.js';
5
5
  import { generateSafeFilename } from '../utils/filename-generator.js';
6
- import { wrapAsync } from './async-handler.js';
7
6
  const HASH_PATTERN = /^[a-f0-9.]+$/i;
8
7
  function validateNamespace(namespace) {
9
8
  return namespace === 'markdown';
@@ -11,8 +10,13 @@ function validateNamespace(namespace) {
11
10
  function validateHash(hash) {
12
11
  return HASH_PATTERN.test(hash) && hash.length >= 8 && hash.length <= 64;
13
12
  }
13
+ function isSingleParam(value) {
14
+ return typeof value === 'string';
15
+ }
14
16
  function parseDownloadParams(req) {
15
17
  const { namespace, hash } = req.params;
18
+ if (!isSingleParam(namespace) || !isSingleParam(hash))
19
+ return null;
16
20
  if (!namespace || !hash)
17
21
  return null;
18
22
  if (!validateNamespace(namespace))
@@ -96,5 +100,5 @@ function handleDownload(req, res) {
96
100
  sendDownloadPayload(res, payload);
97
101
  }
98
102
  export function registerDownloadRoutes(app) {
99
- app.get('/mcp/downloads/:namespace/:hash', wrapAsync(handleDownload));
103
+ app.get('/mcp/downloads/:namespace/:hash', handleDownload);
100
104
  }
@@ -0,0 +1,2 @@
1
+ import type { NextFunction, Request, Response } from 'express';
2
+ export declare function errorHandler(err: Error, req: Request, res: Response, next: NextFunction): void;
@@ -0,0 +1,55 @@
1
+ import { FetchError } from '../errors/app-error.js';
2
+ import { logError } from '../services/logger.js';
3
+ function getStatusCode(fetchError) {
4
+ return fetchError ? fetchError.statusCode : 500;
5
+ }
6
+ function getErrorCode(fetchError) {
7
+ return fetchError ? fetchError.code : 'INTERNAL_ERROR';
8
+ }
9
+ function getFetchErrorMessage(fetchError) {
10
+ return fetchError ? fetchError.message : 'Internal Server Error';
11
+ }
12
+ function getErrorDetails(fetchError) {
13
+ if (fetchError && Object.keys(fetchError.details).length > 0) {
14
+ return fetchError.details;
15
+ }
16
+ return undefined;
17
+ }
18
+ function resolveRetryAfter(fetchError) {
19
+ if (fetchError?.statusCode !== 429)
20
+ return undefined;
21
+ const { retryAfter } = fetchError.details;
22
+ return isRetryAfterValue(retryAfter) ? String(retryAfter) : undefined;
23
+ }
24
+ function isRetryAfterValue(value) {
25
+ return typeof value === 'number' || typeof value === 'string';
26
+ }
27
+ function setRetryAfterHeader(res, fetchError) {
28
+ const retryAfter = resolveRetryAfter(fetchError);
29
+ if (retryAfter === undefined)
30
+ return;
31
+ res.set('Retry-After', retryAfter);
32
+ }
33
+ function buildErrorResponse(fetchError) {
34
+ const details = getErrorDetails(fetchError);
35
+ const response = {
36
+ error: {
37
+ message: getFetchErrorMessage(fetchError),
38
+ code: getErrorCode(fetchError),
39
+ statusCode: getStatusCode(fetchError),
40
+ ...(details && { details }),
41
+ },
42
+ };
43
+ return response;
44
+ }
45
+ export function errorHandler(err, req, res, next) {
46
+ if (res.headersSent) {
47
+ next(err);
48
+ return;
49
+ }
50
+ const fetchError = err instanceof FetchError ? err : null;
51
+ const statusCode = getStatusCode(fetchError);
52
+ logError(`HTTP ${statusCode}: ${err.message} - ${req.method} ${req.path}`, err);
53
+ setRetryAfterHeader(res, fetchError);
54
+ res.status(statusCode).json(buildErrorResponse(fetchError));
55
+ }
@@ -82,8 +82,8 @@ function resolveSessionTransport(sessionId, options, res) {
82
82
  }
83
83
  const MCP_PROTOCOL_VERSION_HEADER = 'mcp-protocol-version';
84
84
  const MCP_PROTOCOL_VERSIONS = {
85
- defaultVersion: '2025-03-26',
86
- supported: new Set(['2025-03-26', '2025-11-25']),
85
+ defaultVersion: '2025-11-25',
86
+ supported: new Set(['2025-11-25']),
87
87
  };
88
88
  function getHeaderValue(req, headerNameLower) {
89
89
  const value = req.headers[headerNameLower];
@@ -1,6 +1,6 @@
1
1
  import type { Request, Response } from 'express';
2
2
  import { StreamableHTTPServerTransport } from '@modelcontextprotocol/sdk/server/streamableHttp.js';
3
- import type { SessionEntry } from '../config/types/runtime.js';
3
+ import type { McpRequestBody, SessionEntry } from '../config/types/runtime.js';
4
4
  export interface SessionStore {
5
5
  get: (sessionId: string) => SessionEntry | undefined;
6
6
  touch: (sessionId: string) => void;
@@ -15,7 +15,7 @@ export interface McpSessionOptions {
15
15
  readonly sessionStore: SessionStore;
16
16
  readonly maxSessions: number;
17
17
  }
18
- export declare function sendJsonRpcError(res: Response, code: number, message: string, status?: number): void;
18
+ export declare function sendJsonRpcError(res: Response, code: number, message: string, status?: number, id?: string | number | null): void;
19
19
  export declare function getSessionId(req: Request): string | undefined;
20
20
  export declare function createSessionStore(sessionTtlMs: number): SessionStore;
21
21
  export declare function reserveSessionSlot(store: SessionStore, maxSessions: number): boolean;
@@ -33,9 +33,7 @@ export declare function ensureSessionCapacity({ store, maxSessions, res, evictOl
33
33
  }): boolean;
34
34
  export declare function resolveTransportForPost({ res, body, sessionId, options, }: {
35
35
  res: Response;
36
- body: {
37
- method: string;
38
- };
36
+ body: Pick<McpRequestBody, 'method' | 'id'>;
39
37
  sessionId: string | undefined;
40
38
  options: McpSessionOptions;
41
39
  }): Promise<StreamableHTTPServerTransport | null>;
@@ -6,14 +6,14 @@ import { config } from '../config/index.js';
6
6
  import { logError, logInfo, logWarn } from '../services/logger.js';
7
7
  import { getErrorMessage } from '../utils/error-details.js';
8
8
  import { createMcpServer } from '../server.js';
9
- export function sendJsonRpcError(res, code, message, status = 400) {
9
+ export function sendJsonRpcError(res, code, message, status = 400, id = null) {
10
10
  res.status(status).json({
11
11
  jsonrpc: '2.0',
12
12
  error: {
13
13
  code,
14
14
  message,
15
15
  },
16
- id: null,
16
+ id,
17
17
  });
18
18
  }
19
19
  export function getSessionId(req) {
@@ -128,10 +128,10 @@ export function ensureSessionCapacity({ store, maxSessions, res, evictOldest, })
128
128
  return false;
129
129
  }
130
130
  function respondServerBusy(res) {
131
- sendJsonRpcError(res, -32000, 'Server busy: maximum sessions reached', 503);
131
+ sendJsonRpcError(res, -32000, 'Server busy: maximum sessions reached', 503, null);
132
132
  }
133
- function respondBadRequest(res) {
134
- sendJsonRpcError(res, -32000, 'Bad Request: Missing session ID or not an initialize request', 400);
133
+ function respondBadRequest(res, id) {
134
+ sendJsonRpcError(res, -32000, 'Bad Request: Missing session ID or not an initialize request', 400, id);
135
135
  }
136
136
  function createTimeoutController() {
137
137
  let initTimeout = null;
@@ -286,7 +286,7 @@ function resolveSessionId({ transport, res, tracker, clearInitTimeout, }) {
286
286
  if (typeof sessionId !== 'string') {
287
287
  clearInitTimeout();
288
288
  tracker.releaseSlot();
289
- respondBadRequest(res);
289
+ respondBadRequest(res, null);
290
290
  return null;
291
291
  }
292
292
  return sessionId;
@@ -343,11 +343,11 @@ export async function resolveTransportForPost({ res, body, sessionId, options, }
343
343
  return existingSession.transport;
344
344
  }
345
345
  // Client supplied a session id but it doesn't exist; Streamable HTTP: invalid session IDs => 404.
346
- sendJsonRpcError(res, -32600, 'Session not found', 404);
346
+ sendJsonRpcError(res, -32600, 'Session not found', 404, body.id ?? null);
347
347
  return null;
348
348
  }
349
349
  if (!isInitializeRequest(body)) {
350
- respondBadRequest(res);
350
+ respondBadRequest(res, body.id ?? null);
351
351
  return null;
352
352
  }
353
353
  evictExpiredSessionsWithClose(options.sessionStore);
@@ -0,0 +1,9 @@
1
+ export interface HttpServerTuningTarget {
2
+ headersTimeout?: number;
3
+ requestTimeout?: number;
4
+ keepAliveTimeout?: number;
5
+ closeIdleConnections?: () => void;
6
+ closeAllConnections?: () => void;
7
+ }
8
+ export declare function applyHttpServerTuning(server: HttpServerTuningTarget): void;
9
+ export declare function drainConnectionsOnShutdown(server: HttpServerTuningTarget): void;
@@ -0,0 +1,45 @@
1
+ import { config } from '../config/index.js';
2
+ import { logDebug } from '../services/logger.js';
3
+ export function applyHttpServerTuning(server) {
4
+ const { headersTimeoutMs, requestTimeoutMs, keepAliveTimeoutMs } = config.server.http;
5
+ if (headersTimeoutMs !== undefined) {
6
+ server.headersTimeout = headersTimeoutMs;
7
+ }
8
+ if (requestTimeoutMs !== undefined) {
9
+ server.requestTimeout = requestTimeoutMs;
10
+ }
11
+ if (keepAliveTimeoutMs !== undefined) {
12
+ server.keepAliveTimeout = keepAliveTimeoutMs;
13
+ }
14
+ if (headersTimeoutMs !== undefined ||
15
+ requestTimeoutMs !== undefined ||
16
+ keepAliveTimeoutMs !== undefined) {
17
+ logDebug('Applied HTTP server tuning', {
18
+ headersTimeoutMs,
19
+ requestTimeoutMs,
20
+ keepAliveTimeoutMs,
21
+ });
22
+ }
23
+ }
24
+ export function drainConnectionsOnShutdown(server) {
25
+ const { shutdownCloseAllConnections, shutdownCloseIdleConnections } = config.server.http;
26
+ if (shutdownCloseAllConnections) {
27
+ if (typeof server.closeAllConnections === 'function') {
28
+ server.closeAllConnections();
29
+ logDebug('Closed all HTTP connections during shutdown');
30
+ }
31
+ else {
32
+ logDebug('HTTP server does not support closeAllConnections()');
33
+ }
34
+ return;
35
+ }
36
+ if (shutdownCloseIdleConnections) {
37
+ if (typeof server.closeIdleConnections === 'function') {
38
+ server.closeIdleConnections();
39
+ logDebug('Closed idle HTTP connections during shutdown');
40
+ }
41
+ else {
42
+ logDebug('HTTP server does not support closeIdleConnections()');
43
+ }
44
+ }
45
+ }
@@ -1,13 +1,3 @@
1
- import type { Express, NextFunction, Request, RequestHandler, Response } from 'express';
2
- export declare function createCorsMiddleware(): (req: Request, res: Response, next: NextFunction) => void;
3
- export declare function attachBaseMiddleware(options: {
4
- app: Express;
5
- jsonParser: RequestHandler;
6
- rateLimitMiddleware: RequestHandler;
7
- corsMiddleware: RequestHandler;
8
- }): void;
9
- export declare function registerDownloadRoutes(app: Express): void;
10
- export declare function errorHandler(err: Error, req: Request, res: Response, next: NextFunction): void;
11
1
  export declare function startHttpServer(): Promise<{
12
2
  shutdown: (signal: string) => Promise<void>;
13
3
  }>;