@j0hanz/superfetch 1.2.4 → 2.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (116) hide show
  1. package/README.md +116 -152
  2. package/dist/config/auth-config.d.ts +16 -0
  3. package/dist/config/auth-config.js +53 -0
  4. package/dist/config/constants.d.ts +11 -13
  5. package/dist/config/constants.js +1 -3
  6. package/dist/config/env-parsers.d.ts +7 -0
  7. package/dist/config/env-parsers.js +84 -0
  8. package/dist/config/formatting.d.ts +2 -2
  9. package/dist/config/index.d.ts +47 -53
  10. package/dist/config/index.js +25 -59
  11. package/dist/config/types/content.d.ts +1 -49
  12. package/dist/config/types/runtime.d.ts +8 -16
  13. package/dist/config/types/tools.d.ts +2 -28
  14. package/dist/http/accept-policy.d.ts +3 -0
  15. package/dist/http/accept-policy.js +45 -0
  16. package/dist/http/async-handler.d.ts +2 -0
  17. package/dist/http/async-handler.js +5 -0
  18. package/dist/http/auth-introspection.d.ts +2 -0
  19. package/dist/http/auth-introspection.js +141 -0
  20. package/dist/http/auth-static.d.ts +2 -0
  21. package/dist/http/auth-static.js +23 -0
  22. package/dist/http/auth.d.ts +3 -2
  23. package/dist/http/auth.js +98 -26
  24. package/dist/http/cors.d.ts +6 -6
  25. package/dist/http/cors.js +7 -42
  26. package/dist/http/download-routes.d.ts +0 -12
  27. package/dist/http/download-routes.js +21 -58
  28. package/dist/http/jsonrpc-http.d.ts +2 -0
  29. package/dist/http/jsonrpc-http.js +10 -0
  30. package/dist/http/mcp-routes.d.ts +0 -1
  31. package/dist/http/mcp-routes.js +43 -30
  32. package/dist/http/mcp-session-helpers.d.ts +0 -1
  33. package/dist/http/mcp-session-helpers.js +1 -1
  34. package/dist/http/mcp-session-transport.d.ts +7 -0
  35. package/dist/http/mcp-session-transport.js +57 -0
  36. package/dist/http/mcp-session.js +60 -73
  37. package/dist/http/mcp-validation.d.ts +1 -0
  38. package/dist/http/mcp-validation.js +11 -10
  39. package/dist/http/protocol-policy.d.ts +2 -0
  40. package/dist/http/protocol-policy.js +31 -0
  41. package/dist/http/rate-limit.js +5 -2
  42. package/dist/http/server-config.d.ts +1 -0
  43. package/dist/http/server-config.js +40 -0
  44. package/dist/http/server-middleware.d.ts +2 -9
  45. package/dist/http/server-middleware.js +96 -43
  46. package/dist/http/server-shutdown.d.ts +4 -0
  47. package/dist/http/server-shutdown.js +43 -0
  48. package/dist/http/server.js +52 -64
  49. package/dist/http/session-cleanup.js +1 -1
  50. package/dist/middleware/error-handler.js +1 -3
  51. package/dist/resources/cached-content.js +50 -108
  52. package/dist/resources/index.js +0 -82
  53. package/dist/server.js +51 -30
  54. package/dist/services/cache-keys.d.ts +7 -0
  55. package/dist/services/cache-keys.js +57 -0
  56. package/dist/services/cache.d.ts +1 -7
  57. package/dist/services/cache.js +53 -119
  58. package/dist/services/context.d.ts +0 -1
  59. package/dist/services/context.js +0 -7
  60. package/dist/services/extractor.js +10 -82
  61. package/dist/services/fetcher/agents.d.ts +2 -2
  62. package/dist/services/fetcher/agents.js +34 -95
  63. package/dist/services/fetcher/dns-selection.d.ts +2 -0
  64. package/dist/services/fetcher/dns-selection.js +72 -0
  65. package/dist/services/fetcher/interceptors.d.ts +0 -22
  66. package/dist/services/fetcher/interceptors.js +30 -13
  67. package/dist/services/fetcher/redirects.js +4 -3
  68. package/dist/services/fetcher/response.js +66 -31
  69. package/dist/services/fetcher.d.ts +1 -3
  70. package/dist/services/fetcher.js +14 -33
  71. package/dist/services/fifo-queue.d.ts +8 -0
  72. package/dist/services/fifo-queue.js +25 -0
  73. package/dist/services/logger.js +2 -2
  74. package/dist/services/metadata-collector.d.ts +1 -9
  75. package/dist/services/metadata-collector.js +71 -2
  76. package/dist/services/transform-worker-pool.d.ts +4 -14
  77. package/dist/services/transform-worker-pool.js +177 -129
  78. package/dist/services/transform-worker-types.d.ts +32 -0
  79. package/dist/services/transform-worker-types.js +14 -0
  80. package/dist/tools/handlers/fetch-markdown.tool.d.ts +3 -4
  81. package/dist/tools/handlers/fetch-markdown.tool.js +20 -72
  82. package/dist/tools/handlers/fetch-single.shared.d.ts +1 -20
  83. package/dist/tools/handlers/fetch-single.shared.js +44 -87
  84. package/dist/tools/handlers/fetch-url.tool.d.ts +1 -1
  85. package/dist/tools/handlers/fetch-url.tool.js +46 -123
  86. package/dist/tools/index.js +21 -40
  87. package/dist/tools/schemas.d.ts +1 -51
  88. package/dist/tools/schemas.js +2 -108
  89. package/dist/tools/utils/cached-markdown.d.ts +5 -0
  90. package/dist/tools/utils/cached-markdown.js +46 -0
  91. package/dist/tools/utils/content-shaping.d.ts +4 -0
  92. package/dist/tools/utils/content-shaping.js +52 -0
  93. package/dist/tools/utils/content-transform.d.ts +2 -17
  94. package/dist/tools/utils/content-transform.js +120 -114
  95. package/dist/tools/utils/fetch-pipeline.d.ts +0 -8
  96. package/dist/tools/utils/fetch-pipeline.js +65 -62
  97. package/dist/tools/utils/inline-content.d.ts +1 -2
  98. package/dist/tools/utils/inline-content.js +4 -7
  99. package/dist/transformers/markdown.transformer.js +109 -34
  100. package/dist/utils/cached-payload.d.ts +7 -0
  101. package/dist/utils/cached-payload.js +36 -0
  102. package/dist/utils/error-utils.js +1 -1
  103. package/dist/utils/filename-generator.js +21 -10
  104. package/dist/utils/guards.d.ts +1 -0
  105. package/dist/utils/guards.js +3 -0
  106. package/dist/utils/header-normalizer.d.ts +0 -3
  107. package/dist/utils/header-normalizer.js +3 -3
  108. package/dist/utils/tool-error-handler.d.ts +2 -2
  109. package/dist/utils/tool-error-handler.js +11 -38
  110. package/dist/utils/url-transformer.d.ts +7 -0
  111. package/dist/utils/url-transformer.js +147 -0
  112. package/dist/utils/url-validator.d.ts +1 -2
  113. package/dist/utils/url-validator.js +20 -93
  114. package/dist/workers/content-transform.worker.d.ts +1 -0
  115. package/dist/workers/content-transform.worker.js +40 -0
  116. package/package.json +13 -16
@@ -1,31 +1,16 @@
1
- import { styleText } from 'node:util';
2
1
  import { config, enableHttpMode } from '../config/index.js';
3
- import { destroyAgents } from '../services/fetcher.js';
4
- import { logError, logInfo, logWarn } from '../services/logger.js';
5
- import { destroyTransformWorkers } from '../services/transform-worker-pool.js';
2
+ import { logError, logInfo } from '../services/logger.js';
6
3
  import { errorHandler } from '../middleware/error-handler.js';
7
- import { getErrorMessage } from '../utils/error-utils.js';
8
- import { createAuthMiddleware } from './auth.js';
4
+ import { createAuthMetadataRouter, createAuthMiddleware } from './auth.js';
9
5
  import { createCorsMiddleware } from './cors.js';
10
6
  import { registerDownloadRoutes } from './download-routes.js';
11
7
  import { registerMcpRoutes } from './mcp-routes.js';
12
8
  import { createRateLimitMiddleware } from './rate-limit.js';
13
- import { attachBaseMiddleware, buildCorsOptions } from './server-middleware.js';
9
+ import { assertHttpConfiguration } from './server-config.js';
10
+ import { attachBaseMiddleware } from './server-middleware.js';
11
+ import { createShutdownHandler, registerSignalHandlers, } from './server-shutdown.js';
14
12
  import { startSessionCleanupLoop } from './session-cleanup.js';
15
13
  import { createSessionStore } from './sessions.js';
16
- function isLoopbackHost(host) {
17
- return host === '127.0.0.1' || host === '::1' || host === 'localhost';
18
- }
19
- function assertHttpConfiguration() {
20
- if (!config.security.allowRemote && !isLoopbackHost(config.server.host)) {
21
- logError('Refusing to bind to non-loopback host without ALLOW_REMOTE=true', { host: config.server.host });
22
- process.exit(1);
23
- }
24
- if (!config.security.apiKey) {
25
- logError('API_KEY is required for HTTP mode; refusing to start');
26
- process.exit(1);
27
- }
28
- }
29
14
  function startListening(app) {
30
15
  return app
31
16
  .listen(config.server.port, config.server.host, () => {
@@ -33,63 +18,69 @@ function startListening(app) {
33
18
  host: config.server.host,
34
19
  port: config.server.port,
35
20
  });
36
- process.stdout.write(`${styleText('green', '✓')} superFetch MCP server running at ${styleText('cyan', `http://${config.server.host}:${config.server.port}`)}\n`);
37
- process.stdout.write(` Health check: ${styleText('dim', `http://${config.server.host}:${config.server.port}/health`)}\n`);
38
- process.stdout.write(` MCP endpoint: ${styleText('dim', `http://${config.server.host}:${config.server.port}/mcp`)}\n`);
39
- process.stdout.write(`\n${styleText('dim', 'Run with --stdio flag for direct stdio integration')}\n`);
21
+ const baseUrl = `http://${config.server.host}:${config.server.port}`;
22
+ logInfo(`superFetch MCP server running at ${baseUrl} (health: ${baseUrl}/health, mcp: ${baseUrl}/mcp)`);
23
+ logInfo('Run with --stdio flag for direct stdio integration');
40
24
  })
41
25
  .on('error', (err) => {
42
26
  logError('Failed to start server', err);
43
27
  process.exit(1);
44
28
  });
45
29
  }
46
- function createShutdownHandler(server, sessionStore, sessionCleanupController, stopRateLimitCleanup) {
47
- return async (signal) => {
48
- process.stdout.write(`\n${styleText('yellow', signal)} received, shutting down gracefully...\n`);
49
- stopRateLimitCleanup();
50
- sessionCleanupController.abort();
51
- const sessions = sessionStore.clear();
52
- await Promise.allSettled(sessions.map((session) => session.transport.close().catch((error) => {
53
- logWarn('Failed to close session during shutdown', {
54
- error: getErrorMessage(error),
55
- });
56
- })));
57
- destroyAgents();
58
- destroyTransformWorkers();
59
- server.close(() => {
60
- logInfo('HTTP server closed');
61
- process.exit(0);
62
- });
63
- setTimeout(() => {
64
- logError('Forced shutdown after timeout');
65
- process.exit(1);
66
- }, 10000).unref();
30
+ function buildMiddleware() {
31
+ const { middleware: rateLimitMiddleware, stop: stopRateLimitCleanup } = createRateLimitMiddleware(config.rateLimit);
32
+ const authMiddleware = createAuthMiddleware();
33
+ // No CORS - MCP clients don't run in browsers
34
+ const corsMiddleware = createCorsMiddleware();
35
+ return {
36
+ rateLimitMiddleware,
37
+ stopRateLimitCleanup,
38
+ authMiddleware,
39
+ corsMiddleware,
67
40
  };
68
41
  }
69
- function registerSignalHandlers(shutdown) {
70
- process.on('SIGINT', () => {
71
- void shutdown('SIGINT');
72
- });
73
- process.on('SIGTERM', () => {
74
- void shutdown('SIGTERM');
75
- });
76
- }
77
- export async function startHttpServer() {
78
- enableHttpMode();
79
- const { app, jsonParser } = await createExpressApp();
80
- const corsOptions = buildCorsOptions();
81
- const { middleware: rateLimitMiddleware, stop: stopRateLimitCleanup } = createRateLimitMiddleware(config.rateLimit);
82
- const authMiddleware = createAuthMiddleware(config.security.apiKey ?? '');
83
- attachBaseMiddleware(app, jsonParser, rateLimitMiddleware, authMiddleware, createCorsMiddleware(corsOptions));
84
- assertHttpConfiguration();
42
+ function createSessionInfrastructure() {
85
43
  const sessionStore = createSessionStore(config.server.sessionTtlMs);
86
44
  const sessionCleanupController = startSessionCleanupLoop(sessionStore, config.server.sessionTtlMs);
45
+ return { sessionStore, sessionCleanupController };
46
+ }
47
+ function registerHttpRoutes(app, sessionStore, authMiddleware) {
48
+ app.use('/mcp', authMiddleware);
49
+ app.use('/mcp/downloads', authMiddleware);
87
50
  registerMcpRoutes(app, {
88
51
  sessionStore,
89
52
  maxSessions: config.server.maxSessions,
90
53
  });
91
54
  registerDownloadRoutes(app);
92
55
  app.use(errorHandler);
56
+ }
57
+ function attachAuthMetadata(app) {
58
+ const authMetadataRouter = createAuthMetadataRouter();
59
+ if (authMetadataRouter) {
60
+ app.use(authMetadataRouter);
61
+ }
62
+ }
63
+ async function buildServerContext() {
64
+ const { app, authMiddleware, stopRateLimitCleanup } = await createAppWithMiddleware();
65
+ const { sessionStore, sessionCleanupController } = attachSessionRoutes(app, authMiddleware);
66
+ return { app, sessionStore, sessionCleanupController, stopRateLimitCleanup };
67
+ }
68
+ async function createAppWithMiddleware() {
69
+ const { app, jsonParser } = await createExpressApp();
70
+ const { rateLimitMiddleware, stopRateLimitCleanup, authMiddleware, corsMiddleware, } = buildMiddleware();
71
+ attachBaseMiddleware(app, jsonParser, rateLimitMiddleware, corsMiddleware);
72
+ attachAuthMetadata(app);
73
+ assertHttpConfiguration();
74
+ return { app, authMiddleware, stopRateLimitCleanup };
75
+ }
76
+ function attachSessionRoutes(app, authMiddleware) {
77
+ const { sessionStore, sessionCleanupController } = createSessionInfrastructure();
78
+ registerHttpRoutes(app, sessionStore, authMiddleware);
79
+ return { sessionStore, sessionCleanupController };
80
+ }
81
+ export async function startHttpServer() {
82
+ enableHttpMode();
83
+ const { app, sessionStore, sessionCleanupController, stopRateLimitCleanup } = await buildServerContext();
93
84
  const server = startListening(app);
94
85
  const shutdown = createShutdownHandler(server, sessionStore, sessionCleanupController, stopRateLimitCleanup);
95
86
  registerSignalHandlers(shutdown);
@@ -98,9 +89,6 @@ export async function startHttpServer() {
98
89
  async function createExpressApp() {
99
90
  const { default: express } = await import('express');
100
91
  const app = express();
101
- if (config.server.trustProxy) {
102
- app.set('trust proxy', true);
103
- }
104
92
  const jsonParser = express.json({ limit: '1mb' });
105
93
  return { app, jsonParser };
106
94
  }
@@ -1,6 +1,6 @@
1
1
  import { setInterval as setIntervalPromise } from 'node:timers/promises';
2
2
  import { logInfo, logWarn } from '../services/logger.js';
3
- import { evictExpiredSessions } from './mcp-routes.js';
3
+ import { evictExpiredSessions } from './mcp-session.js';
4
4
  export function startSessionCleanupLoop(store, sessionTtlMs) {
5
5
  const controller = new AbortController();
6
6
  void runSessionCleanupLoop(store, sessionTtlMs, controller.signal).catch(handleSessionCleanupError);
@@ -31,9 +31,7 @@ function buildErrorResponse(err) {
31
31
  ...(details && { details }),
32
32
  },
33
33
  };
34
- if (process.env.NODE_ENV === 'development' && err.stack) {
35
- response.error.stack = err.stack;
36
- }
34
+ // Never expose stack traces in production
37
35
  return response;
38
36
  }
39
37
  function resolveRetryAfter(err) {
@@ -1,45 +1,33 @@
1
1
  import { ResourceTemplate } from '@modelcontextprotocol/sdk/server/mcp.js';
2
2
  import { ErrorCode, McpError } from '@modelcontextprotocol/sdk/types.js';
3
3
  import * as cache from '../services/cache.js';
4
+ import { parseCacheKey, toResourceUri } from '../services/cache-keys.js';
4
5
  import { logWarn } from '../services/logger.js';
6
+ import { parseCachedPayload, resolveCachedPayloadContent, } from '../utils/cached-payload.js';
5
7
  import { getErrorMessage } from '../utils/error-utils.js';
6
- const VALID_NAMESPACES = new Set(['url', 'markdown', 'links']);
8
+ import { isRecord } from '../utils/guards.js';
9
+ const CACHE_NAMESPACE = 'markdown';
7
10
  const HASH_PATTERN = /^[a-f0-9.]+$/i;
8
11
  function buildResourceEntry(namespace, urlHash) {
9
12
  return {
10
13
  name: `${namespace}:${urlHash}`,
11
14
  uri: `superfetch://cache/${namespace}/${urlHash}`,
12
15
  description: `Cached content entry for ${namespace}`,
13
- mimeType: resolveCacheMimeType(namespace),
16
+ mimeType: 'text/markdown',
14
17
  };
15
18
  }
16
19
  function listCachedResources() {
17
20
  const resources = cache
18
21
  .keys()
19
22
  .map((key) => {
20
- const parts = cache.parseCacheKey(key);
21
- return parts ? buildResourceEntry(parts.namespace, parts.urlHash) : null;
23
+ const parts = parseCacheKey(key);
24
+ if (parts?.namespace !== CACHE_NAMESPACE)
25
+ return null;
26
+ return buildResourceEntry(parts.namespace, parts.urlHash);
22
27
  })
23
28
  .filter((entry) => entry !== null);
24
29
  return { resources };
25
30
  }
26
- function buildCacheListPayload() {
27
- const cacheKeys = cache.keys();
28
- return {
29
- totalEntries: cacheKeys.length,
30
- entries: cacheKeys.map((key) => {
31
- const parts = cache.parseCacheKey(key);
32
- const namespace = parts?.namespace ?? 'unknown';
33
- const urlHash = parts?.urlHash ?? 'unknown';
34
- return {
35
- namespace,
36
- urlHash,
37
- resourceUri: `superfetch://cache/${namespace}/${urlHash}`,
38
- };
39
- }),
40
- timestamp: new Date().toISOString(),
41
- };
42
- }
43
31
  function notifyResourceUpdate(server, uri) {
44
32
  if (!server.isConnected())
45
33
  return;
@@ -52,88 +40,54 @@ function notifyResourceUpdate(server, uri) {
52
40
  }
53
41
  export function registerCachedContentResource(server) {
54
42
  registerCacheContentResource(server);
55
- registerCacheListResource(server);
56
43
  registerCacheUpdateSubscription(server);
57
44
  }
58
45
  function resolveCacheParams(params) {
59
- const namespace = resolveStringParam(params.namespace);
60
- const urlHash = resolveStringParam(params.urlHash);
61
- if (!namespace || !urlHash) {
62
- throw new McpError(ErrorCode.InvalidParams, 'Both namespace and urlHash parameters are required');
63
- }
46
+ const parsed = requireRecordParams(params);
47
+ const namespace = requireParamString(parsed, 'namespace');
48
+ const urlHash = requireParamString(parsed, 'urlHash');
64
49
  if (!isValidNamespace(namespace) || !isValidHash(urlHash)) {
65
50
  throw new McpError(ErrorCode.InvalidParams, 'Invalid cache resource parameters');
66
51
  }
67
52
  return { namespace, urlHash };
68
53
  }
69
- function buildCachedContentResponse(uri, cacheKey, namespace) {
70
- const cached = cache.get(cacheKey);
71
- if (!cached) {
72
- throw new McpError(ErrorCode.InvalidParams, `Content not found in cache for key: ${cacheKey}. Use superfetch://stats to see available cache entries.`);
73
- }
74
- if (namespace !== 'url' && namespace !== 'markdown') {
75
- return {
76
- contents: [
77
- {
78
- uri: uri.href,
79
- mimeType: resolveCacheMimeType(namespace),
80
- text: cached.content,
81
- },
82
- ],
83
- };
54
+ function requireRecordParams(value) {
55
+ if (!isRecord(value)) {
56
+ throw new McpError(ErrorCode.InvalidParams, 'Invalid cache resource parameters');
84
57
  }
85
- const payload = parseCachedPayload(cached.content);
86
- const resolvedContent = payload
87
- ? resolvePayloadContent(payload, namespace)
88
- : null;
89
- if (!resolvedContent) {
90
- throw new McpError(ErrorCode.InternalError, `Cached content is missing for namespace ${namespace}`);
58
+ return value;
59
+ }
60
+ function requireParamString(params, key) {
61
+ const raw = params[key];
62
+ const resolved = resolveStringParam(raw);
63
+ if (!resolved) {
64
+ throw new McpError(ErrorCode.InvalidParams, 'Both namespace and urlHash parameters are required');
91
65
  }
92
- return {
93
- contents: [
94
- {
95
- uri: uri.href,
96
- mimeType: resolveCacheMimeType(namespace),
97
- text: resolvedContent,
98
- },
99
- ],
100
- };
66
+ return resolved;
67
+ }
68
+ function buildCachedContentResponse(uri, cacheKey) {
69
+ const cached = requireCacheEntry(cacheKey);
70
+ return buildMarkdownContentResponse(uri, cached.content);
101
71
  }
102
72
  function registerCacheContentResource(server) {
103
73
  server.registerResource('cached-content', new ResourceTemplate('superfetch://cache/{namespace}/{urlHash}', {
104
74
  list: listCachedResources,
105
75
  }), {
106
76
  title: 'Cached Content',
107
- description: 'Access previously fetched web content from cache. Namespace: url, links, markdown. UrlHash: SHA-256 hash of the URL.',
77
+ description: 'Access previously fetched web content from cache. Namespace: markdown. UrlHash: SHA-256 hash of the URL.',
108
78
  mimeType: 'text/plain',
109
79
  }, (uri, params) => {
110
80
  const { namespace, urlHash } = resolveCacheParams(params);
111
81
  const cacheKey = `${namespace}:${urlHash}`;
112
- return buildCachedContentResponse(uri, cacheKey, namespace);
82
+ return buildCachedContentResponse(uri, cacheKey);
113
83
  });
114
84
  }
115
- function registerCacheListResource(server) {
116
- server.registerResource('cached-urls', 'superfetch://cache/list', {
117
- title: 'Cached URLs List',
118
- description: 'List all URLs currently in cache with their namespaces',
119
- mimeType: 'application/json',
120
- }, (uri) => ({
121
- contents: [
122
- {
123
- uri: uri.href,
124
- mimeType: 'application/json',
125
- text: JSON.stringify(buildCacheListPayload(), null, 2),
126
- },
127
- ],
128
- }));
129
- }
130
85
  function registerCacheUpdateSubscription(server) {
131
86
  const unsubscribe = cache.onCacheUpdate(({ cacheKey }) => {
132
- const resourceUri = cache.toResourceUri(cacheKey);
87
+ const resourceUri = toResourceUri(cacheKey);
133
88
  if (!resourceUri)
134
89
  return;
135
90
  notifyResourceUpdate(server, resourceUri);
136
- notifyResourceUpdate(server, 'superfetch://cache/list');
137
91
  if (server.isConnected()) {
138
92
  server.sendResourceListChanged();
139
93
  }
@@ -144,15 +98,8 @@ function registerCacheUpdateSubscription(server) {
144
98
  unsubscribe();
145
99
  };
146
100
  }
147
- function resolveCacheMimeType(namespace) {
148
- if (namespace === 'markdown')
149
- return 'text/markdown';
150
- if (namespace === 'url')
151
- return 'application/jsonl';
152
- return 'application/json';
153
- }
154
101
  function isValidNamespace(namespace) {
155
- return VALID_NAMESPACES.has(namespace);
102
+ return namespace === CACHE_NAMESPACE;
156
103
  }
157
104
  function isValidHash(hash) {
158
105
  return HASH_PATTERN.test(hash) && hash.length >= 8 && hash.length <= 64;
@@ -160,31 +107,26 @@ function isValidHash(hash) {
160
107
  function resolveStringParam(value) {
161
108
  return typeof value === 'string' ? value : null;
162
109
  }
163
- function parseCachedPayload(raw) {
164
- try {
165
- const parsed = JSON.parse(raw);
166
- return isCachedPayload(parsed) ? parsed : null;
167
- }
168
- catch {
169
- return null;
110
+ function requireCacheEntry(cacheKey) {
111
+ const cached = cache.get(cacheKey);
112
+ if (!cached) {
113
+ throw new McpError(-32002, `Content not found in cache for key: ${cacheKey}`);
170
114
  }
115
+ return cached;
171
116
  }
172
- function isCachedPayload(value) {
173
- if (!value || typeof value !== 'object')
174
- return false;
175
- const record = value;
176
- return ((record.content === undefined || typeof record.content === 'string') &&
177
- (record.markdown === undefined || typeof record.markdown === 'string'));
178
- }
179
- function resolvePayloadContent(payload, namespace) {
180
- if (namespace === 'markdown') {
181
- if (typeof payload.markdown === 'string') {
182
- return payload.markdown;
183
- }
184
- if (typeof payload.content === 'string') {
185
- return payload.content;
186
- }
187
- return null;
117
+ function buildMarkdownContentResponse(uri, content) {
118
+ const payload = parseCachedPayload(content);
119
+ const resolvedContent = payload ? resolveCachedPayloadContent(payload) : null;
120
+ if (!resolvedContent) {
121
+ throw new McpError(ErrorCode.InternalError, 'Cached markdown content is missing');
188
122
  }
189
- return typeof payload.content === 'string' ? payload.content : null;
123
+ return {
124
+ contents: [
125
+ {
126
+ uri: uri.href,
127
+ mimeType: 'text/markdown',
128
+ text: resolvedContent,
129
+ },
130
+ ],
131
+ };
190
132
  }
@@ -1,86 +1,4 @@
1
- import { config } from '../config/index.js';
2
- import * as cache from '../services/cache.js';
3
1
  import { registerCachedContentResource } from './cached-content.js';
4
- function registerJsonResource(server, definition) {
5
- server.registerResource(definition.name, definition.uri, {
6
- title: definition.title,
7
- description: definition.description,
8
- mimeType: 'application/json',
9
- }, (uri) => ({
10
- contents: [
11
- {
12
- uri: uri.href,
13
- mimeType: 'application/json',
14
- text: JSON.stringify(definition.buildPayload(), null, 2),
15
- },
16
- ],
17
- }));
18
- }
19
- function buildHealthPayload() {
20
- const memUsage = process.memoryUsage();
21
- const heapUsedMB = Math.round(memUsage.heapUsed / 1024 / 1024);
22
- const heapTotalMB = Math.round(memUsage.heapTotal / 1024 / 1024);
23
- return {
24
- status: 'healthy',
25
- uptime: process.uptime(),
26
- checks: {
27
- cache: config.cache.enabled,
28
- memory: {
29
- heapUsed: heapUsedMB,
30
- heapTotal: heapTotalMB,
31
- percentage: Math.round((heapUsedMB / heapTotalMB) * 100),
32
- healthy: heapUsedMB < 400,
33
- },
34
- },
35
- timestamp: new Date().toISOString(),
36
- };
37
- }
38
- function buildStatsPayload() {
39
- return {
40
- server: {
41
- name: config.server.name,
42
- version: config.server.version,
43
- uptime: process.uptime(),
44
- nodeVersion: process.version,
45
- memoryUsage: process.memoryUsage(),
46
- },
47
- cache: {
48
- enabled: config.cache.enabled,
49
- ttl: config.cache.ttl,
50
- maxKeys: config.cache.maxKeys,
51
- totalKeys: cache.keys().length,
52
- },
53
- config: {
54
- fetcher: {
55
- timeout: config.fetcher.timeout,
56
- maxRedirects: config.fetcher.maxRedirects,
57
- },
58
- extraction: {
59
- extractMainContent: config.extraction.extractMainContent,
60
- includeMetadata: config.extraction.includeMetadata,
61
- },
62
- },
63
- };
64
- }
65
2
  export function registerResources(server) {
66
3
  registerCachedContentResource(server);
67
- const resources = [
68
- {
69
- name: 'health',
70
- uri: 'superfetch://health',
71
- title: 'Server Health',
72
- description: 'Real-time server health and dependency status',
73
- buildPayload: buildHealthPayload,
74
- },
75
- {
76
- name: 'stats',
77
- uri: 'superfetch://stats',
78
- title: 'Server Statistics',
79
- description: 'Fetch statistics and cache performance metrics',
80
- buildPayload: buildStatsPayload,
81
- },
82
- ];
83
- for (const resource of resources) {
84
- registerJsonResource(server, resource);
85
- }
86
4
  }
package/dist/server.js CHANGED
@@ -1,52 +1,66 @@
1
1
  import { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
2
2
  import { StdioServerTransport } from '@modelcontextprotocol/sdk/server/stdio.js';
3
3
  import { config } from './config/index.js';
4
- import { destroyAgents } from './services/fetcher.js';
4
+ import { destroyAgents } from './services/fetcher/agents.js';
5
5
  import { logError, logInfo } from './services/logger.js';
6
- import { destroyTransformWorkers } from './services/transform-worker-pool.js';
7
6
  import { registerTools } from './tools/index.js';
8
- import { registerResources } from './resources/index.js';
9
- export function createMcpServer() {
10
- const server = new McpServer({
7
+ import { registerCachedContentResource } from './resources/cached-content.js';
8
+ function createServerInfo() {
9
+ return {
11
10
  name: config.server.name,
12
11
  version: config.server.version,
13
- }, {
14
- capabilities: {
15
- tools: { listChanged: false },
16
- resources: { listChanged: true, subscribe: true },
17
- logging: {},
18
- },
19
- instructions: `superFetch MCP server v${config.server.version} - AI-optimized web content fetching with JSONL/Markdown output. Provides tools for fetching, parsing, and transforming web content into structured formats suitable for LLM consumption. Supports resource subscriptions for cache updates.`,
12
+ };
13
+ }
14
+ function createServerCapabilities() {
15
+ return {
16
+ tools: { listChanged: false },
17
+ resources: { listChanged: true, subscribe: true },
18
+ logging: {},
19
+ };
20
+ }
21
+ function createServerInstructions(serverVersion) {
22
+ return `superFetch MCP server |${serverVersion}| A high-performance web content fetching and processing server.`;
23
+ }
24
+ export function createMcpServer() {
25
+ const server = new McpServer(createServerInfo(), {
26
+ capabilities: createServerCapabilities(),
27
+ instructions: createServerInstructions(config.server.version),
20
28
  });
21
29
  registerTools(server);
22
- registerResources(server);
30
+ registerCachedContentResource(server);
23
31
  return server;
24
32
  }
25
- export async function startStdioServer() {
26
- const server = createMcpServer();
27
- const transport = new StdioServerTransport();
33
+ function attachServerErrorHandler(server) {
28
34
  server.server.onerror = (error) => {
29
35
  logError('[MCP Error]', error instanceof Error ? error : { error });
30
36
  };
31
- const handleShutdown = (signal) => {
32
- process.stderr.write(`\n${signal} received, shutting down superFetch MCP server...\n`);
33
- destroyAgents();
34
- destroyTransformWorkers();
35
- server
36
- .close()
37
- .catch((err) => {
38
- logError('Error during shutdown', err instanceof Error ? err : undefined);
39
- })
40
- .finally(() => {
41
- process.exit(0);
42
- });
37
+ }
38
+ function handleShutdownSignal(server, signal) {
39
+ process.stderr.write(`\n${signal} received, shutting down superFetch MCP server...\n`);
40
+ destroyAgents();
41
+ server
42
+ .close()
43
+ .catch((err) => {
44
+ logError('Error during shutdown', err instanceof Error ? err : undefined);
45
+ })
46
+ .finally(() => {
47
+ process.exit(0);
48
+ });
49
+ }
50
+ function createShutdownHandler(server) {
51
+ return (signal) => {
52
+ handleShutdownSignal(server, signal);
43
53
  };
54
+ }
55
+ function registerSignalHandlers(handler) {
44
56
  process.on('SIGINT', () => {
45
- handleShutdown('SIGINT');
57
+ handler('SIGINT');
46
58
  });
47
59
  process.on('SIGTERM', () => {
48
- handleShutdown('SIGTERM');
60
+ handler('SIGTERM');
49
61
  });
62
+ }
63
+ async function connectStdioServer(server, transport) {
50
64
  try {
51
65
  await server.connect(transport);
52
66
  logInfo('superFetch MCP server running on stdio');
@@ -56,3 +70,10 @@ export async function startStdioServer() {
56
70
  process.exit(1);
57
71
  }
58
72
  }
73
+ export async function startStdioServer() {
74
+ const server = createMcpServer();
75
+ const transport = new StdioServerTransport();
76
+ attachServerErrorHandler(server);
77
+ registerSignalHandlers(createShutdownHandler(server));
78
+ await connectStdioServer(server, transport);
79
+ }
@@ -0,0 +1,7 @@
1
+ export interface CacheKeyParts {
2
+ namespace: string;
3
+ urlHash: string;
4
+ }
5
+ export declare function createCacheKey(namespace: string, url: string, vary?: Record<string, unknown> | string): string | null;
6
+ export declare function parseCacheKey(cacheKey: string): CacheKeyParts | null;
7
+ export declare function toResourceUri(cacheKey: string): string | null;
@@ -0,0 +1,57 @@
1
+ import { CACHE_HASH } from '../config/constants.js';
2
+ import { sha256Hex } from '../utils/crypto.js';
3
+ import { isRecord } from '../utils/guards.js';
4
+ function stableStringify(value) {
5
+ if (!isRecord(value)) {
6
+ if (value === null || value === undefined) {
7
+ return '';
8
+ }
9
+ return JSON.stringify(value);
10
+ }
11
+ if (Array.isArray(value)) {
12
+ return `[${value.map((item) => stableStringify(item)).join(',')}]`;
13
+ }
14
+ const entries = Object.entries(value)
15
+ .filter(([, entryValue]) => entryValue !== undefined)
16
+ .sort(([a], [b]) => a.localeCompare(b))
17
+ .map(([key, entryValue]) => `${JSON.stringify(key)}:${stableStringify(entryValue)}`);
18
+ return `{${entries.join(',')}}`;
19
+ }
20
+ function createHashFragment(input, length) {
21
+ return sha256Hex(input).substring(0, length);
22
+ }
23
+ function buildCacheKey(namespace, urlHash, varyHash) {
24
+ return varyHash
25
+ ? `${namespace}:${urlHash}.${varyHash}`
26
+ : `${namespace}:${urlHash}`;
27
+ }
28
+ function getVaryHash(vary) {
29
+ if (!vary)
30
+ return undefined;
31
+ const varyString = typeof vary === 'string' ? vary : stableStringify(vary);
32
+ if (!varyString)
33
+ return undefined;
34
+ return createHashFragment(varyString, CACHE_HASH.VARY_HASH_LENGTH);
35
+ }
36
+ export function createCacheKey(namespace, url, vary) {
37
+ if (!namespace || !url)
38
+ return null;
39
+ const urlHash = createHashFragment(url, CACHE_HASH.URL_HASH_LENGTH);
40
+ const varyHash = getVaryHash(vary);
41
+ return buildCacheKey(namespace, urlHash, varyHash);
42
+ }
43
+ export function parseCacheKey(cacheKey) {
44
+ if (!cacheKey)
45
+ return null;
46
+ const [namespace, ...rest] = cacheKey.split(':');
47
+ const urlHash = rest.join(':');
48
+ if (!namespace || !urlHash)
49
+ return null;
50
+ return { namespace, urlHash };
51
+ }
52
+ export function toResourceUri(cacheKey) {
53
+ const parts = parseCacheKey(cacheKey);
54
+ if (!parts)
55
+ return null;
56
+ return `superfetch://cache/${parts.namespace}/${parts.urlHash}`;
57
+ }