@j0hanz/superfetch 1.2.4 → 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +116 -152
- package/dist/config/auth-config.d.ts +16 -0
- package/dist/config/auth-config.js +53 -0
- package/dist/config/constants.d.ts +11 -13
- package/dist/config/constants.js +1 -3
- package/dist/config/env-parsers.d.ts +7 -0
- package/dist/config/env-parsers.js +84 -0
- package/dist/config/formatting.d.ts +2 -2
- package/dist/config/index.d.ts +47 -53
- package/dist/config/index.js +25 -59
- package/dist/config/types/content.d.ts +1 -49
- package/dist/config/types/runtime.d.ts +8 -16
- package/dist/config/types/tools.d.ts +2 -28
- package/dist/http/accept-policy.d.ts +3 -0
- package/dist/http/accept-policy.js +45 -0
- package/dist/http/async-handler.d.ts +2 -0
- package/dist/http/async-handler.js +5 -0
- package/dist/http/auth-introspection.d.ts +2 -0
- package/dist/http/auth-introspection.js +141 -0
- package/dist/http/auth-static.d.ts +2 -0
- package/dist/http/auth-static.js +23 -0
- package/dist/http/auth.d.ts +3 -2
- package/dist/http/auth.js +98 -26
- package/dist/http/cors.d.ts +6 -6
- package/dist/http/cors.js +7 -42
- package/dist/http/download-routes.d.ts +0 -12
- package/dist/http/download-routes.js +21 -58
- package/dist/http/jsonrpc-http.d.ts +2 -0
- package/dist/http/jsonrpc-http.js +10 -0
- package/dist/http/mcp-routes.d.ts +0 -1
- package/dist/http/mcp-routes.js +43 -30
- package/dist/http/mcp-session-helpers.d.ts +0 -1
- package/dist/http/mcp-session-helpers.js +1 -1
- package/dist/http/mcp-session-transport.d.ts +7 -0
- package/dist/http/mcp-session-transport.js +57 -0
- package/dist/http/mcp-session.js +60 -73
- package/dist/http/mcp-validation.d.ts +1 -0
- package/dist/http/mcp-validation.js +11 -10
- package/dist/http/protocol-policy.d.ts +2 -0
- package/dist/http/protocol-policy.js +31 -0
- package/dist/http/rate-limit.js +5 -2
- package/dist/http/server-config.d.ts +1 -0
- package/dist/http/server-config.js +40 -0
- package/dist/http/server-middleware.d.ts +2 -9
- package/dist/http/server-middleware.js +96 -43
- package/dist/http/server-shutdown.d.ts +4 -0
- package/dist/http/server-shutdown.js +43 -0
- package/dist/http/server.js +52 -64
- package/dist/http/session-cleanup.js +1 -1
- package/dist/middleware/error-handler.js +1 -3
- package/dist/resources/cached-content.js +50 -108
- package/dist/resources/index.js +0 -82
- package/dist/server.js +51 -30
- package/dist/services/cache-keys.d.ts +7 -0
- package/dist/services/cache-keys.js +57 -0
- package/dist/services/cache.d.ts +1 -7
- package/dist/services/cache.js +53 -119
- package/dist/services/context.d.ts +0 -1
- package/dist/services/context.js +0 -7
- package/dist/services/extractor.js +10 -82
- package/dist/services/fetcher/agents.d.ts +2 -2
- package/dist/services/fetcher/agents.js +34 -95
- package/dist/services/fetcher/dns-selection.d.ts +2 -0
- package/dist/services/fetcher/dns-selection.js +72 -0
- package/dist/services/fetcher/interceptors.d.ts +0 -22
- package/dist/services/fetcher/interceptors.js +30 -13
- package/dist/services/fetcher/redirects.js +4 -3
- package/dist/services/fetcher/response.js +66 -31
- package/dist/services/fetcher.d.ts +1 -3
- package/dist/services/fetcher.js +14 -33
- package/dist/services/fifo-queue.d.ts +8 -0
- package/dist/services/fifo-queue.js +25 -0
- package/dist/services/logger.js +2 -2
- package/dist/services/metadata-collector.d.ts +1 -9
- package/dist/services/metadata-collector.js +71 -2
- package/dist/services/transform-worker-pool.d.ts +4 -14
- package/dist/services/transform-worker-pool.js +177 -129
- package/dist/services/transform-worker-types.d.ts +32 -0
- package/dist/services/transform-worker-types.js +14 -0
- package/dist/tools/handlers/fetch-markdown.tool.d.ts +3 -4
- package/dist/tools/handlers/fetch-markdown.tool.js +20 -72
- package/dist/tools/handlers/fetch-single.shared.d.ts +1 -20
- package/dist/tools/handlers/fetch-single.shared.js +44 -87
- package/dist/tools/handlers/fetch-url.tool.d.ts +1 -1
- package/dist/tools/handlers/fetch-url.tool.js +46 -123
- package/dist/tools/index.js +21 -40
- package/dist/tools/schemas.d.ts +1 -51
- package/dist/tools/schemas.js +2 -108
- package/dist/tools/utils/cached-markdown.d.ts +5 -0
- package/dist/tools/utils/cached-markdown.js +46 -0
- package/dist/tools/utils/content-shaping.d.ts +4 -0
- package/dist/tools/utils/content-shaping.js +52 -0
- package/dist/tools/utils/content-transform.d.ts +2 -17
- package/dist/tools/utils/content-transform.js +120 -114
- package/dist/tools/utils/fetch-pipeline.d.ts +0 -8
- package/dist/tools/utils/fetch-pipeline.js +65 -62
- package/dist/tools/utils/inline-content.d.ts +1 -2
- package/dist/tools/utils/inline-content.js +4 -7
- package/dist/transformers/markdown.transformer.js +109 -34
- package/dist/utils/cached-payload.d.ts +7 -0
- package/dist/utils/cached-payload.js +36 -0
- package/dist/utils/error-utils.js +1 -1
- package/dist/utils/filename-generator.js +21 -10
- package/dist/utils/guards.d.ts +1 -0
- package/dist/utils/guards.js +3 -0
- package/dist/utils/header-normalizer.d.ts +0 -3
- package/dist/utils/header-normalizer.js +3 -3
- package/dist/utils/tool-error-handler.d.ts +2 -2
- package/dist/utils/tool-error-handler.js +11 -38
- package/dist/utils/url-transformer.d.ts +7 -0
- package/dist/utils/url-transformer.js +147 -0
- package/dist/utils/url-validator.d.ts +1 -2
- package/dist/utils/url-validator.js +20 -93
- package/dist/workers/content-transform.worker.d.ts +1 -0
- package/dist/workers/content-transform.worker.js +40 -0
- package/package.json +13 -16
package/dist/http/server.js
CHANGED
|
@@ -1,31 +1,16 @@
|
|
|
1
|
-
import { styleText } from 'node:util';
|
|
2
1
|
import { config, enableHttpMode } from '../config/index.js';
|
|
3
|
-
import {
|
|
4
|
-
import { logError, logInfo, logWarn } from '../services/logger.js';
|
|
5
|
-
import { destroyTransformWorkers } from '../services/transform-worker-pool.js';
|
|
2
|
+
import { logError, logInfo } from '../services/logger.js';
|
|
6
3
|
import { errorHandler } from '../middleware/error-handler.js';
|
|
7
|
-
import {
|
|
8
|
-
import { createAuthMiddleware } from './auth.js';
|
|
4
|
+
import { createAuthMetadataRouter, createAuthMiddleware } from './auth.js';
|
|
9
5
|
import { createCorsMiddleware } from './cors.js';
|
|
10
6
|
import { registerDownloadRoutes } from './download-routes.js';
|
|
11
7
|
import { registerMcpRoutes } from './mcp-routes.js';
|
|
12
8
|
import { createRateLimitMiddleware } from './rate-limit.js';
|
|
13
|
-
import {
|
|
9
|
+
import { assertHttpConfiguration } from './server-config.js';
|
|
10
|
+
import { attachBaseMiddleware } from './server-middleware.js';
|
|
11
|
+
import { createShutdownHandler, registerSignalHandlers, } from './server-shutdown.js';
|
|
14
12
|
import { startSessionCleanupLoop } from './session-cleanup.js';
|
|
15
13
|
import { createSessionStore } from './sessions.js';
|
|
16
|
-
function isLoopbackHost(host) {
|
|
17
|
-
return host === '127.0.0.1' || host === '::1' || host === 'localhost';
|
|
18
|
-
}
|
|
19
|
-
function assertHttpConfiguration() {
|
|
20
|
-
if (!config.security.allowRemote && !isLoopbackHost(config.server.host)) {
|
|
21
|
-
logError('Refusing to bind to non-loopback host without ALLOW_REMOTE=true', { host: config.server.host });
|
|
22
|
-
process.exit(1);
|
|
23
|
-
}
|
|
24
|
-
if (!config.security.apiKey) {
|
|
25
|
-
logError('API_KEY is required for HTTP mode; refusing to start');
|
|
26
|
-
process.exit(1);
|
|
27
|
-
}
|
|
28
|
-
}
|
|
29
14
|
function startListening(app) {
|
|
30
15
|
return app
|
|
31
16
|
.listen(config.server.port, config.server.host, () => {
|
|
@@ -33,63 +18,69 @@ function startListening(app) {
|
|
|
33
18
|
host: config.server.host,
|
|
34
19
|
port: config.server.port,
|
|
35
20
|
});
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
process.stdout.write(`\n${styleText('dim', 'Run with --stdio flag for direct stdio integration')}\n`);
|
|
21
|
+
const baseUrl = `http://${config.server.host}:${config.server.port}`;
|
|
22
|
+
logInfo(`superFetch MCP server running at ${baseUrl} (health: ${baseUrl}/health, mcp: ${baseUrl}/mcp)`);
|
|
23
|
+
logInfo('Run with --stdio flag for direct stdio integration');
|
|
40
24
|
})
|
|
41
25
|
.on('error', (err) => {
|
|
42
26
|
logError('Failed to start server', err);
|
|
43
27
|
process.exit(1);
|
|
44
28
|
});
|
|
45
29
|
}
|
|
46
|
-
function
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
})));
|
|
57
|
-
destroyAgents();
|
|
58
|
-
destroyTransformWorkers();
|
|
59
|
-
server.close(() => {
|
|
60
|
-
logInfo('HTTP server closed');
|
|
61
|
-
process.exit(0);
|
|
62
|
-
});
|
|
63
|
-
setTimeout(() => {
|
|
64
|
-
logError('Forced shutdown after timeout');
|
|
65
|
-
process.exit(1);
|
|
66
|
-
}, 10000).unref();
|
|
30
|
+
function buildMiddleware() {
|
|
31
|
+
const { middleware: rateLimitMiddleware, stop: stopRateLimitCleanup } = createRateLimitMiddleware(config.rateLimit);
|
|
32
|
+
const authMiddleware = createAuthMiddleware();
|
|
33
|
+
// No CORS - MCP clients don't run in browsers
|
|
34
|
+
const corsMiddleware = createCorsMiddleware();
|
|
35
|
+
return {
|
|
36
|
+
rateLimitMiddleware,
|
|
37
|
+
stopRateLimitCleanup,
|
|
38
|
+
authMiddleware,
|
|
39
|
+
corsMiddleware,
|
|
67
40
|
};
|
|
68
41
|
}
|
|
69
|
-
function
|
|
70
|
-
process.on('SIGINT', () => {
|
|
71
|
-
void shutdown('SIGINT');
|
|
72
|
-
});
|
|
73
|
-
process.on('SIGTERM', () => {
|
|
74
|
-
void shutdown('SIGTERM');
|
|
75
|
-
});
|
|
76
|
-
}
|
|
77
|
-
export async function startHttpServer() {
|
|
78
|
-
enableHttpMode();
|
|
79
|
-
const { app, jsonParser } = await createExpressApp();
|
|
80
|
-
const corsOptions = buildCorsOptions();
|
|
81
|
-
const { middleware: rateLimitMiddleware, stop: stopRateLimitCleanup } = createRateLimitMiddleware(config.rateLimit);
|
|
82
|
-
const authMiddleware = createAuthMiddleware(config.security.apiKey ?? '');
|
|
83
|
-
attachBaseMiddleware(app, jsonParser, rateLimitMiddleware, authMiddleware, createCorsMiddleware(corsOptions));
|
|
84
|
-
assertHttpConfiguration();
|
|
42
|
+
function createSessionInfrastructure() {
|
|
85
43
|
const sessionStore = createSessionStore(config.server.sessionTtlMs);
|
|
86
44
|
const sessionCleanupController = startSessionCleanupLoop(sessionStore, config.server.sessionTtlMs);
|
|
45
|
+
return { sessionStore, sessionCleanupController };
|
|
46
|
+
}
|
|
47
|
+
function registerHttpRoutes(app, sessionStore, authMiddleware) {
|
|
48
|
+
app.use('/mcp', authMiddleware);
|
|
49
|
+
app.use('/mcp/downloads', authMiddleware);
|
|
87
50
|
registerMcpRoutes(app, {
|
|
88
51
|
sessionStore,
|
|
89
52
|
maxSessions: config.server.maxSessions,
|
|
90
53
|
});
|
|
91
54
|
registerDownloadRoutes(app);
|
|
92
55
|
app.use(errorHandler);
|
|
56
|
+
}
|
|
57
|
+
function attachAuthMetadata(app) {
|
|
58
|
+
const authMetadataRouter = createAuthMetadataRouter();
|
|
59
|
+
if (authMetadataRouter) {
|
|
60
|
+
app.use(authMetadataRouter);
|
|
61
|
+
}
|
|
62
|
+
}
|
|
63
|
+
async function buildServerContext() {
|
|
64
|
+
const { app, authMiddleware, stopRateLimitCleanup } = await createAppWithMiddleware();
|
|
65
|
+
const { sessionStore, sessionCleanupController } = attachSessionRoutes(app, authMiddleware);
|
|
66
|
+
return { app, sessionStore, sessionCleanupController, stopRateLimitCleanup };
|
|
67
|
+
}
|
|
68
|
+
async function createAppWithMiddleware() {
|
|
69
|
+
const { app, jsonParser } = await createExpressApp();
|
|
70
|
+
const { rateLimitMiddleware, stopRateLimitCleanup, authMiddleware, corsMiddleware, } = buildMiddleware();
|
|
71
|
+
attachBaseMiddleware(app, jsonParser, rateLimitMiddleware, corsMiddleware);
|
|
72
|
+
attachAuthMetadata(app);
|
|
73
|
+
assertHttpConfiguration();
|
|
74
|
+
return { app, authMiddleware, stopRateLimitCleanup };
|
|
75
|
+
}
|
|
76
|
+
function attachSessionRoutes(app, authMiddleware) {
|
|
77
|
+
const { sessionStore, sessionCleanupController } = createSessionInfrastructure();
|
|
78
|
+
registerHttpRoutes(app, sessionStore, authMiddleware);
|
|
79
|
+
return { sessionStore, sessionCleanupController };
|
|
80
|
+
}
|
|
81
|
+
export async function startHttpServer() {
|
|
82
|
+
enableHttpMode();
|
|
83
|
+
const { app, sessionStore, sessionCleanupController, stopRateLimitCleanup } = await buildServerContext();
|
|
93
84
|
const server = startListening(app);
|
|
94
85
|
const shutdown = createShutdownHandler(server, sessionStore, sessionCleanupController, stopRateLimitCleanup);
|
|
95
86
|
registerSignalHandlers(shutdown);
|
|
@@ -98,9 +89,6 @@ export async function startHttpServer() {
|
|
|
98
89
|
async function createExpressApp() {
|
|
99
90
|
const { default: express } = await import('express');
|
|
100
91
|
const app = express();
|
|
101
|
-
if (config.server.trustProxy) {
|
|
102
|
-
app.set('trust proxy', true);
|
|
103
|
-
}
|
|
104
92
|
const jsonParser = express.json({ limit: '1mb' });
|
|
105
93
|
return { app, jsonParser };
|
|
106
94
|
}
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { setInterval as setIntervalPromise } from 'node:timers/promises';
|
|
2
2
|
import { logInfo, logWarn } from '../services/logger.js';
|
|
3
|
-
import { evictExpiredSessions } from './mcp-
|
|
3
|
+
import { evictExpiredSessions } from './mcp-session.js';
|
|
4
4
|
export function startSessionCleanupLoop(store, sessionTtlMs) {
|
|
5
5
|
const controller = new AbortController();
|
|
6
6
|
void runSessionCleanupLoop(store, sessionTtlMs, controller.signal).catch(handleSessionCleanupError);
|
|
@@ -31,9 +31,7 @@ function buildErrorResponse(err) {
|
|
|
31
31
|
...(details && { details }),
|
|
32
32
|
},
|
|
33
33
|
};
|
|
34
|
-
|
|
35
|
-
response.error.stack = err.stack;
|
|
36
|
-
}
|
|
34
|
+
// Never expose stack traces in production
|
|
37
35
|
return response;
|
|
38
36
|
}
|
|
39
37
|
function resolveRetryAfter(err) {
|
|
@@ -1,45 +1,33 @@
|
|
|
1
1
|
import { ResourceTemplate } from '@modelcontextprotocol/sdk/server/mcp.js';
|
|
2
2
|
import { ErrorCode, McpError } from '@modelcontextprotocol/sdk/types.js';
|
|
3
3
|
import * as cache from '../services/cache.js';
|
|
4
|
+
import { parseCacheKey, toResourceUri } from '../services/cache-keys.js';
|
|
4
5
|
import { logWarn } from '../services/logger.js';
|
|
6
|
+
import { parseCachedPayload, resolveCachedPayloadContent, } from '../utils/cached-payload.js';
|
|
5
7
|
import { getErrorMessage } from '../utils/error-utils.js';
|
|
6
|
-
|
|
8
|
+
import { isRecord } from '../utils/guards.js';
|
|
9
|
+
const CACHE_NAMESPACE = 'markdown';
|
|
7
10
|
const HASH_PATTERN = /^[a-f0-9.]+$/i;
|
|
8
11
|
function buildResourceEntry(namespace, urlHash) {
|
|
9
12
|
return {
|
|
10
13
|
name: `${namespace}:${urlHash}`,
|
|
11
14
|
uri: `superfetch://cache/${namespace}/${urlHash}`,
|
|
12
15
|
description: `Cached content entry for ${namespace}`,
|
|
13
|
-
mimeType:
|
|
16
|
+
mimeType: 'text/markdown',
|
|
14
17
|
};
|
|
15
18
|
}
|
|
16
19
|
function listCachedResources() {
|
|
17
20
|
const resources = cache
|
|
18
21
|
.keys()
|
|
19
22
|
.map((key) => {
|
|
20
|
-
const parts =
|
|
21
|
-
|
|
23
|
+
const parts = parseCacheKey(key);
|
|
24
|
+
if (parts?.namespace !== CACHE_NAMESPACE)
|
|
25
|
+
return null;
|
|
26
|
+
return buildResourceEntry(parts.namespace, parts.urlHash);
|
|
22
27
|
})
|
|
23
28
|
.filter((entry) => entry !== null);
|
|
24
29
|
return { resources };
|
|
25
30
|
}
|
|
26
|
-
function buildCacheListPayload() {
|
|
27
|
-
const cacheKeys = cache.keys();
|
|
28
|
-
return {
|
|
29
|
-
totalEntries: cacheKeys.length,
|
|
30
|
-
entries: cacheKeys.map((key) => {
|
|
31
|
-
const parts = cache.parseCacheKey(key);
|
|
32
|
-
const namespace = parts?.namespace ?? 'unknown';
|
|
33
|
-
const urlHash = parts?.urlHash ?? 'unknown';
|
|
34
|
-
return {
|
|
35
|
-
namespace,
|
|
36
|
-
urlHash,
|
|
37
|
-
resourceUri: `superfetch://cache/${namespace}/${urlHash}`,
|
|
38
|
-
};
|
|
39
|
-
}),
|
|
40
|
-
timestamp: new Date().toISOString(),
|
|
41
|
-
};
|
|
42
|
-
}
|
|
43
31
|
function notifyResourceUpdate(server, uri) {
|
|
44
32
|
if (!server.isConnected())
|
|
45
33
|
return;
|
|
@@ -52,88 +40,54 @@ function notifyResourceUpdate(server, uri) {
|
|
|
52
40
|
}
|
|
53
41
|
export function registerCachedContentResource(server) {
|
|
54
42
|
registerCacheContentResource(server);
|
|
55
|
-
registerCacheListResource(server);
|
|
56
43
|
registerCacheUpdateSubscription(server);
|
|
57
44
|
}
|
|
58
45
|
function resolveCacheParams(params) {
|
|
59
|
-
const
|
|
60
|
-
const
|
|
61
|
-
|
|
62
|
-
throw new McpError(ErrorCode.InvalidParams, 'Both namespace and urlHash parameters are required');
|
|
63
|
-
}
|
|
46
|
+
const parsed = requireRecordParams(params);
|
|
47
|
+
const namespace = requireParamString(parsed, 'namespace');
|
|
48
|
+
const urlHash = requireParamString(parsed, 'urlHash');
|
|
64
49
|
if (!isValidNamespace(namespace) || !isValidHash(urlHash)) {
|
|
65
50
|
throw new McpError(ErrorCode.InvalidParams, 'Invalid cache resource parameters');
|
|
66
51
|
}
|
|
67
52
|
return { namespace, urlHash };
|
|
68
53
|
}
|
|
69
|
-
function
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
throw new McpError(ErrorCode.InvalidParams, `Content not found in cache for key: ${cacheKey}. Use superfetch://stats to see available cache entries.`);
|
|
73
|
-
}
|
|
74
|
-
if (namespace !== 'url' && namespace !== 'markdown') {
|
|
75
|
-
return {
|
|
76
|
-
contents: [
|
|
77
|
-
{
|
|
78
|
-
uri: uri.href,
|
|
79
|
-
mimeType: resolveCacheMimeType(namespace),
|
|
80
|
-
text: cached.content,
|
|
81
|
-
},
|
|
82
|
-
],
|
|
83
|
-
};
|
|
54
|
+
function requireRecordParams(value) {
|
|
55
|
+
if (!isRecord(value)) {
|
|
56
|
+
throw new McpError(ErrorCode.InvalidParams, 'Invalid cache resource parameters');
|
|
84
57
|
}
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
58
|
+
return value;
|
|
59
|
+
}
|
|
60
|
+
function requireParamString(params, key) {
|
|
61
|
+
const raw = params[key];
|
|
62
|
+
const resolved = resolveStringParam(raw);
|
|
63
|
+
if (!resolved) {
|
|
64
|
+
throw new McpError(ErrorCode.InvalidParams, 'Both namespace and urlHash parameters are required');
|
|
91
65
|
}
|
|
92
|
-
return
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
text: resolvedContent,
|
|
98
|
-
},
|
|
99
|
-
],
|
|
100
|
-
};
|
|
66
|
+
return resolved;
|
|
67
|
+
}
|
|
68
|
+
function buildCachedContentResponse(uri, cacheKey) {
|
|
69
|
+
const cached = requireCacheEntry(cacheKey);
|
|
70
|
+
return buildMarkdownContentResponse(uri, cached.content);
|
|
101
71
|
}
|
|
102
72
|
function registerCacheContentResource(server) {
|
|
103
73
|
server.registerResource('cached-content', new ResourceTemplate('superfetch://cache/{namespace}/{urlHash}', {
|
|
104
74
|
list: listCachedResources,
|
|
105
75
|
}), {
|
|
106
76
|
title: 'Cached Content',
|
|
107
|
-
description: 'Access previously fetched web content from cache. Namespace:
|
|
77
|
+
description: 'Access previously fetched web content from cache. Namespace: markdown. UrlHash: SHA-256 hash of the URL.',
|
|
108
78
|
mimeType: 'text/plain',
|
|
109
79
|
}, (uri, params) => {
|
|
110
80
|
const { namespace, urlHash } = resolveCacheParams(params);
|
|
111
81
|
const cacheKey = `${namespace}:${urlHash}`;
|
|
112
|
-
return buildCachedContentResponse(uri, cacheKey
|
|
82
|
+
return buildCachedContentResponse(uri, cacheKey);
|
|
113
83
|
});
|
|
114
84
|
}
|
|
115
|
-
function registerCacheListResource(server) {
|
|
116
|
-
server.registerResource('cached-urls', 'superfetch://cache/list', {
|
|
117
|
-
title: 'Cached URLs List',
|
|
118
|
-
description: 'List all URLs currently in cache with their namespaces',
|
|
119
|
-
mimeType: 'application/json',
|
|
120
|
-
}, (uri) => ({
|
|
121
|
-
contents: [
|
|
122
|
-
{
|
|
123
|
-
uri: uri.href,
|
|
124
|
-
mimeType: 'application/json',
|
|
125
|
-
text: JSON.stringify(buildCacheListPayload(), null, 2),
|
|
126
|
-
},
|
|
127
|
-
],
|
|
128
|
-
}));
|
|
129
|
-
}
|
|
130
85
|
function registerCacheUpdateSubscription(server) {
|
|
131
86
|
const unsubscribe = cache.onCacheUpdate(({ cacheKey }) => {
|
|
132
|
-
const resourceUri =
|
|
87
|
+
const resourceUri = toResourceUri(cacheKey);
|
|
133
88
|
if (!resourceUri)
|
|
134
89
|
return;
|
|
135
90
|
notifyResourceUpdate(server, resourceUri);
|
|
136
|
-
notifyResourceUpdate(server, 'superfetch://cache/list');
|
|
137
91
|
if (server.isConnected()) {
|
|
138
92
|
server.sendResourceListChanged();
|
|
139
93
|
}
|
|
@@ -144,15 +98,8 @@ function registerCacheUpdateSubscription(server) {
|
|
|
144
98
|
unsubscribe();
|
|
145
99
|
};
|
|
146
100
|
}
|
|
147
|
-
function resolveCacheMimeType(namespace) {
|
|
148
|
-
if (namespace === 'markdown')
|
|
149
|
-
return 'text/markdown';
|
|
150
|
-
if (namespace === 'url')
|
|
151
|
-
return 'application/jsonl';
|
|
152
|
-
return 'application/json';
|
|
153
|
-
}
|
|
154
101
|
function isValidNamespace(namespace) {
|
|
155
|
-
return
|
|
102
|
+
return namespace === CACHE_NAMESPACE;
|
|
156
103
|
}
|
|
157
104
|
function isValidHash(hash) {
|
|
158
105
|
return HASH_PATTERN.test(hash) && hash.length >= 8 && hash.length <= 64;
|
|
@@ -160,31 +107,26 @@ function isValidHash(hash) {
|
|
|
160
107
|
function resolveStringParam(value) {
|
|
161
108
|
return typeof value === 'string' ? value : null;
|
|
162
109
|
}
|
|
163
|
-
function
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
}
|
|
168
|
-
catch {
|
|
169
|
-
return null;
|
|
110
|
+
function requireCacheEntry(cacheKey) {
|
|
111
|
+
const cached = cache.get(cacheKey);
|
|
112
|
+
if (!cached) {
|
|
113
|
+
throw new McpError(-32002, `Content not found in cache for key: ${cacheKey}`);
|
|
170
114
|
}
|
|
115
|
+
return cached;
|
|
171
116
|
}
|
|
172
|
-
function
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
(record.markdown === undefined || typeof record.markdown === 'string'));
|
|
178
|
-
}
|
|
179
|
-
function resolvePayloadContent(payload, namespace) {
|
|
180
|
-
if (namespace === 'markdown') {
|
|
181
|
-
if (typeof payload.markdown === 'string') {
|
|
182
|
-
return payload.markdown;
|
|
183
|
-
}
|
|
184
|
-
if (typeof payload.content === 'string') {
|
|
185
|
-
return payload.content;
|
|
186
|
-
}
|
|
187
|
-
return null;
|
|
117
|
+
function buildMarkdownContentResponse(uri, content) {
|
|
118
|
+
const payload = parseCachedPayload(content);
|
|
119
|
+
const resolvedContent = payload ? resolveCachedPayloadContent(payload) : null;
|
|
120
|
+
if (!resolvedContent) {
|
|
121
|
+
throw new McpError(ErrorCode.InternalError, 'Cached markdown content is missing');
|
|
188
122
|
}
|
|
189
|
-
return
|
|
123
|
+
return {
|
|
124
|
+
contents: [
|
|
125
|
+
{
|
|
126
|
+
uri: uri.href,
|
|
127
|
+
mimeType: 'text/markdown',
|
|
128
|
+
text: resolvedContent,
|
|
129
|
+
},
|
|
130
|
+
],
|
|
131
|
+
};
|
|
190
132
|
}
|
package/dist/resources/index.js
CHANGED
|
@@ -1,86 +1,4 @@
|
|
|
1
|
-
import { config } from '../config/index.js';
|
|
2
|
-
import * as cache from '../services/cache.js';
|
|
3
1
|
import { registerCachedContentResource } from './cached-content.js';
|
|
4
|
-
function registerJsonResource(server, definition) {
|
|
5
|
-
server.registerResource(definition.name, definition.uri, {
|
|
6
|
-
title: definition.title,
|
|
7
|
-
description: definition.description,
|
|
8
|
-
mimeType: 'application/json',
|
|
9
|
-
}, (uri) => ({
|
|
10
|
-
contents: [
|
|
11
|
-
{
|
|
12
|
-
uri: uri.href,
|
|
13
|
-
mimeType: 'application/json',
|
|
14
|
-
text: JSON.stringify(definition.buildPayload(), null, 2),
|
|
15
|
-
},
|
|
16
|
-
],
|
|
17
|
-
}));
|
|
18
|
-
}
|
|
19
|
-
function buildHealthPayload() {
|
|
20
|
-
const memUsage = process.memoryUsage();
|
|
21
|
-
const heapUsedMB = Math.round(memUsage.heapUsed / 1024 / 1024);
|
|
22
|
-
const heapTotalMB = Math.round(memUsage.heapTotal / 1024 / 1024);
|
|
23
|
-
return {
|
|
24
|
-
status: 'healthy',
|
|
25
|
-
uptime: process.uptime(),
|
|
26
|
-
checks: {
|
|
27
|
-
cache: config.cache.enabled,
|
|
28
|
-
memory: {
|
|
29
|
-
heapUsed: heapUsedMB,
|
|
30
|
-
heapTotal: heapTotalMB,
|
|
31
|
-
percentage: Math.round((heapUsedMB / heapTotalMB) * 100),
|
|
32
|
-
healthy: heapUsedMB < 400,
|
|
33
|
-
},
|
|
34
|
-
},
|
|
35
|
-
timestamp: new Date().toISOString(),
|
|
36
|
-
};
|
|
37
|
-
}
|
|
38
|
-
function buildStatsPayload() {
|
|
39
|
-
return {
|
|
40
|
-
server: {
|
|
41
|
-
name: config.server.name,
|
|
42
|
-
version: config.server.version,
|
|
43
|
-
uptime: process.uptime(),
|
|
44
|
-
nodeVersion: process.version,
|
|
45
|
-
memoryUsage: process.memoryUsage(),
|
|
46
|
-
},
|
|
47
|
-
cache: {
|
|
48
|
-
enabled: config.cache.enabled,
|
|
49
|
-
ttl: config.cache.ttl,
|
|
50
|
-
maxKeys: config.cache.maxKeys,
|
|
51
|
-
totalKeys: cache.keys().length,
|
|
52
|
-
},
|
|
53
|
-
config: {
|
|
54
|
-
fetcher: {
|
|
55
|
-
timeout: config.fetcher.timeout,
|
|
56
|
-
maxRedirects: config.fetcher.maxRedirects,
|
|
57
|
-
},
|
|
58
|
-
extraction: {
|
|
59
|
-
extractMainContent: config.extraction.extractMainContent,
|
|
60
|
-
includeMetadata: config.extraction.includeMetadata,
|
|
61
|
-
},
|
|
62
|
-
},
|
|
63
|
-
};
|
|
64
|
-
}
|
|
65
2
|
export function registerResources(server) {
|
|
66
3
|
registerCachedContentResource(server);
|
|
67
|
-
const resources = [
|
|
68
|
-
{
|
|
69
|
-
name: 'health',
|
|
70
|
-
uri: 'superfetch://health',
|
|
71
|
-
title: 'Server Health',
|
|
72
|
-
description: 'Real-time server health and dependency status',
|
|
73
|
-
buildPayload: buildHealthPayload,
|
|
74
|
-
},
|
|
75
|
-
{
|
|
76
|
-
name: 'stats',
|
|
77
|
-
uri: 'superfetch://stats',
|
|
78
|
-
title: 'Server Statistics',
|
|
79
|
-
description: 'Fetch statistics and cache performance metrics',
|
|
80
|
-
buildPayload: buildStatsPayload,
|
|
81
|
-
},
|
|
82
|
-
];
|
|
83
|
-
for (const resource of resources) {
|
|
84
|
-
registerJsonResource(server, resource);
|
|
85
|
-
}
|
|
86
4
|
}
|
package/dist/server.js
CHANGED
|
@@ -1,52 +1,66 @@
|
|
|
1
1
|
import { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
|
|
2
2
|
import { StdioServerTransport } from '@modelcontextprotocol/sdk/server/stdio.js';
|
|
3
3
|
import { config } from './config/index.js';
|
|
4
|
-
import { destroyAgents } from './services/fetcher.js';
|
|
4
|
+
import { destroyAgents } from './services/fetcher/agents.js';
|
|
5
5
|
import { logError, logInfo } from './services/logger.js';
|
|
6
|
-
import { destroyTransformWorkers } from './services/transform-worker-pool.js';
|
|
7
6
|
import { registerTools } from './tools/index.js';
|
|
8
|
-
import {
|
|
9
|
-
|
|
10
|
-
|
|
7
|
+
import { registerCachedContentResource } from './resources/cached-content.js';
|
|
8
|
+
function createServerInfo() {
|
|
9
|
+
return {
|
|
11
10
|
name: config.server.name,
|
|
12
11
|
version: config.server.version,
|
|
13
|
-
}
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
},
|
|
19
|
-
|
|
12
|
+
};
|
|
13
|
+
}
|
|
14
|
+
function createServerCapabilities() {
|
|
15
|
+
return {
|
|
16
|
+
tools: { listChanged: false },
|
|
17
|
+
resources: { listChanged: true, subscribe: true },
|
|
18
|
+
logging: {},
|
|
19
|
+
};
|
|
20
|
+
}
|
|
21
|
+
function createServerInstructions(serverVersion) {
|
|
22
|
+
return `superFetch MCP server |${serverVersion}| A high-performance web content fetching and processing server.`;
|
|
23
|
+
}
|
|
24
|
+
export function createMcpServer() {
|
|
25
|
+
const server = new McpServer(createServerInfo(), {
|
|
26
|
+
capabilities: createServerCapabilities(),
|
|
27
|
+
instructions: createServerInstructions(config.server.version),
|
|
20
28
|
});
|
|
21
29
|
registerTools(server);
|
|
22
|
-
|
|
30
|
+
registerCachedContentResource(server);
|
|
23
31
|
return server;
|
|
24
32
|
}
|
|
25
|
-
|
|
26
|
-
const server = createMcpServer();
|
|
27
|
-
const transport = new StdioServerTransport();
|
|
33
|
+
function attachServerErrorHandler(server) {
|
|
28
34
|
server.server.onerror = (error) => {
|
|
29
35
|
logError('[MCP Error]', error instanceof Error ? error : { error });
|
|
30
36
|
};
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
37
|
+
}
|
|
38
|
+
function handleShutdownSignal(server, signal) {
|
|
39
|
+
process.stderr.write(`\n${signal} received, shutting down superFetch MCP server...\n`);
|
|
40
|
+
destroyAgents();
|
|
41
|
+
server
|
|
42
|
+
.close()
|
|
43
|
+
.catch((err) => {
|
|
44
|
+
logError('Error during shutdown', err instanceof Error ? err : undefined);
|
|
45
|
+
})
|
|
46
|
+
.finally(() => {
|
|
47
|
+
process.exit(0);
|
|
48
|
+
});
|
|
49
|
+
}
|
|
50
|
+
function createShutdownHandler(server) {
|
|
51
|
+
return (signal) => {
|
|
52
|
+
handleShutdownSignal(server, signal);
|
|
43
53
|
};
|
|
54
|
+
}
|
|
55
|
+
function registerSignalHandlers(handler) {
|
|
44
56
|
process.on('SIGINT', () => {
|
|
45
|
-
|
|
57
|
+
handler('SIGINT');
|
|
46
58
|
});
|
|
47
59
|
process.on('SIGTERM', () => {
|
|
48
|
-
|
|
60
|
+
handler('SIGTERM');
|
|
49
61
|
});
|
|
62
|
+
}
|
|
63
|
+
async function connectStdioServer(server, transport) {
|
|
50
64
|
try {
|
|
51
65
|
await server.connect(transport);
|
|
52
66
|
logInfo('superFetch MCP server running on stdio');
|
|
@@ -56,3 +70,10 @@ export async function startStdioServer() {
|
|
|
56
70
|
process.exit(1);
|
|
57
71
|
}
|
|
58
72
|
}
|
|
73
|
+
export async function startStdioServer() {
|
|
74
|
+
const server = createMcpServer();
|
|
75
|
+
const transport = new StdioServerTransport();
|
|
76
|
+
attachServerErrorHandler(server);
|
|
77
|
+
registerSignalHandlers(createShutdownHandler(server));
|
|
78
|
+
await connectStdioServer(server, transport);
|
|
79
|
+
}
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
export interface CacheKeyParts {
|
|
2
|
+
namespace: string;
|
|
3
|
+
urlHash: string;
|
|
4
|
+
}
|
|
5
|
+
export declare function createCacheKey(namespace: string, url: string, vary?: Record<string, unknown> | string): string | null;
|
|
6
|
+
export declare function parseCacheKey(cacheKey: string): CacheKeyParts | null;
|
|
7
|
+
export declare function toResourceUri(cacheKey: string): string | null;
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
import { CACHE_HASH } from '../config/constants.js';
|
|
2
|
+
import { sha256Hex } from '../utils/crypto.js';
|
|
3
|
+
import { isRecord } from '../utils/guards.js';
|
|
4
|
+
function stableStringify(value) {
|
|
5
|
+
if (!isRecord(value)) {
|
|
6
|
+
if (value === null || value === undefined) {
|
|
7
|
+
return '';
|
|
8
|
+
}
|
|
9
|
+
return JSON.stringify(value);
|
|
10
|
+
}
|
|
11
|
+
if (Array.isArray(value)) {
|
|
12
|
+
return `[${value.map((item) => stableStringify(item)).join(',')}]`;
|
|
13
|
+
}
|
|
14
|
+
const entries = Object.entries(value)
|
|
15
|
+
.filter(([, entryValue]) => entryValue !== undefined)
|
|
16
|
+
.sort(([a], [b]) => a.localeCompare(b))
|
|
17
|
+
.map(([key, entryValue]) => `${JSON.stringify(key)}:${stableStringify(entryValue)}`);
|
|
18
|
+
return `{${entries.join(',')}}`;
|
|
19
|
+
}
|
|
20
|
+
function createHashFragment(input, length) {
|
|
21
|
+
return sha256Hex(input).substring(0, length);
|
|
22
|
+
}
|
|
23
|
+
function buildCacheKey(namespace, urlHash, varyHash) {
|
|
24
|
+
return varyHash
|
|
25
|
+
? `${namespace}:${urlHash}.${varyHash}`
|
|
26
|
+
: `${namespace}:${urlHash}`;
|
|
27
|
+
}
|
|
28
|
+
function getVaryHash(vary) {
|
|
29
|
+
if (!vary)
|
|
30
|
+
return undefined;
|
|
31
|
+
const varyString = typeof vary === 'string' ? vary : stableStringify(vary);
|
|
32
|
+
if (!varyString)
|
|
33
|
+
return undefined;
|
|
34
|
+
return createHashFragment(varyString, CACHE_HASH.VARY_HASH_LENGTH);
|
|
35
|
+
}
|
|
36
|
+
export function createCacheKey(namespace, url, vary) {
|
|
37
|
+
if (!namespace || !url)
|
|
38
|
+
return null;
|
|
39
|
+
const urlHash = createHashFragment(url, CACHE_HASH.URL_HASH_LENGTH);
|
|
40
|
+
const varyHash = getVaryHash(vary);
|
|
41
|
+
return buildCacheKey(namespace, urlHash, varyHash);
|
|
42
|
+
}
|
|
43
|
+
export function parseCacheKey(cacheKey) {
|
|
44
|
+
if (!cacheKey)
|
|
45
|
+
return null;
|
|
46
|
+
const [namespace, ...rest] = cacheKey.split(':');
|
|
47
|
+
const urlHash = rest.join(':');
|
|
48
|
+
if (!namespace || !urlHash)
|
|
49
|
+
return null;
|
|
50
|
+
return { namespace, urlHash };
|
|
51
|
+
}
|
|
52
|
+
export function toResourceUri(cacheKey) {
|
|
53
|
+
const parts = parseCacheKey(cacheKey);
|
|
54
|
+
if (!parts)
|
|
55
|
+
return null;
|
|
56
|
+
return `superfetch://cache/${parts.namespace}/${parts.urlHash}`;
|
|
57
|
+
}
|