@j0hanz/superfetch 1.2.5 → 2.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +131 -156
- package/dist/config/auth-config.d.ts +16 -0
- package/dist/config/auth-config.js +53 -0
- package/dist/config/constants.d.ts +11 -13
- package/dist/config/constants.js +1 -3
- package/dist/config/env-parsers.d.ts +7 -0
- package/dist/config/env-parsers.js +84 -0
- package/dist/config/formatting.d.ts +2 -2
- package/dist/config/index.d.ts +47 -53
- package/dist/config/index.js +35 -64
- package/dist/config/types/content.d.ts +1 -49
- package/dist/config/types/runtime.d.ts +8 -16
- package/dist/config/types/tools.d.ts +2 -28
- package/dist/http/accept-policy.d.ts +3 -0
- package/dist/http/accept-policy.js +45 -0
- package/dist/http/async-handler.d.ts +2 -0
- package/dist/http/async-handler.js +5 -0
- package/dist/http/auth-introspection.d.ts +2 -0
- package/dist/http/auth-introspection.js +141 -0
- package/dist/http/auth-static.d.ts +2 -0
- package/dist/http/auth-static.js +23 -0
- package/dist/http/auth.d.ts +3 -2
- package/dist/http/auth.js +254 -23
- package/dist/http/cors.d.ts +6 -6
- package/dist/http/cors.js +7 -42
- package/dist/http/download-routes.d.ts +0 -12
- package/dist/http/download-routes.js +21 -58
- package/dist/http/host-allowlist.d.ts +3 -0
- package/dist/http/host-allowlist.js +117 -0
- package/dist/http/jsonrpc-http.d.ts +2 -0
- package/dist/http/jsonrpc-http.js +10 -0
- package/dist/http/mcp-routes.d.ts +8 -3
- package/dist/http/mcp-routes.js +137 -31
- package/dist/http/mcp-session-eviction.d.ts +3 -0
- package/dist/http/mcp-session-eviction.js +24 -0
- package/dist/http/mcp-session-helpers.d.ts +0 -1
- package/dist/http/mcp-session-helpers.js +1 -1
- package/dist/http/mcp-session-init.d.ts +7 -0
- package/dist/http/mcp-session-init.js +94 -0
- package/dist/http/mcp-session-slots.d.ts +17 -0
- package/dist/http/mcp-session-slots.js +55 -0
- package/dist/http/mcp-session-transport-init.d.ts +7 -0
- package/dist/http/mcp-session-transport-init.js +41 -0
- package/dist/http/mcp-session-transport.d.ts +7 -0
- package/dist/http/mcp-session-transport.js +57 -0
- package/dist/http/mcp-session-types.d.ts +5 -0
- package/dist/http/mcp-session-types.js +1 -0
- package/dist/http/mcp-session.d.ts +9 -9
- package/dist/http/mcp-session.js +15 -137
- package/dist/http/mcp-sessions.d.ts +43 -0
- package/dist/http/mcp-sessions.js +392 -0
- package/dist/http/mcp-validation.d.ts +1 -0
- package/dist/http/mcp-validation.js +11 -10
- package/dist/http/protocol-policy.d.ts +2 -0
- package/dist/http/protocol-policy.js +31 -0
- package/dist/http/rate-limit.js +7 -4
- package/dist/http/server-config.d.ts +1 -0
- package/dist/http/server-config.js +40 -0
- package/dist/http/server-middleware.d.ts +7 -9
- package/dist/http/server-middleware.js +9 -70
- package/dist/http/server-shutdown.d.ts +4 -0
- package/dist/http/server-shutdown.js +43 -0
- package/dist/http/server.d.ts +10 -0
- package/dist/http/server.js +546 -61
- package/dist/http/session-cleanup.js +8 -5
- package/dist/middleware/error-handler.d.ts +1 -1
- package/dist/middleware/error-handler.js +32 -33
- package/dist/resources/cached-content-params.d.ts +5 -0
- package/dist/resources/cached-content-params.js +36 -0
- package/dist/resources/cached-content.js +67 -125
- package/dist/resources/index.js +0 -82
- package/dist/server.js +50 -29
- package/dist/services/cache-events.d.ts +8 -0
- package/dist/services/cache-events.js +19 -0
- package/dist/services/cache-keys.d.ts +7 -0
- package/dist/services/cache-keys.js +57 -0
- package/dist/services/cache.d.ts +4 -9
- package/dist/services/cache.js +77 -139
- package/dist/services/context.d.ts +0 -1
- package/dist/services/context.js +0 -7
- package/dist/services/extractor.js +55 -116
- package/dist/services/fetcher/agents.d.ts +2 -2
- package/dist/services/fetcher/agents.js +35 -96
- package/dist/services/fetcher/dns-selection.d.ts +2 -0
- package/dist/services/fetcher/dns-selection.js +72 -0
- package/dist/services/fetcher/interceptors.d.ts +0 -22
- package/dist/services/fetcher/interceptors.js +18 -32
- package/dist/services/fetcher/redirects.js +16 -7
- package/dist/services/fetcher/response.js +79 -34
- package/dist/services/fetcher.d.ts +22 -3
- package/dist/services/fetcher.js +544 -44
- package/dist/services/fifo-queue.d.ts +8 -0
- package/dist/services/fifo-queue.js +25 -0
- package/dist/services/logger.js +2 -2
- package/dist/services/metadata-collector.d.ts +1 -9
- package/dist/services/metadata-collector.js +71 -2
- package/dist/services/transform-worker-pool.d.ts +4 -14
- package/dist/services/transform-worker-pool.js +177 -129
- package/dist/services/transform-worker-types.d.ts +32 -0
- package/dist/services/transform-worker-types.js +14 -0
- package/dist/tools/handlers/fetch-markdown.tool.d.ts +3 -4
- package/dist/tools/handlers/fetch-markdown.tool.js +20 -72
- package/dist/tools/handlers/fetch-single.shared.d.ts +11 -22
- package/dist/tools/handlers/fetch-single.shared.js +175 -89
- package/dist/tools/handlers/fetch-url.tool.d.ts +7 -1
- package/dist/tools/handlers/fetch-url.tool.js +84 -119
- package/dist/tools/index.js +21 -40
- package/dist/tools/schemas.d.ts +1 -51
- package/dist/tools/schemas.js +1 -107
- package/dist/tools/utils/cached-markdown.d.ts +5 -0
- package/dist/tools/utils/cached-markdown.js +46 -0
- package/dist/tools/utils/content-shaping.d.ts +4 -0
- package/dist/tools/utils/content-shaping.js +67 -0
- package/dist/tools/utils/content-transform.d.ts +5 -17
- package/dist/tools/utils/content-transform.js +134 -114
- package/dist/tools/utils/fetch-pipeline.d.ts +0 -8
- package/dist/tools/utils/fetch-pipeline.js +57 -63
- package/dist/tools/utils/frontmatter.d.ts +3 -0
- package/dist/tools/utils/frontmatter.js +73 -0
- package/dist/tools/utils/inline-content.d.ts +1 -2
- package/dist/tools/utils/inline-content.js +4 -7
- package/dist/tools/utils/markdown-heuristics.d.ts +1 -0
- package/dist/tools/utils/markdown-heuristics.js +19 -0
- package/dist/tools/utils/markdown-signals.d.ts +1 -0
- package/dist/tools/utils/markdown-signals.js +19 -0
- package/dist/tools/utils/raw-markdown-frontmatter.d.ts +3 -0
- package/dist/tools/utils/raw-markdown-frontmatter.js +73 -0
- package/dist/tools/utils/raw-markdown.d.ts +6 -0
- package/dist/tools/utils/raw-markdown.js +135 -0
- package/dist/transformers/markdown/fenced-code-rule.d.ts +2 -0
- package/dist/transformers/markdown/fenced-code-rule.js +38 -0
- package/dist/transformers/markdown/frontmatter.d.ts +2 -0
- package/dist/transformers/markdown/frontmatter.js +45 -0
- package/dist/transformers/markdown/noise-rule.d.ts +2 -0
- package/dist/transformers/markdown/noise-rule.js +80 -0
- package/dist/transformers/markdown/turndown-instance.d.ts +2 -0
- package/dist/transformers/markdown/turndown-instance.js +19 -0
- package/dist/transformers/markdown.d.ts +2 -0
- package/dist/transformers/markdown.js +185 -0
- package/dist/transformers/markdown.transformer.js +5 -117
- package/dist/utils/cached-payload.d.ts +7 -0
- package/dist/utils/cached-payload.js +36 -0
- package/dist/utils/code-language-bash.d.ts +1 -0
- package/dist/utils/code-language-bash.js +48 -0
- package/dist/utils/code-language-core.d.ts +2 -0
- package/dist/utils/code-language-core.js +13 -0
- package/dist/utils/code-language-detectors.d.ts +5 -0
- package/dist/utils/code-language-detectors.js +142 -0
- package/dist/utils/code-language-helpers.d.ts +5 -0
- package/dist/utils/code-language-helpers.js +62 -0
- package/dist/utils/code-language-parsing.d.ts +5 -0
- package/dist/utils/code-language-parsing.js +62 -0
- package/dist/utils/code-language.d.ts +9 -0
- package/dist/utils/code-language.js +250 -46
- package/dist/utils/error-details.d.ts +3 -0
- package/dist/utils/error-details.js +12 -0
- package/dist/utils/error-utils.js +1 -1
- package/dist/utils/filename-generator.js +34 -12
- package/dist/utils/guards.d.ts +1 -0
- package/dist/utils/guards.js +3 -0
- package/dist/utils/header-normalizer.d.ts +0 -3
- package/dist/utils/header-normalizer.js +3 -3
- package/dist/utils/ip-address.d.ts +4 -0
- package/dist/utils/ip-address.js +6 -0
- package/dist/utils/tool-error-handler.d.ts +2 -2
- package/dist/utils/tool-error-handler.js +14 -46
- package/dist/utils/url-transformer.d.ts +7 -0
- package/dist/utils/url-transformer.js +147 -0
- package/dist/utils/url-validator.d.ts +1 -2
- package/dist/utils/url-validator.js +53 -114
- package/dist/workers/content-transform.worker.d.ts +1 -0
- package/dist/workers/content-transform.worker.js +40 -0
- package/package.json +17 -18
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
import { config } from '../config/index.js';
|
|
2
|
+
import { logError } from '../services/logger.js';
|
|
3
|
+
export function assertHttpConfiguration() {
|
|
4
|
+
ensureBindAllowed();
|
|
5
|
+
ensureStaticTokens();
|
|
6
|
+
if (config.auth.mode === 'oauth') {
|
|
7
|
+
ensureOauthConfiguration();
|
|
8
|
+
}
|
|
9
|
+
}
|
|
10
|
+
function ensureBindAllowed() {
|
|
11
|
+
const isLoopback = ['127.0.0.1', '::1', 'localhost'].includes(config.server.host);
|
|
12
|
+
if (!config.security.allowRemote && !isLoopback) {
|
|
13
|
+
logError('Refusing to bind to non-loopback host without ALLOW_REMOTE=true', { host: config.server.host });
|
|
14
|
+
process.exit(1);
|
|
15
|
+
}
|
|
16
|
+
if (config.security.allowRemote && config.auth.mode !== 'oauth') {
|
|
17
|
+
logError('Remote HTTP mode requires OAuth configuration; refusing to start');
|
|
18
|
+
process.exit(1);
|
|
19
|
+
}
|
|
20
|
+
}
|
|
21
|
+
function ensureStaticTokens() {
|
|
22
|
+
if (config.auth.mode === 'static' && config.auth.staticTokens.length === 0) {
|
|
23
|
+
logError('At least one static access token is required for HTTP mode');
|
|
24
|
+
process.exit(1);
|
|
25
|
+
}
|
|
26
|
+
}
|
|
27
|
+
function ensureOauthConfiguration() {
|
|
28
|
+
if (!config.auth.issuerUrl || !config.auth.authorizationUrl) {
|
|
29
|
+
logError('OAUTH_ISSUER_URL and OAUTH_AUTHORIZATION_URL are required for OAuth mode');
|
|
30
|
+
process.exit(1);
|
|
31
|
+
}
|
|
32
|
+
if (!config.auth.tokenUrl) {
|
|
33
|
+
logError('OAUTH_TOKEN_URL is required for OAuth mode');
|
|
34
|
+
process.exit(1);
|
|
35
|
+
}
|
|
36
|
+
if (!config.auth.introspectionUrl) {
|
|
37
|
+
logError('OAUTH_INTROSPECTION_URL is required for OAuth mode');
|
|
38
|
+
process.exit(1);
|
|
39
|
+
}
|
|
40
|
+
}
|
|
@@ -1,9 +1,7 @@
|
|
|
1
|
-
import type { Express,
|
|
2
|
-
export declare function
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
export declare function registerHealthRoute(app: Express): void;
|
|
9
|
-
export declare function attachBaseMiddleware(app: Express, jsonParser: RequestHandler, rateLimitMiddleware: RequestHandler, authMiddleware: RequestHandler, corsMiddleware: RequestHandler): void;
|
|
1
|
+
import type { Express, RequestHandler } from 'express';
|
|
2
|
+
export declare function attachBaseMiddleware(options: {
|
|
3
|
+
app: Express;
|
|
4
|
+
jsonParser: RequestHandler;
|
|
5
|
+
rateLimitMiddleware: RequestHandler;
|
|
6
|
+
corsMiddleware: RequestHandler;
|
|
7
|
+
}): void;
|
|
@@ -1,70 +1,9 @@
|
|
|
1
1
|
import { randomUUID } from 'node:crypto';
|
|
2
2
|
import { config } from '../config/index.js';
|
|
3
|
-
import {
|
|
3
|
+
import { runWithRequestContext } from '../services/context.js';
|
|
4
|
+
import { createHostValidationMiddleware, createOriginValidationMiddleware, } from './host-allowlist.js';
|
|
4
5
|
import { getSessionId } from './sessions.js';
|
|
5
|
-
|
|
6
|
-
function normalizeHost(value) {
|
|
7
|
-
const trimmed = value.trim().toLowerCase();
|
|
8
|
-
if (!trimmed)
|
|
9
|
-
return null;
|
|
10
|
-
const first = trimmed.split(',')[0]?.trim();
|
|
11
|
-
if (!first)
|
|
12
|
-
return null;
|
|
13
|
-
if (first.startsWith('[')) {
|
|
14
|
-
const end = first.indexOf(']');
|
|
15
|
-
if (end === -1)
|
|
16
|
-
return null;
|
|
17
|
-
return first.slice(1, end);
|
|
18
|
-
}
|
|
19
|
-
const colonIndex = first.indexOf(':');
|
|
20
|
-
if (colonIndex !== -1) {
|
|
21
|
-
return first.slice(0, colonIndex);
|
|
22
|
-
}
|
|
23
|
-
return first;
|
|
24
|
-
}
|
|
25
|
-
function buildAllowedHosts() {
|
|
26
|
-
const allowedHosts = new Set();
|
|
27
|
-
const raw = process.env.ALLOWED_HOSTS ?? '';
|
|
28
|
-
for (const entry of raw.split(',')) {
|
|
29
|
-
const normalized = normalizeHost(entry);
|
|
30
|
-
if (normalized) {
|
|
31
|
-
allowedHosts.add(normalized);
|
|
32
|
-
}
|
|
33
|
-
}
|
|
34
|
-
for (const host of LOOPBACK_HOSTS) {
|
|
35
|
-
allowedHosts.add(host);
|
|
36
|
-
}
|
|
37
|
-
const configuredHost = normalizeHost(config.server.host);
|
|
38
|
-
if (configuredHost &&
|
|
39
|
-
configuredHost !== '0.0.0.0' &&
|
|
40
|
-
configuredHost !== '::') {
|
|
41
|
-
allowedHosts.add(configuredHost);
|
|
42
|
-
}
|
|
43
|
-
return allowedHosts;
|
|
44
|
-
}
|
|
45
|
-
function createHostValidationMiddleware() {
|
|
46
|
-
const allowedHosts = buildAllowedHosts();
|
|
47
|
-
return (req, res, next) => {
|
|
48
|
-
const hostHeader = typeof req.headers.host === 'string' ? req.headers.host : '';
|
|
49
|
-
const normalized = normalizeHost(hostHeader);
|
|
50
|
-
if (!normalized || !allowedHosts.has(normalized)) {
|
|
51
|
-
res.status(403).json({
|
|
52
|
-
error: 'Host not allowed',
|
|
53
|
-
code: 'HOST_NOT_ALLOWED',
|
|
54
|
-
});
|
|
55
|
-
return;
|
|
56
|
-
}
|
|
57
|
-
next();
|
|
58
|
-
};
|
|
59
|
-
}
|
|
60
|
-
export function buildCorsOptions() {
|
|
61
|
-
const allowedOrigins = process.env.ALLOWED_ORIGINS
|
|
62
|
-
? process.env.ALLOWED_ORIGINS.split(',').map((o) => o.trim())
|
|
63
|
-
: [];
|
|
64
|
-
const allowAllOrigins = process.env.CORS_ALLOW_ALL === 'true';
|
|
65
|
-
return { allowedOrigins, allowAllOrigins };
|
|
66
|
-
}
|
|
67
|
-
export function createJsonParseErrorHandler() {
|
|
6
|
+
function createJsonParseErrorHandler() {
|
|
68
7
|
return (err, _req, res, next) => {
|
|
69
8
|
if (err instanceof SyntaxError && 'body' in err) {
|
|
70
9
|
res.status(400).json({
|
|
@@ -80,18 +19,17 @@ export function createJsonParseErrorHandler() {
|
|
|
80
19
|
next();
|
|
81
20
|
};
|
|
82
21
|
}
|
|
83
|
-
|
|
22
|
+
function createContextMiddleware() {
|
|
84
23
|
return (req, _res, next) => {
|
|
85
24
|
const requestId = randomUUID();
|
|
86
25
|
const sessionId = getSessionId(req);
|
|
87
26
|
const context = sessionId === undefined ? { requestId } : { requestId, sessionId };
|
|
88
27
|
runWithRequestContext(context, () => {
|
|
89
|
-
|
|
90
|
-
boundNext();
|
|
28
|
+
next();
|
|
91
29
|
});
|
|
92
30
|
};
|
|
93
31
|
}
|
|
94
|
-
|
|
32
|
+
function registerHealthRoute(app) {
|
|
95
33
|
app.get('/health', (_req, res) => {
|
|
96
34
|
res.json({
|
|
97
35
|
status: 'healthy',
|
|
@@ -101,13 +39,14 @@ export function registerHealthRoute(app) {
|
|
|
101
39
|
});
|
|
102
40
|
});
|
|
103
41
|
}
|
|
104
|
-
export function attachBaseMiddleware(
|
|
42
|
+
export function attachBaseMiddleware(options) {
|
|
43
|
+
const { app, jsonParser, rateLimitMiddleware, corsMiddleware } = options;
|
|
105
44
|
app.use(createHostValidationMiddleware());
|
|
45
|
+
app.use(createOriginValidationMiddleware());
|
|
106
46
|
app.use(jsonParser);
|
|
107
47
|
app.use(createContextMiddleware());
|
|
108
48
|
app.use(createJsonParseErrorHandler());
|
|
109
49
|
app.use(corsMiddleware);
|
|
110
50
|
app.use('/mcp', rateLimitMiddleware);
|
|
111
|
-
app.use(authMiddleware);
|
|
112
51
|
registerHealthRoute(app);
|
|
113
52
|
}
|
|
@@ -0,0 +1,4 @@
|
|
|
1
|
+
import type { Express } from 'express';
|
|
2
|
+
import type { SessionStore } from './sessions.js';
|
|
3
|
+
export declare function createShutdownHandler(server: ReturnType<Express['listen']>, sessionStore: SessionStore, sessionCleanupController: AbortController, stopRateLimitCleanup: () => void): (signal: string) => Promise<void>;
|
|
4
|
+
export declare function registerSignalHandlers(shutdown: (signal: string) => Promise<void>): void;
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
import { destroyAgents } from '../services/fetcher/agents.js';
|
|
2
|
+
import { logError, logInfo, logWarn } from '../services/logger.js';
|
|
3
|
+
import { getErrorMessage } from '../utils/error-details.js';
|
|
4
|
+
export function createShutdownHandler(server, sessionStore, sessionCleanupController, stopRateLimitCleanup) {
|
|
5
|
+
return (signal) => shutdownServer(signal, server, sessionStore, sessionCleanupController, stopRateLimitCleanup);
|
|
6
|
+
}
|
|
7
|
+
async function shutdownServer(signal, server, sessionStore, sessionCleanupController, stopRateLimitCleanup) {
|
|
8
|
+
logInfo(`${signal} received, shutting down gracefully...`);
|
|
9
|
+
stopRateLimitCleanup();
|
|
10
|
+
sessionCleanupController.abort();
|
|
11
|
+
await closeSessions(sessionStore);
|
|
12
|
+
destroyAgents();
|
|
13
|
+
closeServer(server);
|
|
14
|
+
scheduleForcedShutdown(10000);
|
|
15
|
+
}
|
|
16
|
+
async function closeSessions(sessionStore) {
|
|
17
|
+
const sessions = sessionStore.clear();
|
|
18
|
+
await Promise.allSettled(sessions.map((session) => session.transport.close().catch((error) => {
|
|
19
|
+
logWarn('Failed to close session during shutdown', {
|
|
20
|
+
error: getErrorMessage(error),
|
|
21
|
+
});
|
|
22
|
+
})));
|
|
23
|
+
}
|
|
24
|
+
function closeServer(server) {
|
|
25
|
+
server.close(() => {
|
|
26
|
+
logInfo('HTTP server closed');
|
|
27
|
+
process.exit(0);
|
|
28
|
+
});
|
|
29
|
+
}
|
|
30
|
+
function scheduleForcedShutdown(timeoutMs) {
|
|
31
|
+
setTimeout(() => {
|
|
32
|
+
logError('Forced shutdown after timeout');
|
|
33
|
+
process.exit(1);
|
|
34
|
+
}, timeoutMs).unref();
|
|
35
|
+
}
|
|
36
|
+
export function registerSignalHandlers(shutdown) {
|
|
37
|
+
process.on('SIGINT', () => {
|
|
38
|
+
void shutdown('SIGINT');
|
|
39
|
+
});
|
|
40
|
+
process.on('SIGTERM', () => {
|
|
41
|
+
void shutdown('SIGTERM');
|
|
42
|
+
});
|
|
43
|
+
}
|
package/dist/http/server.d.ts
CHANGED
|
@@ -1,3 +1,13 @@
|
|
|
1
|
+
import type { Express, NextFunction, Request, RequestHandler, Response } from 'express';
|
|
2
|
+
export declare function createCorsMiddleware(): (req: Request, res: Response, next: NextFunction) => void;
|
|
3
|
+
export declare function attachBaseMiddleware(options: {
|
|
4
|
+
app: Express;
|
|
5
|
+
jsonParser: RequestHandler;
|
|
6
|
+
rateLimitMiddleware: RequestHandler;
|
|
7
|
+
corsMiddleware: RequestHandler;
|
|
8
|
+
}): void;
|
|
9
|
+
export declare function registerDownloadRoutes(app: Express): void;
|
|
10
|
+
export declare function errorHandler(err: Error, req: Request, res: Response, next: NextFunction): void;
|
|
1
11
|
export declare function startHttpServer(): Promise<{
|
|
2
12
|
shutdown: (signal: string) => Promise<void>;
|
|
3
13
|
}>;
|