@j0hanz/superfetch 1.2.5 → 2.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +131 -156
- package/dist/config/auth-config.d.ts +16 -0
- package/dist/config/auth-config.js +53 -0
- package/dist/config/constants.d.ts +11 -13
- package/dist/config/constants.js +1 -3
- package/dist/config/env-parsers.d.ts +7 -0
- package/dist/config/env-parsers.js +84 -0
- package/dist/config/formatting.d.ts +2 -2
- package/dist/config/index.d.ts +47 -53
- package/dist/config/index.js +35 -64
- package/dist/config/types/content.d.ts +1 -49
- package/dist/config/types/runtime.d.ts +8 -16
- package/dist/config/types/tools.d.ts +2 -28
- package/dist/http/accept-policy.d.ts +3 -0
- package/dist/http/accept-policy.js +45 -0
- package/dist/http/async-handler.d.ts +2 -0
- package/dist/http/async-handler.js +5 -0
- package/dist/http/auth-introspection.d.ts +2 -0
- package/dist/http/auth-introspection.js +141 -0
- package/dist/http/auth-static.d.ts +2 -0
- package/dist/http/auth-static.js +23 -0
- package/dist/http/auth.d.ts +3 -2
- package/dist/http/auth.js +254 -23
- package/dist/http/cors.d.ts +6 -6
- package/dist/http/cors.js +7 -42
- package/dist/http/download-routes.d.ts +0 -12
- package/dist/http/download-routes.js +21 -58
- package/dist/http/host-allowlist.d.ts +3 -0
- package/dist/http/host-allowlist.js +117 -0
- package/dist/http/jsonrpc-http.d.ts +2 -0
- package/dist/http/jsonrpc-http.js +10 -0
- package/dist/http/mcp-routes.d.ts +8 -3
- package/dist/http/mcp-routes.js +137 -31
- package/dist/http/mcp-session-eviction.d.ts +3 -0
- package/dist/http/mcp-session-eviction.js +24 -0
- package/dist/http/mcp-session-helpers.d.ts +0 -1
- package/dist/http/mcp-session-helpers.js +1 -1
- package/dist/http/mcp-session-init.d.ts +7 -0
- package/dist/http/mcp-session-init.js +94 -0
- package/dist/http/mcp-session-slots.d.ts +17 -0
- package/dist/http/mcp-session-slots.js +55 -0
- package/dist/http/mcp-session-transport-init.d.ts +7 -0
- package/dist/http/mcp-session-transport-init.js +41 -0
- package/dist/http/mcp-session-transport.d.ts +7 -0
- package/dist/http/mcp-session-transport.js +57 -0
- package/dist/http/mcp-session-types.d.ts +5 -0
- package/dist/http/mcp-session-types.js +1 -0
- package/dist/http/mcp-session.d.ts +9 -9
- package/dist/http/mcp-session.js +15 -137
- package/dist/http/mcp-sessions.d.ts +43 -0
- package/dist/http/mcp-sessions.js +392 -0
- package/dist/http/mcp-validation.d.ts +1 -0
- package/dist/http/mcp-validation.js +11 -10
- package/dist/http/protocol-policy.d.ts +2 -0
- package/dist/http/protocol-policy.js +31 -0
- package/dist/http/rate-limit.js +7 -4
- package/dist/http/server-config.d.ts +1 -0
- package/dist/http/server-config.js +40 -0
- package/dist/http/server-middleware.d.ts +7 -9
- package/dist/http/server-middleware.js +9 -70
- package/dist/http/server-shutdown.d.ts +4 -0
- package/dist/http/server-shutdown.js +43 -0
- package/dist/http/server.d.ts +10 -0
- package/dist/http/server.js +546 -61
- package/dist/http/session-cleanup.js +8 -5
- package/dist/middleware/error-handler.d.ts +1 -1
- package/dist/middleware/error-handler.js +32 -33
- package/dist/resources/cached-content-params.d.ts +5 -0
- package/dist/resources/cached-content-params.js +36 -0
- package/dist/resources/cached-content.js +67 -125
- package/dist/resources/index.js +0 -82
- package/dist/server.js +50 -29
- package/dist/services/cache-events.d.ts +8 -0
- package/dist/services/cache-events.js +19 -0
- package/dist/services/cache-keys.d.ts +7 -0
- package/dist/services/cache-keys.js +57 -0
- package/dist/services/cache.d.ts +4 -9
- package/dist/services/cache.js +77 -139
- package/dist/services/context.d.ts +0 -1
- package/dist/services/context.js +0 -7
- package/dist/services/extractor.js +55 -116
- package/dist/services/fetcher/agents.d.ts +2 -2
- package/dist/services/fetcher/agents.js +35 -96
- package/dist/services/fetcher/dns-selection.d.ts +2 -0
- package/dist/services/fetcher/dns-selection.js +72 -0
- package/dist/services/fetcher/interceptors.d.ts +0 -22
- package/dist/services/fetcher/interceptors.js +18 -32
- package/dist/services/fetcher/redirects.js +16 -7
- package/dist/services/fetcher/response.js +79 -34
- package/dist/services/fetcher.d.ts +22 -3
- package/dist/services/fetcher.js +544 -44
- package/dist/services/fifo-queue.d.ts +8 -0
- package/dist/services/fifo-queue.js +25 -0
- package/dist/services/logger.js +2 -2
- package/dist/services/metadata-collector.d.ts +1 -9
- package/dist/services/metadata-collector.js +71 -2
- package/dist/services/transform-worker-pool.d.ts +4 -14
- package/dist/services/transform-worker-pool.js +177 -129
- package/dist/services/transform-worker-types.d.ts +32 -0
- package/dist/services/transform-worker-types.js +14 -0
- package/dist/tools/handlers/fetch-markdown.tool.d.ts +3 -4
- package/dist/tools/handlers/fetch-markdown.tool.js +20 -72
- package/dist/tools/handlers/fetch-single.shared.d.ts +11 -22
- package/dist/tools/handlers/fetch-single.shared.js +175 -89
- package/dist/tools/handlers/fetch-url.tool.d.ts +7 -1
- package/dist/tools/handlers/fetch-url.tool.js +84 -119
- package/dist/tools/index.js +21 -40
- package/dist/tools/schemas.d.ts +1 -51
- package/dist/tools/schemas.js +1 -107
- package/dist/tools/utils/cached-markdown.d.ts +5 -0
- package/dist/tools/utils/cached-markdown.js +46 -0
- package/dist/tools/utils/content-shaping.d.ts +4 -0
- package/dist/tools/utils/content-shaping.js +67 -0
- package/dist/tools/utils/content-transform.d.ts +5 -17
- package/dist/tools/utils/content-transform.js +134 -114
- package/dist/tools/utils/fetch-pipeline.d.ts +0 -8
- package/dist/tools/utils/fetch-pipeline.js +57 -63
- package/dist/tools/utils/frontmatter.d.ts +3 -0
- package/dist/tools/utils/frontmatter.js +73 -0
- package/dist/tools/utils/inline-content.d.ts +1 -2
- package/dist/tools/utils/inline-content.js +4 -7
- package/dist/tools/utils/markdown-heuristics.d.ts +1 -0
- package/dist/tools/utils/markdown-heuristics.js +19 -0
- package/dist/tools/utils/markdown-signals.d.ts +1 -0
- package/dist/tools/utils/markdown-signals.js +19 -0
- package/dist/tools/utils/raw-markdown-frontmatter.d.ts +3 -0
- package/dist/tools/utils/raw-markdown-frontmatter.js +73 -0
- package/dist/tools/utils/raw-markdown.d.ts +6 -0
- package/dist/tools/utils/raw-markdown.js +135 -0
- package/dist/transformers/markdown/fenced-code-rule.d.ts +2 -0
- package/dist/transformers/markdown/fenced-code-rule.js +38 -0
- package/dist/transformers/markdown/frontmatter.d.ts +2 -0
- package/dist/transformers/markdown/frontmatter.js +45 -0
- package/dist/transformers/markdown/noise-rule.d.ts +2 -0
- package/dist/transformers/markdown/noise-rule.js +80 -0
- package/dist/transformers/markdown/turndown-instance.d.ts +2 -0
- package/dist/transformers/markdown/turndown-instance.js +19 -0
- package/dist/transformers/markdown.d.ts +2 -0
- package/dist/transformers/markdown.js +185 -0
- package/dist/transformers/markdown.transformer.js +5 -117
- package/dist/utils/cached-payload.d.ts +7 -0
- package/dist/utils/cached-payload.js +36 -0
- package/dist/utils/code-language-bash.d.ts +1 -0
- package/dist/utils/code-language-bash.js +48 -0
- package/dist/utils/code-language-core.d.ts +2 -0
- package/dist/utils/code-language-core.js +13 -0
- package/dist/utils/code-language-detectors.d.ts +5 -0
- package/dist/utils/code-language-detectors.js +142 -0
- package/dist/utils/code-language-helpers.d.ts +5 -0
- package/dist/utils/code-language-helpers.js +62 -0
- package/dist/utils/code-language-parsing.d.ts +5 -0
- package/dist/utils/code-language-parsing.js +62 -0
- package/dist/utils/code-language.d.ts +9 -0
- package/dist/utils/code-language.js +250 -46
- package/dist/utils/error-details.d.ts +3 -0
- package/dist/utils/error-details.js +12 -0
- package/dist/utils/error-utils.js +1 -1
- package/dist/utils/filename-generator.js +34 -12
- package/dist/utils/guards.d.ts +1 -0
- package/dist/utils/guards.js +3 -0
- package/dist/utils/header-normalizer.d.ts +0 -3
- package/dist/utils/header-normalizer.js +3 -3
- package/dist/utils/ip-address.d.ts +4 -0
- package/dist/utils/ip-address.js +6 -0
- package/dist/utils/tool-error-handler.d.ts +2 -2
- package/dist/utils/tool-error-handler.js +14 -46
- package/dist/utils/url-transformer.d.ts +7 -0
- package/dist/utils/url-transformer.js +147 -0
- package/dist/utils/url-validator.d.ts +1 -2
- package/dist/utils/url-validator.js +53 -114
- package/dist/workers/content-transform.worker.d.ts +1 -0
- package/dist/workers/content-transform.worker.js +40 -0
- package/package.json +17 -18
package/dist/config/index.d.ts
CHANGED
|
@@ -1,60 +1,54 @@
|
|
|
1
|
-
import type { LogLevel } from './types/runtime.js';
|
|
2
1
|
interface RuntimeState {
|
|
3
2
|
httpMode: boolean;
|
|
4
3
|
}
|
|
5
4
|
export declare const config: {
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
};
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
readonly maxRequests: number;
|
|
54
|
-
readonly windowMs: number;
|
|
55
|
-
readonly cleanupIntervalMs: number;
|
|
56
|
-
};
|
|
57
|
-
readonly runtime: RuntimeState;
|
|
5
|
+
server: {
|
|
6
|
+
name: string;
|
|
7
|
+
version: string;
|
|
8
|
+
port: number;
|
|
9
|
+
host: string;
|
|
10
|
+
sessionTtlMs: number;
|
|
11
|
+
sessionInitTimeoutMs: number;
|
|
12
|
+
maxSessions: number;
|
|
13
|
+
};
|
|
14
|
+
fetcher: {
|
|
15
|
+
timeout: number;
|
|
16
|
+
maxRedirects: number;
|
|
17
|
+
userAgent: string;
|
|
18
|
+
maxContentLength: number;
|
|
19
|
+
};
|
|
20
|
+
cache: {
|
|
21
|
+
enabled: boolean;
|
|
22
|
+
ttl: number;
|
|
23
|
+
maxKeys: number;
|
|
24
|
+
};
|
|
25
|
+
extraction: {
|
|
26
|
+
maxBlockLength: number;
|
|
27
|
+
minParagraphLength: number;
|
|
28
|
+
};
|
|
29
|
+
logging: {
|
|
30
|
+
level: import("./types/runtime.js").LogLevel;
|
|
31
|
+
};
|
|
32
|
+
constants: {
|
|
33
|
+
maxHtmlSize: number;
|
|
34
|
+
maxUrlLength: number;
|
|
35
|
+
maxInlineContentChars: number;
|
|
36
|
+
};
|
|
37
|
+
security: {
|
|
38
|
+
blockedHosts: Set<string>;
|
|
39
|
+
blockedIpPatterns: RegExp[];
|
|
40
|
+
allowedHosts: Set<string>;
|
|
41
|
+
apiKey: string | undefined;
|
|
42
|
+
allowRemote: boolean;
|
|
43
|
+
};
|
|
44
|
+
auth: import("./auth-config.js").AuthConfig;
|
|
45
|
+
rateLimit: {
|
|
46
|
+
enabled: boolean;
|
|
47
|
+
maxRequests: number;
|
|
48
|
+
windowMs: number;
|
|
49
|
+
cleanupIntervalMs: number;
|
|
50
|
+
};
|
|
51
|
+
runtime: RuntimeState;
|
|
58
52
|
};
|
|
59
53
|
export declare function enableHttpMode(): void;
|
|
60
54
|
export {};
|
package/dist/config/index.js
CHANGED
|
@@ -1,39 +1,22 @@
|
|
|
1
|
+
import { buildIpv4 } from '../utils/ip-address.js';
|
|
1
2
|
import packageJson from '../../package.json' with { type: 'json' };
|
|
3
|
+
import { buildAuthConfig } from './auth-config.js';
|
|
2
4
|
import { SIZE_LIMITS, TIMEOUT } from './constants.js';
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
if (min !== undefined && parsed < min)
|
|
10
|
-
return defaultValue;
|
|
11
|
-
if (max !== undefined && parsed > max)
|
|
12
|
-
return defaultValue;
|
|
13
|
-
return parsed;
|
|
5
|
+
import { parseAllowedHosts, parseBoolean, parseInteger, parseLogLevel, } from './env-parsers.js';
|
|
6
|
+
function formatHostForUrl(hostname) {
|
|
7
|
+
if (hostname.includes(':') && !hostname.startsWith('[')) {
|
|
8
|
+
return `[${hostname}]`;
|
|
9
|
+
}
|
|
10
|
+
return hostname;
|
|
14
11
|
}
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
return 'info';
|
|
24
|
-
return isLogLevel(level) ? level : 'info';
|
|
25
|
-
}
|
|
26
|
-
const ALLOWED_LOG_LEVELS = new Set([
|
|
27
|
-
'debug',
|
|
28
|
-
'info',
|
|
29
|
-
'warn',
|
|
30
|
-
'error',
|
|
31
|
-
]);
|
|
32
|
-
function isLogLevel(value) {
|
|
33
|
-
return ALLOWED_LOG_LEVELS.has(value);
|
|
34
|
-
}
|
|
35
|
-
const host = process.env.HOST ?? '127.0.0.1';
|
|
36
|
-
const isLoopbackHost = host === '127.0.0.1' || host === '::1' || host === 'localhost';
|
|
12
|
+
const LOOPBACK_V4 = buildIpv4([127, 0, 0, 1]);
|
|
13
|
+
const ANY_V4 = buildIpv4([0, 0, 0, 0]);
|
|
14
|
+
const METADATA_V4_AWS = buildIpv4([169, 254, 169, 254]);
|
|
15
|
+
const METADATA_V4_AZURE = buildIpv4([100, 100, 100, 200]);
|
|
16
|
+
const host = process.env.HOST ?? LOOPBACK_V4;
|
|
17
|
+
const port = parseInteger(process.env.PORT, 3000, 1024, 65535);
|
|
18
|
+
const baseUrl = new URL(`http://${formatHostForUrl(host)}:${port}`);
|
|
19
|
+
const isRemoteHost = host === ANY_V4 || host === '::';
|
|
37
20
|
const runtimeState = {
|
|
38
21
|
httpMode: false,
|
|
39
22
|
};
|
|
@@ -41,50 +24,45 @@ export const config = {
|
|
|
41
24
|
server: {
|
|
42
25
|
name: 'superFetch',
|
|
43
26
|
version: packageJson.version,
|
|
44
|
-
port
|
|
27
|
+
port,
|
|
45
28
|
host,
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
maxSessions: parseInteger(process.env.MAX_SESSIONS, 200, 10, 10000),
|
|
29
|
+
sessionTtlMs: TIMEOUT.DEFAULT_SESSION_TTL_MS,
|
|
30
|
+
sessionInitTimeoutMs: 10000,
|
|
31
|
+
maxSessions: 200,
|
|
50
32
|
},
|
|
51
33
|
fetcher: {
|
|
52
|
-
timeout:
|
|
34
|
+
timeout: TIMEOUT.DEFAULT_FETCH_TIMEOUT_MS,
|
|
53
35
|
maxRedirects: 5,
|
|
54
|
-
userAgent: process.env.USER_AGENT ?? 'superFetch-MCP/
|
|
36
|
+
userAgent: process.env.USER_AGENT ?? 'superFetch-MCP/2.0',
|
|
55
37
|
maxContentLength: SIZE_LIMITS.TEN_MB,
|
|
56
38
|
},
|
|
57
39
|
cache: {
|
|
58
40
|
enabled: parseBoolean(process.env.CACHE_ENABLED, true),
|
|
59
41
|
ttl: parseInteger(process.env.CACHE_TTL, 3600, 60, 86400),
|
|
60
|
-
maxKeys:
|
|
42
|
+
maxKeys: 100,
|
|
61
43
|
},
|
|
62
44
|
extraction: {
|
|
63
|
-
extractMainContent: parseBoolean(process.env.EXTRACT_MAIN_CONTENT, true),
|
|
64
|
-
includeMetadata: parseBoolean(process.env.INCLUDE_METADATA, true),
|
|
65
45
|
maxBlockLength: 5000,
|
|
66
46
|
minParagraphLength: 10,
|
|
67
47
|
},
|
|
68
48
|
logging: {
|
|
69
49
|
level: parseLogLevel(process.env.LOG_LEVEL),
|
|
70
|
-
enabled: parseBoolean(process.env.ENABLE_LOGGING, true),
|
|
71
50
|
},
|
|
72
51
|
constants: {
|
|
73
52
|
maxHtmlSize: SIZE_LIMITS.TEN_MB,
|
|
74
|
-
maxContentSize: SIZE_LIMITS.FIVE_MB,
|
|
75
53
|
maxUrlLength: 2048,
|
|
76
|
-
maxInlineContentChars:
|
|
54
|
+
maxInlineContentChars: 20000,
|
|
77
55
|
},
|
|
78
56
|
security: {
|
|
79
57
|
blockedHosts: new Set([
|
|
80
58
|
'localhost',
|
|
81
|
-
|
|
82
|
-
|
|
59
|
+
LOOPBACK_V4,
|
|
60
|
+
ANY_V4,
|
|
83
61
|
'::1',
|
|
84
|
-
|
|
62
|
+
METADATA_V4_AWS,
|
|
85
63
|
'metadata.google.internal',
|
|
86
64
|
'metadata.azure.com',
|
|
87
|
-
|
|
65
|
+
METADATA_V4_AZURE,
|
|
88
66
|
'instance-data',
|
|
89
67
|
]),
|
|
90
68
|
blockedIpPatterns: [
|
|
@@ -104,23 +82,16 @@ export const config = {
|
|
|
104
82
|
/^::ffff:192\.168\./,
|
|
105
83
|
/^::ffff:169\.254\./,
|
|
106
84
|
],
|
|
107
|
-
|
|
108
|
-
'host',
|
|
109
|
-
'authorization',
|
|
110
|
-
'cookie',
|
|
111
|
-
'x-forwarded-for',
|
|
112
|
-
'x-real-ip',
|
|
113
|
-
'proxy-authorization',
|
|
114
|
-
]),
|
|
85
|
+
allowedHosts: parseAllowedHosts(process.env.ALLOWED_HOSTS),
|
|
115
86
|
apiKey: process.env.API_KEY,
|
|
116
|
-
allowRemote:
|
|
117
|
-
requireAuth: parseBoolean(process.env.REQUIRE_AUTH, !isLoopbackHost),
|
|
87
|
+
allowRemote: isRemoteHost,
|
|
118
88
|
},
|
|
89
|
+
auth: buildAuthConfig(baseUrl),
|
|
119
90
|
rateLimit: {
|
|
120
|
-
enabled:
|
|
121
|
-
maxRequests:
|
|
122
|
-
windowMs:
|
|
123
|
-
cleanupIntervalMs:
|
|
91
|
+
enabled: true,
|
|
92
|
+
maxRequests: 100,
|
|
93
|
+
windowMs: 60000,
|
|
94
|
+
cleanupIntervalMs: 60000,
|
|
124
95
|
},
|
|
125
96
|
runtime: runtimeState,
|
|
126
97
|
};
|
|
@@ -1,8 +1,4 @@
|
|
|
1
|
-
|
|
2
|
-
interface ContentBlock {
|
|
3
|
-
type: ContentBlockType;
|
|
4
|
-
}
|
|
5
|
-
export interface MetadataBlock extends ContentBlock {
|
|
1
|
+
export interface MetadataBlock {
|
|
6
2
|
type: 'metadata';
|
|
7
3
|
title?: string;
|
|
8
4
|
description?: string;
|
|
@@ -10,40 +6,6 @@ export interface MetadataBlock extends ContentBlock {
|
|
|
10
6
|
url: string;
|
|
11
7
|
fetchedAt: string;
|
|
12
8
|
}
|
|
13
|
-
export interface HeadingBlock extends ContentBlock {
|
|
14
|
-
type: 'heading';
|
|
15
|
-
level: number;
|
|
16
|
-
text: string;
|
|
17
|
-
}
|
|
18
|
-
export interface ParagraphBlock extends ContentBlock {
|
|
19
|
-
type: 'paragraph';
|
|
20
|
-
text: string;
|
|
21
|
-
}
|
|
22
|
-
export interface ListBlock extends ContentBlock {
|
|
23
|
-
type: 'list';
|
|
24
|
-
ordered: boolean;
|
|
25
|
-
readonly items: readonly string[];
|
|
26
|
-
}
|
|
27
|
-
export interface CodeBlock extends ContentBlock {
|
|
28
|
-
type: 'code';
|
|
29
|
-
language?: string;
|
|
30
|
-
text: string;
|
|
31
|
-
}
|
|
32
|
-
export interface TableBlock extends ContentBlock {
|
|
33
|
-
type: 'table';
|
|
34
|
-
readonly headers?: readonly string[];
|
|
35
|
-
readonly rows: readonly (readonly string[])[];
|
|
36
|
-
}
|
|
37
|
-
export interface ImageBlock extends ContentBlock {
|
|
38
|
-
type: 'image';
|
|
39
|
-
src: string;
|
|
40
|
-
alt?: string;
|
|
41
|
-
}
|
|
42
|
-
export interface BlockquoteBlock extends ContentBlock {
|
|
43
|
-
type: 'blockquote';
|
|
44
|
-
text: string;
|
|
45
|
-
}
|
|
46
|
-
export type ContentBlockUnion = MetadataBlock | HeadingBlock | ParagraphBlock | ListBlock | CodeBlock | TableBlock | ImageBlock | BlockquoteBlock;
|
|
47
9
|
export interface ExtractedArticle {
|
|
48
10
|
title?: string;
|
|
49
11
|
byline?: string;
|
|
@@ -68,21 +30,11 @@ export interface ExtractionResult {
|
|
|
68
30
|
article: ExtractedArticle | null;
|
|
69
31
|
metadata: ExtractedMetadata;
|
|
70
32
|
}
|
|
71
|
-
export type ParseableTagName = 'h1' | 'h2' | 'h3' | 'h4' | 'h5' | 'h6' | 'p' | 'ul' | 'ol' | 'pre' | 'code' | 'table' | 'img' | 'blockquote';
|
|
72
33
|
export interface MarkdownTransformResult {
|
|
73
34
|
markdown: string;
|
|
74
35
|
title: string | undefined;
|
|
75
36
|
truncated: boolean;
|
|
76
37
|
}
|
|
77
38
|
export interface TransformOptions {
|
|
78
|
-
extractMainContent: boolean;
|
|
79
39
|
includeMetadata: boolean;
|
|
80
|
-
maxContentLength?: number;
|
|
81
|
-
}
|
|
82
|
-
export interface JsonlTransformResult {
|
|
83
|
-
content: string;
|
|
84
|
-
contentBlocks: number;
|
|
85
|
-
title: string | undefined;
|
|
86
|
-
truncated?: boolean;
|
|
87
40
|
}
|
|
88
|
-
export {};
|
|
@@ -28,13 +28,7 @@ export type ToolContentBlock = {
|
|
|
28
28
|
};
|
|
29
29
|
};
|
|
30
30
|
export interface FetchOptions {
|
|
31
|
-
customHeaders?: Record<string, string>;
|
|
32
31
|
signal?: AbortSignal;
|
|
33
|
-
timeout?: number;
|
|
34
|
-
}
|
|
35
|
-
export interface TruncationResult {
|
|
36
|
-
readonly content: string;
|
|
37
|
-
readonly truncated: boolean;
|
|
38
32
|
}
|
|
39
33
|
export interface SessionEntry {
|
|
40
34
|
readonly transport: StreamableHTTPServerTransport;
|
|
@@ -43,25 +37,23 @@ export interface SessionEntry {
|
|
|
43
37
|
}
|
|
44
38
|
export type LogLevel = 'debug' | 'info' | 'warn' | 'error';
|
|
45
39
|
export type LogMetadata = Record<string, unknown>;
|
|
40
|
+
export interface McpRequestParams {
|
|
41
|
+
_meta?: Record<string, unknown>;
|
|
42
|
+
[key: string]: unknown;
|
|
43
|
+
}
|
|
46
44
|
export interface McpRequestBody {
|
|
47
|
-
|
|
45
|
+
jsonrpc: '2.0';
|
|
46
|
+
method: string;
|
|
48
47
|
id?: string | number;
|
|
49
|
-
|
|
50
|
-
params?: unknown;
|
|
48
|
+
params?: McpRequestParams;
|
|
51
49
|
}
|
|
52
50
|
export interface FetchPipelineOptions<T> {
|
|
53
51
|
/** URL to fetch */
|
|
54
52
|
url: string;
|
|
55
|
-
/** Cache namespace (e.g., '
|
|
53
|
+
/** Cache namespace (e.g., 'markdown') */
|
|
56
54
|
cacheNamespace: string;
|
|
57
|
-
/** Optional custom HTTP headers */
|
|
58
|
-
customHeaders?: Record<string, string>;
|
|
59
|
-
/** Optional: number of retry attempts (1-10, defaults to 3) */
|
|
60
|
-
retries?: number;
|
|
61
55
|
/** Optional: AbortSignal for request cancellation */
|
|
62
56
|
signal?: AbortSignal;
|
|
63
|
-
/** Optional: per-request timeout override in milliseconds */
|
|
64
|
-
timeout?: number;
|
|
65
57
|
/** Optional: cache variation input for headers/flags */
|
|
66
58
|
cacheVary?: Record<string, unknown> | string;
|
|
67
59
|
/** Transform function to process HTML into desired format */
|
|
@@ -1,30 +1,6 @@
|
|
|
1
1
|
import type { ToolContentBlock } from './runtime.js';
|
|
2
|
-
interface
|
|
3
|
-
/** Custom HTTP headers for the request */
|
|
4
|
-
customHeaders?: Record<string, string> | undefined;
|
|
5
|
-
/** Request timeout in milliseconds (1000-120000) */
|
|
6
|
-
timeout?: number | undefined;
|
|
7
|
-
/** Number of retry attempts (1-10) */
|
|
8
|
-
retries?: number | undefined;
|
|
9
|
-
}
|
|
10
|
-
export interface FetchUrlInput extends RequestOptions {
|
|
11
|
-
url: string;
|
|
12
|
-
extractMainContent?: boolean | undefined;
|
|
13
|
-
includeMetadata?: boolean | undefined;
|
|
14
|
-
maxContentLength?: number | undefined;
|
|
15
|
-
format?: 'jsonl' | 'markdown' | undefined;
|
|
16
|
-
includeContentBlocks?: boolean | undefined;
|
|
17
|
-
}
|
|
18
|
-
export interface FetchMarkdownInput extends RequestOptions {
|
|
2
|
+
export interface FetchUrlInput {
|
|
19
3
|
url: string;
|
|
20
|
-
extractMainContent?: boolean | undefined;
|
|
21
|
-
includeMetadata?: boolean | undefined;
|
|
22
|
-
maxContentLength?: number | undefined;
|
|
23
|
-
}
|
|
24
|
-
export interface FileDownloadInfo {
|
|
25
|
-
downloadUrl: string;
|
|
26
|
-
fileName: string;
|
|
27
|
-
expiresAt: string;
|
|
28
4
|
}
|
|
29
5
|
export interface ErrorResponse {
|
|
30
6
|
error: {
|
|
@@ -41,8 +17,7 @@ export interface ToolErrorResponse {
|
|
|
41
17
|
structuredContent: {
|
|
42
18
|
error: string;
|
|
43
19
|
url: string;
|
|
44
|
-
|
|
45
|
-
} & Record<string, unknown>;
|
|
20
|
+
};
|
|
46
21
|
isError: true;
|
|
47
22
|
}
|
|
48
23
|
export interface ToolResponseBase {
|
|
@@ -51,4 +26,3 @@ export interface ToolResponseBase {
|
|
|
51
26
|
structuredContent?: Record<string, unknown>;
|
|
52
27
|
isError?: boolean;
|
|
53
28
|
}
|
|
54
|
-
export {};
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
function getAcceptHeader(req) {
|
|
2
|
+
const value = req.headers.accept;
|
|
3
|
+
if (typeof value === 'string')
|
|
4
|
+
return value;
|
|
5
|
+
return '';
|
|
6
|
+
}
|
|
7
|
+
function setAcceptHeader(req, value) {
|
|
8
|
+
req.headers.accept = value;
|
|
9
|
+
const { rawHeaders } = req;
|
|
10
|
+
if (!Array.isArray(rawHeaders))
|
|
11
|
+
return;
|
|
12
|
+
for (let i = 0; i + 1 < rawHeaders.length; i += 2) {
|
|
13
|
+
const key = rawHeaders[i];
|
|
14
|
+
if (typeof key === 'string' && key.toLowerCase() === 'accept') {
|
|
15
|
+
rawHeaders[i + 1] = value;
|
|
16
|
+
return;
|
|
17
|
+
}
|
|
18
|
+
}
|
|
19
|
+
rawHeaders.push('Accept', value);
|
|
20
|
+
}
|
|
21
|
+
function hasToken(header, token) {
|
|
22
|
+
return header
|
|
23
|
+
.split(',')
|
|
24
|
+
.map((part) => part.trim().toLowerCase())
|
|
25
|
+
.some((part) => part === token || part.startsWith(`${token};`));
|
|
26
|
+
}
|
|
27
|
+
export function ensurePostAcceptHeader(req) {
|
|
28
|
+
const accept = getAcceptHeader(req);
|
|
29
|
+
// Some clients send */* or omit Accept; the SDK transport is picky.
|
|
30
|
+
if (!accept || hasToken(accept, '*/*')) {
|
|
31
|
+
setAcceptHeader(req, 'application/json, text/event-stream');
|
|
32
|
+
return;
|
|
33
|
+
}
|
|
34
|
+
const hasJson = hasToken(accept, 'application/json');
|
|
35
|
+
const hasSse = hasToken(accept, 'text/event-stream');
|
|
36
|
+
if (!hasJson || !hasSse) {
|
|
37
|
+
setAcceptHeader(req, 'application/json, text/event-stream');
|
|
38
|
+
}
|
|
39
|
+
}
|
|
40
|
+
export function acceptsEventStream(req) {
|
|
41
|
+
const accept = getAcceptHeader(req);
|
|
42
|
+
if (!accept)
|
|
43
|
+
return false;
|
|
44
|
+
return hasToken(accept, 'text/event-stream');
|
|
45
|
+
}
|
|
@@ -0,0 +1,141 @@
|
|
|
1
|
+
import { InvalidTokenError, ServerError, } from '@modelcontextprotocol/sdk/server/auth/errors.js';
|
|
2
|
+
import { config } from '../config/index.js';
|
|
3
|
+
import { isRecord } from '../utils/guards.js';
|
|
4
|
+
function stripHash(url) {
|
|
5
|
+
const copy = new URL(url.href);
|
|
6
|
+
copy.hash = '';
|
|
7
|
+
return copy.href;
|
|
8
|
+
}
|
|
9
|
+
function parseScopes(value) {
|
|
10
|
+
if (typeof value === 'string') {
|
|
11
|
+
return value
|
|
12
|
+
.split(' ')
|
|
13
|
+
.map((scope) => scope.trim())
|
|
14
|
+
.filter((scope) => scope.length > 0);
|
|
15
|
+
}
|
|
16
|
+
if (Array.isArray(value)) {
|
|
17
|
+
return value.filter((scope) => typeof scope === 'string');
|
|
18
|
+
}
|
|
19
|
+
return [];
|
|
20
|
+
}
|
|
21
|
+
function parseResourceUrl(value) {
|
|
22
|
+
if (typeof value !== 'string')
|
|
23
|
+
return undefined;
|
|
24
|
+
if (!URL.canParse(value))
|
|
25
|
+
return undefined;
|
|
26
|
+
return new URL(value);
|
|
27
|
+
}
|
|
28
|
+
function parseAudResource(aud) {
|
|
29
|
+
if (typeof aud === 'string') {
|
|
30
|
+
return parseResourceUrl(aud);
|
|
31
|
+
}
|
|
32
|
+
if (Array.isArray(aud)) {
|
|
33
|
+
for (const entry of aud) {
|
|
34
|
+
const parsed = parseResourceUrl(entry);
|
|
35
|
+
if (parsed)
|
|
36
|
+
return parsed;
|
|
37
|
+
}
|
|
38
|
+
}
|
|
39
|
+
return undefined;
|
|
40
|
+
}
|
|
41
|
+
function extractResource(data) {
|
|
42
|
+
const resource = parseResourceUrl(data.resource);
|
|
43
|
+
if (resource)
|
|
44
|
+
return resource;
|
|
45
|
+
return parseAudResource(data.aud);
|
|
46
|
+
}
|
|
47
|
+
function extractScopes(data) {
|
|
48
|
+
if (data.scope !== undefined) {
|
|
49
|
+
return parseScopes(data.scope);
|
|
50
|
+
}
|
|
51
|
+
if (data.scopes !== undefined) {
|
|
52
|
+
return parseScopes(data.scopes);
|
|
53
|
+
}
|
|
54
|
+
if (data.scp !== undefined) {
|
|
55
|
+
return parseScopes(data.scp);
|
|
56
|
+
}
|
|
57
|
+
return [];
|
|
58
|
+
}
|
|
59
|
+
function readExpiresAt(data) {
|
|
60
|
+
const expiresAt = typeof data.exp === 'number' ? data.exp : Number.NaN;
|
|
61
|
+
if (!Number.isFinite(expiresAt)) {
|
|
62
|
+
throw new InvalidTokenError('Token has no expiration time');
|
|
63
|
+
}
|
|
64
|
+
return expiresAt;
|
|
65
|
+
}
|
|
66
|
+
function resolveClientId(data) {
|
|
67
|
+
if (typeof data.client_id === 'string')
|
|
68
|
+
return data.client_id;
|
|
69
|
+
if (typeof data.cid === 'string')
|
|
70
|
+
return data.cid;
|
|
71
|
+
if (typeof data.sub === 'string')
|
|
72
|
+
return data.sub;
|
|
73
|
+
return 'unknown';
|
|
74
|
+
}
|
|
75
|
+
function ensureResourceMatch(resource) {
|
|
76
|
+
if (resource && stripHash(resource) !== stripHash(config.auth.resourceUrl)) {
|
|
77
|
+
throw new InvalidTokenError('Token resource mismatch');
|
|
78
|
+
}
|
|
79
|
+
return resource;
|
|
80
|
+
}
|
|
81
|
+
function buildIntrospectionAuthInfo(token, data) {
|
|
82
|
+
const resource = ensureResourceMatch(extractResource(data));
|
|
83
|
+
return {
|
|
84
|
+
token,
|
|
85
|
+
clientId: resolveClientId(data),
|
|
86
|
+
scopes: extractScopes(data),
|
|
87
|
+
expiresAt: readExpiresAt(data),
|
|
88
|
+
resource: resource ?? config.auth.resourceUrl,
|
|
89
|
+
extra: data,
|
|
90
|
+
};
|
|
91
|
+
}
|
|
92
|
+
function buildBasicAuthHeader(clientId, clientSecret) {
|
|
93
|
+
const secret = clientSecret ?? '';
|
|
94
|
+
const basic = Buffer.from(`${clientId}:${secret}`, 'utf8').toString('base64');
|
|
95
|
+
return `Basic ${basic}`;
|
|
96
|
+
}
|
|
97
|
+
function buildIntrospectionRequest(token, resourceUrl, clientId, clientSecret) {
|
|
98
|
+
const body = new URLSearchParams({
|
|
99
|
+
token,
|
|
100
|
+
token_type_hint: 'access_token',
|
|
101
|
+
resource: stripHash(resourceUrl),
|
|
102
|
+
}).toString();
|
|
103
|
+
const headers = {
|
|
104
|
+
'content-type': 'application/x-www-form-urlencoded',
|
|
105
|
+
};
|
|
106
|
+
if (clientId) {
|
|
107
|
+
headers.authorization = buildBasicAuthHeader(clientId, clientSecret);
|
|
108
|
+
}
|
|
109
|
+
return { body, headers };
|
|
110
|
+
}
|
|
111
|
+
async function requestIntrospection(introspectionUrl, request, timeoutMs) {
|
|
112
|
+
const response = await fetch(introspectionUrl, {
|
|
113
|
+
method: 'POST',
|
|
114
|
+
headers: request.headers,
|
|
115
|
+
body: request.body,
|
|
116
|
+
signal: AbortSignal.timeout(timeoutMs),
|
|
117
|
+
});
|
|
118
|
+
if (!response.ok) {
|
|
119
|
+
await response.body?.cancel();
|
|
120
|
+
throw new ServerError(`Token introspection failed: ${response.status}`);
|
|
121
|
+
}
|
|
122
|
+
return response.json();
|
|
123
|
+
}
|
|
124
|
+
function parseIntrospectionPayload(payload) {
|
|
125
|
+
if (!isRecord(payload) || Array.isArray(payload)) {
|
|
126
|
+
throw new ServerError('Invalid introspection response');
|
|
127
|
+
}
|
|
128
|
+
if (payload.active !== true) {
|
|
129
|
+
throw new InvalidTokenError('Token is inactive');
|
|
130
|
+
}
|
|
131
|
+
return payload;
|
|
132
|
+
}
|
|
133
|
+
export async function verifyWithIntrospection(token) {
|
|
134
|
+
const { auth } = config;
|
|
135
|
+
if (!auth.introspectionUrl) {
|
|
136
|
+
throw new ServerError('Token introspection is not configured');
|
|
137
|
+
}
|
|
138
|
+
const request = buildIntrospectionRequest(token, auth.resourceUrl, auth.clientId, auth.clientSecret);
|
|
139
|
+
const payload = await requestIntrospection(auth.introspectionUrl, request, auth.introspectionTimeoutMs);
|
|
140
|
+
return buildIntrospectionAuthInfo(token, parseIntrospectionPayload(payload));
|
|
141
|
+
}
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
import { InvalidTokenError } from '@modelcontextprotocol/sdk/server/auth/errors.js';
|
|
2
|
+
import { config } from '../config/index.js';
|
|
3
|
+
import { timingSafeEqualUtf8 } from '../utils/crypto.js';
|
|
4
|
+
const STATIC_TOKEN_TTL_SECONDS = 60 * 60 * 24;
|
|
5
|
+
function buildStaticAuthInfo(token) {
|
|
6
|
+
return {
|
|
7
|
+
token,
|
|
8
|
+
clientId: 'static-token',
|
|
9
|
+
scopes: config.auth.requiredScopes,
|
|
10
|
+
expiresAt: Math.floor(Date.now() / 1000) + STATIC_TOKEN_TTL_SECONDS,
|
|
11
|
+
resource: config.auth.resourceUrl,
|
|
12
|
+
};
|
|
13
|
+
}
|
|
14
|
+
export function verifyStaticToken(token) {
|
|
15
|
+
if (config.auth.staticTokens.length === 0) {
|
|
16
|
+
throw new InvalidTokenError('No static tokens configured');
|
|
17
|
+
}
|
|
18
|
+
const matched = config.auth.staticTokens.some((candidate) => timingSafeEqualUtf8(candidate, token));
|
|
19
|
+
if (!matched) {
|
|
20
|
+
throw new InvalidTokenError('Invalid token');
|
|
21
|
+
}
|
|
22
|
+
return buildStaticAuthInfo(token);
|
|
23
|
+
}
|
package/dist/http/auth.d.ts
CHANGED
|
@@ -1,2 +1,3 @@
|
|
|
1
|
-
import type {
|
|
2
|
-
export declare function createAuthMiddleware(
|
|
1
|
+
import type { RequestHandler, Router } from 'express';
|
|
2
|
+
export declare function createAuthMiddleware(): RequestHandler;
|
|
3
|
+
export declare function createAuthMetadataRouter(): Router | null;
|