@j0hanz/superfetch 2.2.2 → 2.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +363 -363
- package/dist/cache.d.ts +0 -1
- package/dist/cache.js +13 -25
- package/dist/config.d.ts +0 -1
- package/dist/config.js +9 -7
- package/dist/crypto.d.ts +0 -1
- package/dist/crypto.js +0 -1
- package/dist/dom-noise-removal.d.ts +0 -1
- package/dist/dom-noise-removal.js +35 -32
- package/dist/errors.d.ts +0 -1
- package/dist/errors.js +0 -1
- package/dist/fetch.d.ts +0 -1
- package/dist/fetch.js +45 -29
- package/dist/host-normalization.d.ts +1 -0
- package/dist/host-normalization.js +47 -0
- package/dist/http-native.d.ts +0 -1
- package/dist/http-native.js +73 -25
- package/dist/index.d.ts +0 -1
- package/dist/index.js +0 -1
- package/dist/instructions.md +41 -41
- package/dist/json.d.ts +0 -1
- package/dist/json.js +0 -1
- package/dist/language-detection.d.ts +0 -1
- package/dist/language-detection.js +10 -2
- package/dist/markdown-cleanup.d.ts +0 -1
- package/dist/markdown-cleanup.js +10 -10
- package/dist/mcp-validator.d.ts +14 -0
- package/dist/mcp-validator.js +22 -0
- package/dist/mcp.d.ts +0 -1
- package/dist/mcp.js +0 -1
- package/dist/observability.d.ts +0 -1
- package/dist/observability.js +5 -3
- package/dist/server-tuning.d.ts +9 -0
- package/dist/server-tuning.js +30 -0
- package/dist/{http-utils.d.ts → session.d.ts} +0 -25
- package/dist/{http-utils.js → session.js} +11 -104
- package/dist/tools.d.ts +0 -1
- package/dist/tools.js +19 -29
- package/dist/transform-types.d.ts +0 -1
- package/dist/transform-types.js +0 -1
- package/dist/transform.d.ts +0 -1
- package/dist/transform.js +85 -79
- package/dist/type-guards.d.ts +0 -1
- package/dist/type-guards.js +0 -1
- package/dist/workers/transform-worker.d.ts +0 -1
- package/dist/workers/transform-worker.js +29 -19
- package/package.json +85 -85
- package/dist/cache.d.ts.map +0 -1
- package/dist/cache.js.map +0 -1
- package/dist/config.d.ts.map +0 -1
- package/dist/config.js.map +0 -1
- package/dist/crypto.d.ts.map +0 -1
- package/dist/crypto.js.map +0 -1
- package/dist/dom-noise-removal.d.ts.map +0 -1
- package/dist/dom-noise-removal.js.map +0 -1
- package/dist/errors.d.ts.map +0 -1
- package/dist/errors.js.map +0 -1
- package/dist/fetch.d.ts.map +0 -1
- package/dist/fetch.js.map +0 -1
- package/dist/http-native.d.ts.map +0 -1
- package/dist/http-native.js.map +0 -1
- package/dist/http-utils.d.ts.map +0 -1
- package/dist/http-utils.js.map +0 -1
- package/dist/index.d.ts.map +0 -1
- package/dist/index.js.map +0 -1
- package/dist/json.d.ts.map +0 -1
- package/dist/json.js.map +0 -1
- package/dist/language-detection.d.ts.map +0 -1
- package/dist/language-detection.js.map +0 -1
- package/dist/markdown-cleanup.d.ts.map +0 -1
- package/dist/markdown-cleanup.js.map +0 -1
- package/dist/mcp.d.ts.map +0 -1
- package/dist/mcp.js.map +0 -1
- package/dist/observability.d.ts.map +0 -1
- package/dist/observability.js.map +0 -1
- package/dist/tools.d.ts.map +0 -1
- package/dist/tools.js.map +0 -1
- package/dist/transform-types.d.ts.map +0 -1
- package/dist/transform-types.js.map +0 -1
- package/dist/transform.d.ts.map +0 -1
- package/dist/transform.js.map +0 -1
- package/dist/type-guards.d.ts.map +0 -1
- package/dist/type-guards.js.map +0 -1
- package/dist/workers/transform-worker.d.ts.map +0 -1
- package/dist/workers/transform-worker.js.map +0 -1
package/dist/instructions.md
CHANGED
|
@@ -1,41 +1,41 @@
|
|
|
1
|
-
# superFetch Instructions
|
|
2
|
-
|
|
3
|
-
> Guidance for the Agent: These instructions are available as a resource (`internal://instructions`) or prompt (`get-help`). Load them when you are unsure about tool usage.
|
|
4
|
-
|
|
5
|
-
## 1. Core Capability
|
|
6
|
-
|
|
7
|
-
- **Domain:** Fetch public http(s) URLs, extract readable content, and return clean Markdown.
|
|
8
|
-
- **Primary Resources:** `fetch-url` output (`markdown`, `title`, `url`) and cache resources (`superfetch://cache/markdown/{urlHash}`).
|
|
9
|
-
|
|
10
|
-
## 2. The "Golden Path" Workflows (Critical)
|
|
11
|
-
|
|
12
|
-
_Describe the standard order of operations using ONLY tools that exist._
|
|
13
|
-
|
|
14
|
-
### Workflow A: Fetch and Read
|
|
15
|
-
|
|
16
|
-
1. Call `fetch-url` with `url`.
|
|
17
|
-
2. Read `structuredContent.markdown` and `structuredContent.title` from the result.
|
|
18
|
-
3. If content is truncated (look for `...[truncated]`), follow the returned `resource_link` URI.
|
|
19
|
-
> Constraint: Never guess resource URIs. Use the returned `resource_link` or list resources first.
|
|
20
|
-
|
|
21
|
-
### Workflow B: Retrieve Cached Content
|
|
22
|
-
|
|
23
|
-
1. List resources to find available cached pages (`superfetch://cache/...`).
|
|
24
|
-
2. Read the specific `superfetch://cache/markdown/{urlHash}` URI.
|
|
25
|
-
|
|
26
|
-
## 3. Tool Nuances & Gotchas
|
|
27
|
-
|
|
28
|
-
_Do NOT repeat JSON schema. Focus on behavior and pitfalls._
|
|
29
|
-
|
|
30
|
-
- **`fetch-url`**
|
|
31
|
-
- **Purpose:** Fetches a webpage and converts it to clean Markdown format.
|
|
32
|
-
- **Inputs:** `url` (Must be public http/https. Private patterns like localhost/127.0.0.1 are blocked).
|
|
33
|
-
- **Side effects:** Open world network request; writes to internal LRU cache.
|
|
34
|
-
- **Latency/limits:** Network-bound. Large content exceeds inline limits and returns a `resource_link`.
|
|
35
|
-
- **Common failure modes:** `VALIDATION_ERROR` (private/blocked URL), `FETCH_ERROR` (network timeout/404).
|
|
36
|
-
|
|
37
|
-
## 4. Error Handling Strategy
|
|
38
|
-
|
|
39
|
-
- **`VALIDATION_ERROR`**: Ensure the URL is valid and publicly accessible.
|
|
40
|
-
- **`FETCH_ERROR`**: Retry once. If persistent, the site may be blocking automated requests.
|
|
41
|
-
- **Truncation**: If `isError` is false but content ends in `...[truncated]`, you MUST read the provided `resource_link` URI to get the full markdown.
|
|
1
|
+
# superFetch Instructions
|
|
2
|
+
|
|
3
|
+
> Guidance for the Agent: These instructions are available as a resource (`internal://instructions`) or prompt (`get-help`). Load them when you are unsure about tool usage.
|
|
4
|
+
|
|
5
|
+
## 1. Core Capability
|
|
6
|
+
|
|
7
|
+
- **Domain:** Fetch public http(s) URLs, extract readable content, and return clean Markdown.
|
|
8
|
+
- **Primary Resources:** `fetch-url` output (`markdown`, `title`, `url`) and cache resources (`superfetch://cache/markdown/{urlHash}`).
|
|
9
|
+
|
|
10
|
+
## 2. The "Golden Path" Workflows (Critical)
|
|
11
|
+
|
|
12
|
+
_Describe the standard order of operations using ONLY tools that exist._
|
|
13
|
+
|
|
14
|
+
### Workflow A: Fetch and Read
|
|
15
|
+
|
|
16
|
+
1. Call `fetch-url` with `url`.
|
|
17
|
+
2. Read `structuredContent.markdown` and `structuredContent.title` from the result.
|
|
18
|
+
3. If content is truncated (look for `...[truncated]`), follow the returned `resource_link` URI.
|
|
19
|
+
> Constraint: Never guess resource URIs. Use the returned `resource_link` or list resources first.
|
|
20
|
+
|
|
21
|
+
### Workflow B: Retrieve Cached Content
|
|
22
|
+
|
|
23
|
+
1. List resources to find available cached pages (`superfetch://cache/...`).
|
|
24
|
+
2. Read the specific `superfetch://cache/markdown/{urlHash}` URI.
|
|
25
|
+
|
|
26
|
+
## 3. Tool Nuances & Gotchas
|
|
27
|
+
|
|
28
|
+
_Do NOT repeat JSON schema. Focus on behavior and pitfalls._
|
|
29
|
+
|
|
30
|
+
- **`fetch-url`**
|
|
31
|
+
- **Purpose:** Fetches a webpage and converts it to clean Markdown format.
|
|
32
|
+
- **Inputs:** `url` (Must be public http/https. Private patterns like localhost/127.0.0.1 are blocked).
|
|
33
|
+
- **Side effects:** Open world network request; writes to internal LRU cache.
|
|
34
|
+
- **Latency/limits:** Network-bound. Large content exceeds inline limits and returns a `resource_link`.
|
|
35
|
+
- **Common failure modes:** `VALIDATION_ERROR` (private/blocked URL), `FETCH_ERROR` (network timeout/404).
|
|
36
|
+
|
|
37
|
+
## 4. Error Handling Strategy
|
|
38
|
+
|
|
39
|
+
- **`VALIDATION_ERROR`**: Ensure the URL is valid and publicly accessible.
|
|
40
|
+
- **`FETCH_ERROR`**: Retry once. If persistent, the site may be blocking automated requests.
|
|
41
|
+
- **Truncation**: If `isError` is false but content ends in `...[truncated]`, you MUST read the provided `resource_link` URI to get the full markdown.
|
package/dist/json.d.ts
CHANGED
package/dist/json.js
CHANGED
|
@@ -10,4 +10,3 @@ export declare function detectLanguageFromCode(code: string): string | undefined
|
|
|
10
10
|
* Resolve language from HTML attributes (class name and data-language).
|
|
11
11
|
*/
|
|
12
12
|
export declare function resolveLanguageFromAttributes(className: string, dataLang: string): string | undefined;
|
|
13
|
-
//# sourceMappingURL=language-detection.d.ts.map
|
|
@@ -6,7 +6,16 @@
|
|
|
6
6
|
* Check if source contains the given word as a standalone word (not part of another word).
|
|
7
7
|
*/
|
|
8
8
|
function containsWord(source, word) {
|
|
9
|
-
return
|
|
9
|
+
return getWordRegex(word).test(source);
|
|
10
|
+
}
|
|
11
|
+
const WORD_REGEX_CACHE = new Map();
|
|
12
|
+
function getWordRegex(word) {
|
|
13
|
+
const cached = WORD_REGEX_CACHE.get(word);
|
|
14
|
+
if (cached)
|
|
15
|
+
return cached;
|
|
16
|
+
const compiled = new RegExp(`\\b${word}\\b`);
|
|
17
|
+
WORD_REGEX_CACHE.set(word, compiled);
|
|
18
|
+
return compiled;
|
|
10
19
|
}
|
|
11
20
|
/**
|
|
12
21
|
* Extract language from class name (e.g., "language-typescript", "lang-js", "hljs javascript").
|
|
@@ -280,4 +289,3 @@ export function resolveLanguageFromAttributes(className, dataLang) {
|
|
|
280
289
|
const classMatch = extractLanguageFromClassName(className);
|
|
281
290
|
return classMatch ?? resolveLanguageFromDataAttribute(dataLang);
|
|
282
291
|
}
|
|
283
|
-
//# sourceMappingURL=language-detection.js.map
|
|
@@ -16,4 +16,3 @@ export declare function cleanupMarkdownArtifacts(content: string): string;
|
|
|
16
16
|
* Fence-aware: never modifies content inside fenced code blocks.
|
|
17
17
|
*/
|
|
18
18
|
export declare function promoteOrphanHeadings(markdown: string): string;
|
|
19
|
-
//# sourceMappingURL=markdown-cleanup.d.ts.map
|
package/dist/markdown-cleanup.js
CHANGED
|
@@ -165,6 +165,15 @@ function normalizeListsAndSpacing(text) {
|
|
|
165
165
|
// Collapse excessive blank lines
|
|
166
166
|
return text.replace(/\n{3,}/g, '\n\n');
|
|
167
167
|
}
|
|
168
|
+
const CLEANUP_STEPS = [
|
|
169
|
+
fixOrphanHeadings,
|
|
170
|
+
removeEmptyHeadings,
|
|
171
|
+
removeSkipLinksAndEmptyAnchors,
|
|
172
|
+
ensureBlankLineAfterHeadings,
|
|
173
|
+
removeTocBlocks,
|
|
174
|
+
tidyLinksAndEscapes,
|
|
175
|
+
normalizeListsAndSpacing,
|
|
176
|
+
];
|
|
168
177
|
// ─────────────────────────────────────────────────────────────────────────────
|
|
169
178
|
// Public API
|
|
170
179
|
// ─────────────────────────────────────────────────────────────────────────────
|
|
@@ -176,15 +185,7 @@ export function cleanupMarkdownArtifacts(content) {
|
|
|
176
185
|
if (!content)
|
|
177
186
|
return '';
|
|
178
187
|
const cleaned = mapOutsideFences(content, (outside) => {
|
|
179
|
-
|
|
180
|
-
text = fixOrphanHeadings(text);
|
|
181
|
-
text = removeEmptyHeadings(text);
|
|
182
|
-
text = removeSkipLinksAndEmptyAnchors(text);
|
|
183
|
-
text = ensureBlankLineAfterHeadings(text);
|
|
184
|
-
text = removeTocBlocks(text);
|
|
185
|
-
text = tidyLinksAndEscapes(text);
|
|
186
|
-
text = normalizeListsAndSpacing(text);
|
|
187
|
-
return text;
|
|
188
|
+
return CLEANUP_STEPS.reduce((text, step) => step(text), outside);
|
|
188
189
|
});
|
|
189
190
|
return cleaned.trim();
|
|
190
191
|
}
|
|
@@ -280,4 +281,3 @@ export function promoteOrphanHeadings(markdown) {
|
|
|
280
281
|
}
|
|
281
282
|
return result.join('\n');
|
|
282
283
|
}
|
|
283
|
-
//# sourceMappingURL=markdown-cleanup.js.map
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
export type JsonRpcId = string | number | null;
|
|
2
|
+
export interface McpRequestParams {
|
|
3
|
+
_meta?: Record<string, unknown>;
|
|
4
|
+
[key: string]: unknown;
|
|
5
|
+
}
|
|
6
|
+
export interface McpRequestBody {
|
|
7
|
+
jsonrpc: '2.0';
|
|
8
|
+
method: string;
|
|
9
|
+
id?: JsonRpcId;
|
|
10
|
+
params?: McpRequestParams;
|
|
11
|
+
}
|
|
12
|
+
export declare function isJsonRpcBatchRequest(body: unknown): boolean;
|
|
13
|
+
export declare function isMcpRequestBody(body: unknown): body is McpRequestBody;
|
|
14
|
+
export declare function acceptsEventStream(header: string | null | undefined): boolean;
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
import { z } from 'zod';
|
|
2
|
+
// --- Validation ---
|
|
3
|
+
const paramsSchema = z.looseObject({});
|
|
4
|
+
const mcpRequestSchema = z.looseObject({
|
|
5
|
+
jsonrpc: z.literal('2.0'),
|
|
6
|
+
method: z.string().min(1),
|
|
7
|
+
id: z.union([z.string(), z.number(), z.null()]).optional(),
|
|
8
|
+
params: paramsSchema.optional(),
|
|
9
|
+
});
|
|
10
|
+
export function isJsonRpcBatchRequest(body) {
|
|
11
|
+
return Array.isArray(body);
|
|
12
|
+
}
|
|
13
|
+
export function isMcpRequestBody(body) {
|
|
14
|
+
return mcpRequestSchema.safeParse(body).success;
|
|
15
|
+
}
|
|
16
|
+
export function acceptsEventStream(header) {
|
|
17
|
+
if (!header)
|
|
18
|
+
return false;
|
|
19
|
+
return header
|
|
20
|
+
.split(',')
|
|
21
|
+
.some((value) => value.trim().toLowerCase().startsWith('text/event-stream'));
|
|
22
|
+
}
|
package/dist/mcp.d.ts
CHANGED
package/dist/mcp.js
CHANGED
package/dist/observability.d.ts
CHANGED
|
@@ -15,4 +15,3 @@ export declare function logError(message: string, error?: Error | LogMetadata):
|
|
|
15
15
|
export declare function redactUrl(rawUrl: string): string;
|
|
16
16
|
export declare function redactHeaders(headers: Record<string, unknown>): Record<string, unknown>;
|
|
17
17
|
export {};
|
|
18
|
-
//# sourceMappingURL=observability.d.ts.map
|
package/dist/observability.js
CHANGED
|
@@ -13,7 +13,7 @@ export function getSessionId() {
|
|
|
13
13
|
export function getOperationId() {
|
|
14
14
|
return requestContext.getStore()?.operationId;
|
|
15
15
|
}
|
|
16
|
-
function
|
|
16
|
+
function buildContextMetadata() {
|
|
17
17
|
const requestId = getRequestId();
|
|
18
18
|
const sessionId = getSessionId();
|
|
19
19
|
const operationId = getOperationId();
|
|
@@ -24,7 +24,10 @@ function formatMetadata(meta) {
|
|
|
24
24
|
contextMeta.sessionId = sessionId;
|
|
25
25
|
if (operationId)
|
|
26
26
|
contextMeta.operationId = operationId;
|
|
27
|
-
|
|
27
|
+
return contextMeta;
|
|
28
|
+
}
|
|
29
|
+
function formatMetadata(meta) {
|
|
30
|
+
const merged = { ...buildContextMetadata(), ...meta };
|
|
28
31
|
return Object.keys(merged).length > 0 ? ` ${JSON.stringify(merged)}` : '';
|
|
29
32
|
}
|
|
30
33
|
function createTimestamp() {
|
|
@@ -83,4 +86,3 @@ export function redactHeaders(headers) {
|
|
|
83
86
|
}
|
|
84
87
|
return redacted;
|
|
85
88
|
}
|
|
86
|
-
//# sourceMappingURL=observability.js.map
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
export interface HttpServerTuningTarget {
|
|
2
|
+
headersTimeout?: number;
|
|
3
|
+
requestTimeout?: number;
|
|
4
|
+
keepAliveTimeout?: number;
|
|
5
|
+
closeIdleConnections?: () => void;
|
|
6
|
+
closeAllConnections?: () => void;
|
|
7
|
+
}
|
|
8
|
+
export declare function applyHttpServerTuning(server: HttpServerTuningTarget): void;
|
|
9
|
+
export declare function drainConnectionsOnShutdown(server: HttpServerTuningTarget): void;
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
import { config } from './config.js';
|
|
2
|
+
import { logDebug } from './observability.js';
|
|
3
|
+
export function applyHttpServerTuning(server) {
|
|
4
|
+
const { headersTimeoutMs, requestTimeoutMs, keepAliveTimeoutMs } = config.server.http;
|
|
5
|
+
if (headersTimeoutMs !== undefined) {
|
|
6
|
+
server.headersTimeout = headersTimeoutMs;
|
|
7
|
+
}
|
|
8
|
+
if (requestTimeoutMs !== undefined) {
|
|
9
|
+
server.requestTimeout = requestTimeoutMs;
|
|
10
|
+
}
|
|
11
|
+
if (keepAliveTimeoutMs !== undefined) {
|
|
12
|
+
server.keepAliveTimeout = keepAliveTimeoutMs;
|
|
13
|
+
}
|
|
14
|
+
}
|
|
15
|
+
export function drainConnectionsOnShutdown(server) {
|
|
16
|
+
const { shutdownCloseAllConnections, shutdownCloseIdleConnections } = config.server.http;
|
|
17
|
+
if (shutdownCloseAllConnections) {
|
|
18
|
+
if (typeof server.closeAllConnections === 'function') {
|
|
19
|
+
server.closeAllConnections();
|
|
20
|
+
logDebug('Closed all HTTP connections during shutdown');
|
|
21
|
+
}
|
|
22
|
+
return;
|
|
23
|
+
}
|
|
24
|
+
if (shutdownCloseIdleConnections) {
|
|
25
|
+
if (typeof server.closeIdleConnections === 'function') {
|
|
26
|
+
server.closeIdleConnections();
|
|
27
|
+
logDebug('Closed idle HTTP connections during shutdown');
|
|
28
|
+
}
|
|
29
|
+
}
|
|
30
|
+
}
|
|
@@ -1,15 +1,4 @@
|
|
|
1
1
|
import type { StreamableHTTPServerTransport } from '@modelcontextprotocol/sdk/server/streamableHttp.js';
|
|
2
|
-
export type JsonRpcId = string | number | null;
|
|
3
|
-
export interface McpRequestParams {
|
|
4
|
-
_meta?: Record<string, unknown>;
|
|
5
|
-
[key: string]: unknown;
|
|
6
|
-
}
|
|
7
|
-
export interface McpRequestBody {
|
|
8
|
-
jsonrpc: '2.0';
|
|
9
|
-
method: string;
|
|
10
|
-
id?: JsonRpcId;
|
|
11
|
-
params?: McpRequestParams;
|
|
12
|
-
}
|
|
13
2
|
export interface SessionEntry {
|
|
14
3
|
readonly transport: StreamableHTTPServerTransport;
|
|
15
4
|
createdAt: number;
|
|
@@ -34,14 +23,10 @@ export interface SlotTracker {
|
|
|
34
23
|
readonly markInitialized: () => void;
|
|
35
24
|
readonly isInitialized: () => boolean;
|
|
36
25
|
}
|
|
37
|
-
export declare function normalizeHost(value: string): string | null;
|
|
38
26
|
export type CloseHandler = (() => void) | undefined;
|
|
39
27
|
export declare function composeCloseHandlers(first: CloseHandler, second: CloseHandler): CloseHandler;
|
|
40
28
|
export declare function startSessionCleanupLoop(store: SessionStore, sessionTtlMs: number): AbortController;
|
|
41
29
|
export declare function createSessionStore(sessionTtlMs: number): SessionStore;
|
|
42
|
-
export declare function isJsonRpcBatchRequest(body: unknown): boolean;
|
|
43
|
-
export declare function isMcpRequestBody(body: unknown): body is McpRequestBody;
|
|
44
|
-
export declare function acceptsEventStream(header: string | null | undefined): boolean;
|
|
45
30
|
export declare function createSlotTracker(store: SessionStore): SlotTracker;
|
|
46
31
|
export declare function reserveSessionSlot(store: SessionStore, maxSessions: number): boolean;
|
|
47
32
|
export declare function ensureSessionCapacity({ store, maxSessions, evictOldest, }: {
|
|
@@ -49,13 +34,3 @@ export declare function ensureSessionCapacity({ store, maxSessions, evictOldest,
|
|
|
49
34
|
maxSessions: number;
|
|
50
35
|
evictOldest: (store: SessionStore) => boolean;
|
|
51
36
|
}): boolean;
|
|
52
|
-
export interface HttpServerTuningTarget {
|
|
53
|
-
headersTimeout?: number;
|
|
54
|
-
requestTimeout?: number;
|
|
55
|
-
keepAliveTimeout?: number;
|
|
56
|
-
closeIdleConnections?: () => void;
|
|
57
|
-
closeAllConnections?: () => void;
|
|
58
|
-
}
|
|
59
|
-
export declare function applyHttpServerTuning(server: HttpServerTuningTarget): void;
|
|
60
|
-
export declare function drainConnectionsOnShutdown(server: HttpServerTuningTarget): void;
|
|
61
|
-
//# sourceMappingURL=http-utils.d.ts.map
|
|
@@ -1,55 +1,5 @@
|
|
|
1
|
-
import { isIP } from 'node:net';
|
|
2
1
|
import { setInterval as setIntervalPromise } from 'node:timers/promises';
|
|
3
|
-
import {
|
|
4
|
-
import { config } from './config.js';
|
|
5
|
-
import { logDebug, logInfo, logWarn } from './observability.js';
|
|
6
|
-
// --- Host Normalization ---
|
|
7
|
-
export function normalizeHost(value) {
|
|
8
|
-
const trimmed = value.trim().toLowerCase();
|
|
9
|
-
if (!trimmed)
|
|
10
|
-
return null;
|
|
11
|
-
const first = takeFirstHostValue(trimmed);
|
|
12
|
-
if (!first)
|
|
13
|
-
return null;
|
|
14
|
-
const ipv6 = stripIpv6Brackets(first);
|
|
15
|
-
if (ipv6)
|
|
16
|
-
return stripTrailingDots(ipv6);
|
|
17
|
-
if (isIpV6Literal(first)) {
|
|
18
|
-
return stripTrailingDots(first);
|
|
19
|
-
}
|
|
20
|
-
return stripTrailingDots(stripPortIfPresent(first));
|
|
21
|
-
}
|
|
22
|
-
function takeFirstHostValue(value) {
|
|
23
|
-
const first = value.split(',')[0];
|
|
24
|
-
if (!first)
|
|
25
|
-
return null;
|
|
26
|
-
const trimmed = first.trim();
|
|
27
|
-
return trimmed ? trimmed : null;
|
|
28
|
-
}
|
|
29
|
-
function stripIpv6Brackets(value) {
|
|
30
|
-
if (!value.startsWith('['))
|
|
31
|
-
return null;
|
|
32
|
-
const end = value.indexOf(']');
|
|
33
|
-
if (end === -1)
|
|
34
|
-
return null;
|
|
35
|
-
return value.slice(1, end);
|
|
36
|
-
}
|
|
37
|
-
function stripPortIfPresent(value) {
|
|
38
|
-
const colonIndex = value.indexOf(':');
|
|
39
|
-
if (colonIndex === -1)
|
|
40
|
-
return value;
|
|
41
|
-
return value.slice(0, colonIndex);
|
|
42
|
-
}
|
|
43
|
-
function isIpV6Literal(value) {
|
|
44
|
-
return isIP(value) === 6;
|
|
45
|
-
}
|
|
46
|
-
function stripTrailingDots(value) {
|
|
47
|
-
let result = value;
|
|
48
|
-
while (result.endsWith('.')) {
|
|
49
|
-
result = result.slice(0, -1);
|
|
50
|
-
}
|
|
51
|
-
return result;
|
|
52
|
-
}
|
|
2
|
+
import { logInfo, logWarn } from './observability.js';
|
|
53
3
|
export function composeCloseHandlers(first, second) {
|
|
54
4
|
if (!first)
|
|
55
5
|
return second;
|
|
@@ -79,6 +29,13 @@ function handleSessionCleanupError(error) {
|
|
|
79
29
|
error: error instanceof Error ? error.message : 'Unknown error',
|
|
80
30
|
});
|
|
81
31
|
}
|
|
32
|
+
function moveSessionToEnd(sessions, sessionId, session) {
|
|
33
|
+
sessions.delete(sessionId);
|
|
34
|
+
sessions.set(sessionId, session);
|
|
35
|
+
}
|
|
36
|
+
function isSessionExpired(session, now, sessionTtlMs) {
|
|
37
|
+
return now - session.lastSeen > sessionTtlMs;
|
|
38
|
+
}
|
|
82
39
|
async function runSessionCleanupLoop(store, sessionTtlMs, signal) {
|
|
83
40
|
const intervalMs = getCleanupIntervalMs(sessionTtlMs);
|
|
84
41
|
for await (const getNow of setIntervalPromise(intervalMs, Date.now, {
|
|
@@ -115,8 +72,8 @@ export function createSessionStore(sessionTtlMs) {
|
|
|
115
72
|
const session = sessions.get(sessionId);
|
|
116
73
|
if (session) {
|
|
117
74
|
session.lastSeen = Date.now();
|
|
118
|
-
|
|
119
|
-
sessions
|
|
75
|
+
// Move to end (LRU behavior if needed, but Map insertion order)
|
|
76
|
+
moveSessionToEnd(sessions, sessionId, session);
|
|
120
77
|
}
|
|
121
78
|
},
|
|
122
79
|
set: (sessionId, entry) => {
|
|
@@ -145,7 +102,7 @@ export function createSessionStore(sessionTtlMs) {
|
|
|
145
102
|
const now = Date.now();
|
|
146
103
|
const evicted = [];
|
|
147
104
|
for (const [id, session] of sessions.entries()) {
|
|
148
|
-
if (
|
|
105
|
+
if (isSessionExpired(session, now, sessionTtlMs)) {
|
|
149
106
|
sessions.delete(id);
|
|
150
107
|
evicted.push(session);
|
|
151
108
|
}
|
|
@@ -163,27 +120,6 @@ export function createSessionStore(sessionTtlMs) {
|
|
|
163
120
|
},
|
|
164
121
|
};
|
|
165
122
|
}
|
|
166
|
-
// --- Validation ---
|
|
167
|
-
const paramsSchema = z.looseObject({});
|
|
168
|
-
const mcpRequestSchema = z.looseObject({
|
|
169
|
-
jsonrpc: z.literal('2.0'),
|
|
170
|
-
method: z.string().min(1),
|
|
171
|
-
id: z.union([z.string(), z.number(), z.null()]).optional(),
|
|
172
|
-
params: paramsSchema.optional(),
|
|
173
|
-
});
|
|
174
|
-
export function isJsonRpcBatchRequest(body) {
|
|
175
|
-
return Array.isArray(body);
|
|
176
|
-
}
|
|
177
|
-
export function isMcpRequestBody(body) {
|
|
178
|
-
return mcpRequestSchema.safeParse(body).success;
|
|
179
|
-
}
|
|
180
|
-
export function acceptsEventStream(header) {
|
|
181
|
-
if (!header)
|
|
182
|
-
return false;
|
|
183
|
-
return header
|
|
184
|
-
.split(',')
|
|
185
|
-
.some((value) => value.trim().toLowerCase().startsWith('text/event-stream'));
|
|
186
|
-
}
|
|
187
123
|
// --- Slot Tracker ---
|
|
188
124
|
export function createSlotTracker(store) {
|
|
189
125
|
let slotReleased = false;
|
|
@@ -221,32 +157,3 @@ export function ensureSessionCapacity({ store, maxSessions, evictOldest, }) {
|
|
|
221
157
|
}
|
|
222
158
|
return false;
|
|
223
159
|
}
|
|
224
|
-
export function applyHttpServerTuning(server) {
|
|
225
|
-
const { headersTimeoutMs, requestTimeoutMs, keepAliveTimeoutMs } = config.server.http;
|
|
226
|
-
if (headersTimeoutMs !== undefined) {
|
|
227
|
-
server.headersTimeout = headersTimeoutMs;
|
|
228
|
-
}
|
|
229
|
-
if (requestTimeoutMs !== undefined) {
|
|
230
|
-
server.requestTimeout = requestTimeoutMs;
|
|
231
|
-
}
|
|
232
|
-
if (keepAliveTimeoutMs !== undefined) {
|
|
233
|
-
server.keepAliveTimeout = keepAliveTimeoutMs;
|
|
234
|
-
}
|
|
235
|
-
}
|
|
236
|
-
export function drainConnectionsOnShutdown(server) {
|
|
237
|
-
const { shutdownCloseAllConnections, shutdownCloseIdleConnections } = config.server.http;
|
|
238
|
-
if (shutdownCloseAllConnections) {
|
|
239
|
-
if (typeof server.closeAllConnections === 'function') {
|
|
240
|
-
server.closeAllConnections();
|
|
241
|
-
logDebug('Closed all HTTP connections during shutdown');
|
|
242
|
-
}
|
|
243
|
-
return;
|
|
244
|
-
}
|
|
245
|
-
if (shutdownCloseIdleConnections) {
|
|
246
|
-
if (typeof server.closeIdleConnections === 'function') {
|
|
247
|
-
server.closeIdleConnections();
|
|
248
|
-
logDebug('Closed idle HTTP connections during shutdown');
|
|
249
|
-
}
|
|
250
|
-
}
|
|
251
|
-
}
|
|
252
|
-
//# sourceMappingURL=http-utils.js.map
|
package/dist/tools.d.ts
CHANGED
|
@@ -125,4 +125,3 @@ export declare function fetchUrlToolHandler(input: FetchUrlInput, extra?: ToolHa
|
|
|
125
125
|
export declare function withRequestContextIfMissing<TParams, TResult, TExtra = unknown>(handler: (params: TParams, extra?: TExtra) => Promise<TResult>): (params: TParams, extra?: TExtra) => Promise<TResult>;
|
|
126
126
|
export declare function registerTools(server: McpServer): void;
|
|
127
127
|
export {};
|
|
128
|
-
//# sourceMappingURL=tools.d.ts.map
|
package/dist/tools.js
CHANGED
|
@@ -106,23 +106,16 @@ function buildEmbeddedResource(content, url, title) {
|
|
|
106
106
|
},
|
|
107
107
|
};
|
|
108
108
|
}
|
|
109
|
-
function
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
if (!url)
|
|
119
|
-
return;
|
|
120
|
-
const embeddedResource = buildEmbeddedResource(contentToEmbed, url, title);
|
|
121
|
-
if (embeddedResource) {
|
|
122
|
-
blocks.push(embeddedResource);
|
|
109
|
+
function appendResourceBlocks({ blocks, inlineResult, resourceName, url, title, fullContent, }) {
|
|
110
|
+
const contentToEmbed = config.runtime.httpMode
|
|
111
|
+
? inlineResult.content
|
|
112
|
+
: (fullContent ?? inlineResult.content);
|
|
113
|
+
if (contentToEmbed && url) {
|
|
114
|
+
const embeddedResource = buildEmbeddedResource(contentToEmbed, url, title);
|
|
115
|
+
if (embeddedResource) {
|
|
116
|
+
blocks.push(embeddedResource);
|
|
117
|
+
}
|
|
123
118
|
}
|
|
124
|
-
}
|
|
125
|
-
function maybeAppendResourceLink(blocks, inlineResult, resourceName) {
|
|
126
119
|
const resourceLink = buildResourceLink(inlineResult, resourceName);
|
|
127
120
|
if (resourceLink) {
|
|
128
121
|
blocks.push(resourceLink);
|
|
@@ -136,9 +129,14 @@ function buildTextBlock(structuredContent) {
|
|
|
136
129
|
}
|
|
137
130
|
function buildToolContentBlocks(structuredContent, fromCache, inlineResult, resourceName, cacheKey, fullContent, url, title) {
|
|
138
131
|
const blocks = [buildTextBlock(structuredContent)];
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
132
|
+
appendResourceBlocks({
|
|
133
|
+
blocks,
|
|
134
|
+
inlineResult,
|
|
135
|
+
resourceName,
|
|
136
|
+
url,
|
|
137
|
+
title,
|
|
138
|
+
fullContent,
|
|
139
|
+
});
|
|
142
140
|
return blocks;
|
|
143
141
|
}
|
|
144
142
|
function applyInlineContentLimit(content, cacheKey) {
|
|
@@ -266,14 +264,6 @@ function logRawUrlTransformation(resolvedUrl) {
|
|
|
266
264
|
original: resolvedUrl.originalUrl,
|
|
267
265
|
});
|
|
268
266
|
}
|
|
269
|
-
function applyOptionalPipelineSerialization(pipelineOptions, options) {
|
|
270
|
-
if (options.serialize !== undefined) {
|
|
271
|
-
pipelineOptions.serialize = options.serialize;
|
|
272
|
-
}
|
|
273
|
-
if (options.deserialize !== undefined) {
|
|
274
|
-
pipelineOptions.deserialize = options.deserialize;
|
|
275
|
-
}
|
|
276
|
-
}
|
|
277
267
|
export async function performSharedFetch(options, deps = {}) {
|
|
278
268
|
const executePipeline = deps.executeFetchPipeline ?? executeFetchPipeline;
|
|
279
269
|
const pipelineOptions = {
|
|
@@ -281,8 +271,9 @@ export async function performSharedFetch(options, deps = {}) {
|
|
|
281
271
|
cacheNamespace: 'markdown',
|
|
282
272
|
...(options.signal === undefined ? {} : { signal: options.signal }),
|
|
283
273
|
transform: options.transform,
|
|
274
|
+
...(options.serialize ? { serialize: options.serialize } : {}),
|
|
275
|
+
...(options.deserialize ? { deserialize: options.deserialize } : {}),
|
|
284
276
|
};
|
|
285
|
-
applyOptionalPipelineSerialization(pipelineOptions, options);
|
|
286
277
|
const pipeline = await executePipeline(pipelineOptions);
|
|
287
278
|
const inlineResult = applyInlineContentLimit(pipeline.data.content, pipeline.cacheKey ?? null);
|
|
288
279
|
return { pipeline, inlineResult };
|
|
@@ -485,4 +476,3 @@ export function registerTools(server) {
|
|
|
485
476
|
annotations: TOOL_DEFINITION.annotations,
|
|
486
477
|
}, withRequestContextIfMissing(TOOL_DEFINITION.handler));
|
|
487
478
|
}
|
|
488
|
-
//# sourceMappingURL=tools.js.map
|
package/dist/transform-types.js
CHANGED
package/dist/transform.d.ts
CHANGED
|
@@ -23,4 +23,3 @@ export declare function createContentMetadataBlock(url: string, article: Extract
|
|
|
23
23
|
export declare function transformHtmlToMarkdownInProcess(html: string, url: string, options: TransformOptions): MarkdownTransformResult;
|
|
24
24
|
export declare function shutdownTransformWorkerPool(): Promise<void>;
|
|
25
25
|
export declare function transformHtmlToMarkdown(html: string, url: string, options: TransformOptions): Promise<MarkdownTransformResult>;
|
|
26
|
-
//# sourceMappingURL=transform.d.ts.map
|