@j0hanz/fetch-url-mcp 1.6.1 → 1.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +171 -181
- package/dist/cli.d.ts +1 -0
- package/dist/cli.d.ts.map +1 -0
- package/dist/http/auth.d.ts +12 -0
- package/dist/http/auth.d.ts.map +1 -0
- package/dist/http/auth.js +84 -5
- package/dist/http/health.d.ts +1 -0
- package/dist/http/health.d.ts.map +1 -0
- package/dist/http/helpers.d.ts +1 -0
- package/dist/http/helpers.d.ts.map +1 -0
- package/dist/http/native.d.ts +1 -0
- package/dist/http/native.d.ts.map +1 -0
- package/dist/http/native.js +80 -63
- package/dist/http/rate-limit.d.ts +1 -0
- package/dist/http/rate-limit.d.ts.map +1 -0
- package/dist/index.d.ts +1 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/lib/content.d.ts +3 -0
- package/dist/lib/content.d.ts.map +1 -0
- package/dist/lib/content.js +16 -11
- package/dist/lib/core.d.ts +5 -8
- package/dist/lib/core.d.ts.map +1 -0
- package/dist/lib/core.js +110 -97
- package/dist/lib/fetch-pipeline.d.ts +1 -1
- package/dist/lib/fetch-pipeline.d.ts.map +1 -0
- package/dist/lib/fetch-pipeline.js +54 -44
- package/dist/lib/http.d.ts +1 -0
- package/dist/lib/http.d.ts.map +1 -0
- package/dist/lib/mcp-tools.d.ts +45 -7
- package/dist/lib/mcp-tools.d.ts.map +1 -0
- package/dist/lib/mcp-tools.js +37 -6
- package/dist/lib/net-utils.d.ts +1 -0
- package/dist/lib/net-utils.d.ts.map +1 -0
- package/dist/lib/progress.d.ts +1 -0
- package/dist/lib/progress.d.ts.map +1 -0
- package/dist/lib/task-handlers.d.ts +3 -2
- package/dist/lib/task-handlers.d.ts.map +1 -0
- package/dist/lib/task-handlers.js +19 -36
- package/dist/lib/types.d.ts +4 -0
- package/dist/lib/types.d.ts.map +1 -0
- package/dist/lib/types.js +12 -1
- package/dist/lib/url.d.ts +3 -0
- package/dist/lib/url.d.ts.map +1 -0
- package/dist/lib/url.js +78 -151
- package/dist/lib/utils.d.ts +2 -2
- package/dist/lib/utils.d.ts.map +1 -0
- package/dist/lib/utils.js +60 -94
- package/dist/lib/zod.d.ts +3 -0
- package/dist/lib/zod.d.ts.map +1 -0
- package/dist/lib/zod.js +33 -0
- package/dist/prompts/index.d.ts +2 -1
- package/dist/prompts/index.d.ts.map +1 -0
- package/dist/prompts/index.js +2 -13
- package/dist/resources/index.d.ts +2 -1
- package/dist/resources/index.d.ts.map +1 -0
- package/dist/resources/index.js +5 -19
- package/dist/resources/instructions.d.ts +1 -0
- package/dist/resources/instructions.d.ts.map +1 -0
- package/dist/resources/instructions.js +2 -0
- package/dist/schemas/cache.d.ts +18 -0
- package/dist/schemas/cache.d.ts.map +1 -0
- package/dist/schemas/cache.js +19 -0
- package/dist/schemas/inputs.d.ts +1 -0
- package/dist/schemas/inputs.d.ts.map +1 -0
- package/dist/schemas/outputs.d.ts +6 -5
- package/dist/schemas/outputs.d.ts.map +1 -0
- package/dist/schemas/outputs.js +5 -9
- package/dist/server.d.ts +1 -0
- package/dist/server.d.ts.map +1 -0
- package/dist/server.js +6 -6
- package/dist/tasks/execution.d.ts +1 -0
- package/dist/tasks/execution.d.ts.map +1 -0
- package/dist/tasks/execution.js +3 -21
- package/dist/tasks/manager.d.ts +2 -6
- package/dist/tasks/manager.d.ts.map +1 -0
- package/dist/tasks/manager.js +2 -4
- package/dist/tasks/owner.d.ts +1 -0
- package/dist/tasks/owner.d.ts.map +1 -0
- package/dist/tasks/tool-registry.d.ts +1 -0
- package/dist/tasks/tool-registry.d.ts.map +1 -0
- package/dist/tools/fetch-url.d.ts +4 -6
- package/dist/tools/fetch-url.d.ts.map +1 -0
- package/dist/tools/fetch-url.js +46 -44
- package/dist/tools/index.d.ts +1 -0
- package/dist/tools/index.d.ts.map +1 -0
- package/dist/transform/html-translators.d.ts +1 -0
- package/dist/transform/html-translators.d.ts.map +1 -0
- package/dist/transform/html-translators.js +5 -2
- package/dist/transform/metadata.d.ts +1 -0
- package/dist/transform/metadata.d.ts.map +1 -0
- package/dist/transform/metadata.js +1 -0
- package/dist/transform/{workers/shared.d.ts → shared.d.ts} +2 -1
- package/dist/transform/shared.d.ts.map +1 -0
- package/dist/transform/{workers/shared.js → shared.js} +1 -1
- package/dist/transform/transform.d.ts +1 -0
- package/dist/transform/transform.d.ts.map +1 -0
- package/dist/transform/transform.js +21 -14
- package/dist/transform/types.d.ts +1 -4
- package/dist/transform/types.d.ts.map +1 -0
- package/dist/transform/worker-pool.d.ts +3 -18
- package/dist/transform/worker-pool.d.ts.map +1 -0
- package/dist/transform/worker-pool.js +51 -167
- package/package.json +9 -6
- package/dist/transform/workers/transform-child.d.ts +0 -1
- package/dist/transform/workers/transform-child.js +0 -15
- package/dist/transform/workers/transform-worker.d.ts +0 -1
- package/dist/transform/workers/transform-worker.js +0 -13
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"core.d.ts","sourceRoot":"","sources":["../../src/lib/core.ts"],"names":[],"mappings":"AAYA,OAAO,EAAE,KAAK,SAAS,EAAE,MAAM,yCAAyC,CAAC;AACzE,OAAO,EAAE,KAAK,6BAA6B,EAAE,MAAM,oDAAoD,CAAC;AAExG,OAAO,EAAE,KAAK,aAAa,EAAuB,MAAM,qBAAqB,CAAC;AAe9E,eAAO,MAAM,aAAa,EAAE,MAA2C,CAAC;AACxE,KAAK,QAAQ,GAAG,OAAO,GAAG,MAAM,GAAG,MAAM,GAAG,OAAO,CAAC;AAwBpD,KAAK,mBAAmB,GAAG,SAAS,GAAG,SAAS,CAAC;AACjD,KAAK,QAAQ,GAAG,OAAO,GAAG,QAAQ,CAAC;AAyNnC,UAAU,oBAAoB;IAC5B,sBAAsB,CAAC,EAAE,MAAM,CAAC;IAChC,wBAAwB,CAAC,EAAE,MAAM,CAAC;IAClC,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AA+BD,UAAU,UAAU;IAClB,IAAI,EAAE,QAAQ,CAAC;IACf,SAAS,EAAE,GAAG,GAAG,SAAS,CAAC;IAC3B,gBAAgB,EAAE,GAAG,GAAG,SAAS,CAAC;IAClC,QAAQ,EAAE,GAAG,GAAG,SAAS,CAAC;IAC1B,aAAa,EAAE,GAAG,GAAG,SAAS,CAAC;IAC/B,eAAe,EAAE,GAAG,GAAG,SAAS,CAAC;IACjC,gBAAgB,EAAE,GAAG,GAAG,SAAS,CAAC;IAClC,WAAW,EAAE,GAAG,CAAC;IACjB,cAAc,EAAE,MAAM,EAAE,CAAC;IACzB,QAAQ,EAAE,MAAM,GAAG,SAAS,CAAC;IAC7B,YAAY,EAAE,MAAM,GAAG,SAAS,CAAC;IACjC,sBAAsB,EAAE,MAAM,CAAC;IAC/B,YAAY,EAAE,MAAM,EAAE,CAAC;CACxB;AAED,UAAU,WAAW;IACnB,OAAO,EAAE,OAAO,CAAC;IACjB,OAAO,EAAE,MAAM,GAAG,SAAS,CAAC;IAC5B,QAAQ,EAAE,MAAM,GAAG,SAAS,CAAC;IAC7B,MAAM,EAAE,MAAM,GAAG,SAAS,CAAC;CAC5B;AA8ID,UAAU,YAAY;IACpB,QAAQ,EAAE,OAAO,CAAC;CACnB;AAKD,eAAO,MAAM,MAAM;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CA2HlB,CAAC;AACF,wBAAgB,cAAc,IAAI,IAAI,CAErC;AAED,UAAU,UAAU;IAClB,GAAG,EAAE,MAAM,CAAC;IACZ,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,MAAM,CAAC;IAChB,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;CACnB;AACD,UAAU,aAAa;IACrB,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;CACjB;AACD,UAAU,eAAe;IACvB,KAAK,CAAC,EAAE,OAAO,CAAC;CACjB;AACD,UAAU,eAAe;IACvB,KAAK,CAAC,EAAE,OAAO,CAAC;CACjB;AACD,UAAU,kBAAkB;IAC1B,GAAG,EAAE,MAAM,CAAC;IACZ,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAID,UAAU,gBAAgB;IACxB,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;IAChB,WAAW,EAAE,OAAO,CAAC;CACtB;AACD,KAAK,mBAAmB,GAAG,CAAC,KAAK,EAAE,gBAAgB,KAAK,OAAO,CAAC;AAKhE,wBAAgB,kBAAkB,CAAC,GAAG,EAAE,MAAM,GAAG,aAAa,GAAG,IAAI,CAQpE;AACD,wBAAgB,2BAA2B,CACzC,OAAO,EAAE,aAAa,GACrB,MAAM,GAAG,IAAI,CAEf;AAwBD,wBAAgB,cAAc,CAC5B,SAAS,EAAE,MAAM,EACjB,GAAG,EAAE,MAAM,EACX,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,MAAM,GACtC,MAAM,GAAG,IAAI,CAiBf;AACD,wBAAgB,aAAa,CAAC,QAAQ,EAAE,MAAM,GAAG,aAAa,GAAG,IAAI,CASpE;AAoLD,wBAAgB,aAAa,CAAC,QAAQ,EAAE,mBAAmB,GAAG,MAAM,IAAI,CAEvE;AACD,wBAAgB,GAAG,CACjB,QAAQ,EAAE,MAAM,GAAG,IAAI,EACvB,OAAO,CAAC,EAAE,eAAe,GACxB,UAAU,GAAG,SAAS,CAExB;AACD,wBAAgB,GAAG,CACjB,QAAQ,EAAE,MAAM,GAAG,IAAI,EACvB,OAAO,EAAE,MAAM,EACf,QAAQ,EAAE,kBAAkB,EAC5B,OAAO,CAAC,EAAE,eAAe,GACxB,IAAI,CAEN;AACD,wBAAgB,IAAI,IAAI,SAAS,MAAM,EAAE,CAExC;AACD,wBAAgB,YAAY,CAC1B,QAAQ,EAAE,MAAM,GACf;IAAE,GAAG,EAAE,MAAM,CAAC;IAAC,KAAK,CAAC,EAAE,MAAM,CAAC;IAAC,SAAS,CAAC,EAAE,MAAM,CAAA;CAAE,GAAG,SAAS,CAQjE;AACD,wBAAgB,SAAS,IAAI,OAAO,CAEnC;AA4BD,KAAK,WAAW,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;AAC3C,UAAU,cAAc;IACtB,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,CAAC;CAC/B;AAYD,wBAAgB,YAAY,CAAC,MAAM,EAAE,SAAS,GAAG,IAAI,CAEpD;AACD,wBAAgB,wBAAwB,CACtC,SAAS,EAAE,MAAM,EACjB,MAAM,EAAE,SAAS,GAChB,IAAI,CAGN;AACD,wBAAgB,0BAA0B,CAAC,SAAS,EAAE,MAAM,GAAG,IAAI,CAIlE;AACD,wBAAgB,kCAAkC,CAAC,MAAM,EAAE,SAAS,GAAG,IAAI,CAM1E;AACD,wBAAgB,2BAA2B,CACzC,MAAM,EAAE,SAAS,GAChB,MAAM,GAAG,SAAS,CAKpB;AACD,wBAAgB,qBAAqB,CAAC,CAAC,EACrC,OAAO,EAAE,cAAc,EACvB,EAAE,EAAE,MAAM,CAAC,GACV,CAAC,CAEH;AAID,wBAAgB,YAAY,IAAI,MAAM,GAAG,SAAS,CAGjD;AACD,wBAAgB,YAAY,IAAI,MAAM,GAAG,SAAS,CAEjD;AACD,wBAAgB,cAAc,IAAI,MAAM,GAAG,SAAS,CAEnD;AA2KD,wBAAgB,OAAO,CAAC,OAAO,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,WAAW,GAAG,IAAI,CAEjE;AACD,wBAAgB,QAAQ,CAAC,OAAO,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,WAAW,GAAG,IAAI,CAElE;AACD,wBAAgB,OAAO,CAAC,OAAO,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,WAAW,GAAG,IAAI,CAEjE;AAcD,wBAAgB,QAAQ,CAAC,OAAO,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,KAAK,GAAG,WAAW,GAAG,IAAI,CAI3E;AACD,wBAAgB,cAAc,CAAC,SAAS,CAAC,EAAE,MAAM,GAAG,QAAQ,CAE3D;AACD,wBAAgB,WAAW,CAAC,KAAK,EAAE,MAAM,EAAE,SAAS,CAAC,EAAE,MAAM,GAAG,IAAI,CAUnE;AACD,wBAAgB,SAAS,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,CAWhD;AACD,MAAM,WAAW,YAAY;IAC3B,QAAQ,CAAC,MAAM,EAAE,SAAS,CAAC;IAC3B,QAAQ,CAAC,SAAS,EAAE,6BAA6B,CAAC;IAClD,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,EAAE,MAAM,CAAC;IACjB,mBAAmB,EAAE,OAAO,CAAC;IAC7B,yBAAyB,EAAE,MAAM,CAAC;IAClC,eAAe,EAAE,MAAM,CAAC;CACzB;AACD,MAAM,WAAW,YAAY;IAC3B,GAAG,EAAE,CAAC,SAAS,EAAE,MAAM,KAAK,YAAY,GAAG,SAAS,CAAC;IACrD,KAAK,EAAE,CAAC,SAAS,EAAE,MAAM,KAAK,IAAI,CAAC;IACnC,GAAG,EAAE,CAAC,SAAS,EAAE,MAAM,EAAE,KAAK,EAAE,YAAY,KAAK,IAAI,CAAC;IACtD,MAAM,EAAE,CAAC,SAAS,EAAE,MAAM,KAAK,YAAY,GAAG,SAAS,CAAC;IACxD,IAAI,EAAE,MAAM,MAAM,CAAC;IACnB,QAAQ,EAAE,MAAM,MAAM,CAAC;IACvB,iBAAiB,EAAE,MAAM,IAAI,CAAC;IAC9B,iBAAiB,EAAE,MAAM,IAAI,CAAC;IAC9B,KAAK,EAAE,MAAM,YAAY,EAAE,CAAC;IAC5B,YAAY,EAAE,MAAM,YAAY,EAAE,CAAC;IACnC,WAAW,EAAE,MAAM,YAAY,GAAG,SAAS,CAAC;CAC7C;AACD,UAAU,WAAW;IACnB,QAAQ,CAAC,WAAW,EAAE,MAAM,IAAI,CAAC;IACjC,QAAQ,CAAC,eAAe,EAAE,MAAM,IAAI,CAAC;IACrC,QAAQ,CAAC,aAAa,EAAE,MAAM,OAAO,CAAC;CACvC;AACD,KAAK,YAAY,GAAG,CAAC,MAAM,IAAI,CAAC,GAAG,SAAS,CAAC;AAC7C,wBAAgB,oBAAoB,CAClC,KAAK,EAAE,YAAY,EACnB,MAAM,EAAE,YAAY,GACnB,YAAY,CAWd;AAgID,wBAAgB,uBAAuB,CACrC,KAAK,EAAE,YAAY,EACnB,YAAY,EAAE,MAAM,EACpB,OAAO,CAAC,EAAE;IACR,cAAc,CAAC,EAAE,CAAC,OAAO,EAAE,YAAY,KAAK,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC;IACjE,iBAAiB,CAAC,EAAE,MAAM,CAAC;CAC5B,GACA,eAAe,CAOjB;AA8FD,wBAAgB,kBAAkB,CAAC,YAAY,EAAE,MAAM,GAAG,YAAY,CAErE;AAqBD,wBAAgB,iBAAiB,CAAC,KAAK,EAAE,YAAY,GAAG,WAAW,CAElE;AAID,wBAAgB,kBAAkB,CAChC,KAAK,EAAE,YAAY,EACnB,WAAW,EAAE,MAAM,GAClB,OAAO,CAMT;AAID,wBAAgB,qBAAqB,CAAC,EACpC,KAAK,EACL,WAAW,EACX,WAAW,GACZ,EAAE;IACD,KAAK,EAAE,YAAY,CAAC;IACpB,WAAW,EAAE,MAAM,CAAC;IACpB,WAAW,EAAE,CAAC,KAAK,EAAE,YAAY,KAAK,OAAO,CAAC;CAC/C,GAAG,OAAO,CAeV"}
|
package/dist/lib/core.js
CHANGED
|
@@ -4,10 +4,10 @@ import { accessSync, constants as fsConstants, readFileSync } from 'node:fs';
|
|
|
4
4
|
import { findPackageJSON } from 'node:module';
|
|
5
5
|
import { isIP } from 'node:net';
|
|
6
6
|
import process from 'node:process';
|
|
7
|
-
import { inspect, stripVTControlCharacters } from 'node:util';
|
|
7
|
+
import { getSystemErrorMessage, inspect, stripVTControlCharacters, } from 'node:util';
|
|
8
8
|
import {} from '@modelcontextprotocol/sdk/server/mcp.js';
|
|
9
9
|
import {} from '@modelcontextprotocol/sdk/server/streamableHttp.js';
|
|
10
|
-
import {
|
|
10
|
+
import { cachedPayloadSchema } from '../schemas/cache.js';
|
|
11
11
|
import { buildIpv4, normalizeHostname, stripTrailingDots, } from './net-utils.js';
|
|
12
12
|
import { getErrorMessage, isAbortError, sha256Hex, stableStringify as stableJsonStringify, startAbortableIntervalLoop, } from './utils.js';
|
|
13
13
|
export const serverVersion = readServerVersion(import.meta.url);
|
|
@@ -66,29 +66,22 @@ function normalizeHostValue(value) {
|
|
|
66
66
|
const raw = value.trim();
|
|
67
67
|
if (!raw)
|
|
68
68
|
return null;
|
|
69
|
-
|
|
70
|
-
if (raw.includes('://')) {
|
|
71
|
-
if (!URL.canParse(raw))
|
|
72
|
-
return null;
|
|
69
|
+
if (raw.includes('://') && URL.canParse(raw)) {
|
|
73
70
|
return normalizeHostname(new URL(raw).hostname);
|
|
74
71
|
}
|
|
75
|
-
// host[:port]
|
|
76
72
|
const candidateUrl = `http://${raw}`;
|
|
77
73
|
if (URL.canParse(candidateUrl)) {
|
|
78
74
|
return normalizeHostname(new URL(candidateUrl).hostname);
|
|
79
75
|
}
|
|
80
76
|
const lowered = raw.toLowerCase();
|
|
81
|
-
// [::1]:port
|
|
82
77
|
if (lowered.startsWith('[')) {
|
|
83
78
|
const end = lowered.indexOf(']');
|
|
84
79
|
if (end === -1)
|
|
85
80
|
return null;
|
|
86
81
|
return normalizeHostname(lowered.slice(1, end));
|
|
87
82
|
}
|
|
88
|
-
// Bare IPv6
|
|
89
83
|
if (isIP(lowered) === 6)
|
|
90
84
|
return stripTrailingDots(lowered);
|
|
91
|
-
// Split host:port (single colon only)
|
|
92
85
|
const firstColon = lowered.indexOf(':');
|
|
93
86
|
if (firstColon === -1)
|
|
94
87
|
return normalizeHostname(lowered);
|
|
@@ -126,7 +119,7 @@ function parseList(envValue) {
|
|
|
126
119
|
return envValue
|
|
127
120
|
.split(/[\s,]+/)
|
|
128
121
|
.map((entry) => entry.trim())
|
|
129
|
-
.filter(
|
|
122
|
+
.filter(Boolean);
|
|
130
123
|
}
|
|
131
124
|
function parseListOrDefault(envValue, defaultValue) {
|
|
132
125
|
const parsed = parseList(envValue);
|
|
@@ -177,13 +170,9 @@ function readUrlEnv(name) {
|
|
|
177
170
|
return parseUrlEnv(env[name], name);
|
|
178
171
|
}
|
|
179
172
|
function parseAllowedHosts(envValue) {
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
if (normalized)
|
|
184
|
-
hosts.add(normalized);
|
|
185
|
-
}
|
|
186
|
-
return hosts;
|
|
173
|
+
return new Set(parseList(envValue)
|
|
174
|
+
.map(normalizeHostValue)
|
|
175
|
+
.filter((h) => h !== null));
|
|
187
176
|
}
|
|
188
177
|
function readOptionalFilePath(value) {
|
|
189
178
|
if (!value)
|
|
@@ -268,10 +257,10 @@ function resolveAuthMode(urls) {
|
|
|
268
257
|
return oauthConfigured ? 'oauth' : 'static';
|
|
269
258
|
}
|
|
270
259
|
function collectStaticTokens() {
|
|
271
|
-
const
|
|
260
|
+
const tokens = parseList(env['ACCESS_TOKENS']);
|
|
272
261
|
if (env['API_KEY'])
|
|
273
|
-
|
|
274
|
-
return [...
|
|
262
|
+
tokens.push(env['API_KEY']);
|
|
263
|
+
return [...new Set(tokens)];
|
|
275
264
|
}
|
|
276
265
|
function buildAuthConfig(baseUrl) {
|
|
277
266
|
const urls = readOAuthUrls(baseUrl);
|
|
@@ -453,11 +442,6 @@ export const config = {
|
|
|
453
442
|
export function enableHttpMode() {
|
|
454
443
|
runtimeState.httpMode = true;
|
|
455
444
|
}
|
|
456
|
-
const CachedPayloadSchema = z.strictObject({
|
|
457
|
-
content: z.string().optional(),
|
|
458
|
-
markdown: z.string().optional(),
|
|
459
|
-
title: z.string().optional(),
|
|
460
|
-
});
|
|
461
445
|
const CACHE_CONSTANTS = {
|
|
462
446
|
URL_HASH_LENGTH: 32,
|
|
463
447
|
VARY_HASH_LENGTH: 16,
|
|
@@ -465,7 +449,8 @@ const CACHE_CONSTANTS = {
|
|
|
465
449
|
export function parseCachedPayload(raw) {
|
|
466
450
|
try {
|
|
467
451
|
const parsed = JSON.parse(raw);
|
|
468
|
-
|
|
452
|
+
const result = cachedPayloadSchema.safeParse(parsed);
|
|
453
|
+
return result.success ? result.data : null;
|
|
469
454
|
}
|
|
470
455
|
catch {
|
|
471
456
|
return null;
|
|
@@ -496,16 +481,14 @@ export function createCacheKey(namespace, url, vary) {
|
|
|
496
481
|
if (!namespace || !url)
|
|
497
482
|
return null;
|
|
498
483
|
const urlHash = createHashFragment(url, CACHE_CONSTANTS.URL_HASH_LENGTH);
|
|
499
|
-
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
|
|
506
|
-
|
|
507
|
-
}
|
|
508
|
-
return buildCacheKey(namespace, urlHash, varyHash);
|
|
484
|
+
if (!vary)
|
|
485
|
+
return buildCacheKey(namespace, urlHash);
|
|
486
|
+
const varyString = resolveVaryString(vary);
|
|
487
|
+
if (varyString === null)
|
|
488
|
+
return null;
|
|
489
|
+
return buildCacheKey(namespace, urlHash, varyString
|
|
490
|
+
? createHashFragment(varyString, CACHE_CONSTANTS.VARY_HASH_LENGTH)
|
|
491
|
+
: undefined);
|
|
509
492
|
}
|
|
510
493
|
export function parseCacheKey(cacheKey) {
|
|
511
494
|
if (!cacheKey)
|
|
@@ -697,10 +680,9 @@ export function isEnabled() {
|
|
|
697
680
|
return store.isEnabled();
|
|
698
681
|
}
|
|
699
682
|
function hasPackageJsonVersion(value) {
|
|
700
|
-
|
|
701
|
-
|
|
702
|
-
|
|
703
|
-
return typeof record.version === 'string';
|
|
683
|
+
return (typeof value === 'object' &&
|
|
684
|
+
value !== null &&
|
|
685
|
+
typeof value.version === 'string');
|
|
704
686
|
}
|
|
705
687
|
function readServerVersion(moduleUrl) {
|
|
706
688
|
const packageJsonPath = findPackageJSON(moduleUrl);
|
|
@@ -723,6 +705,8 @@ const requestContext = new AsyncLocalStorage({
|
|
|
723
705
|
});
|
|
724
706
|
let mcpServer;
|
|
725
707
|
const sessionServers = new Map();
|
|
708
|
+
const sessionMcpLogLevels = new Map();
|
|
709
|
+
let stdioMcpLogLevel;
|
|
726
710
|
let stderrAvailable = true;
|
|
727
711
|
process.stderr.on('error', () => {
|
|
728
712
|
stderrAvailable = false;
|
|
@@ -739,12 +723,14 @@ export function unregisterMcpSessionServer(sessionId) {
|
|
|
739
723
|
if (!sessionId)
|
|
740
724
|
return;
|
|
741
725
|
sessionServers.delete(sessionId);
|
|
726
|
+
sessionMcpLogLevels.delete(sessionId);
|
|
742
727
|
}
|
|
743
728
|
export function unregisterMcpSessionServerByServer(server) {
|
|
744
729
|
for (const [sessionId, mappedServer] of sessionServers.entries()) {
|
|
745
730
|
if (mappedServer !== server)
|
|
746
731
|
continue;
|
|
747
732
|
sessionServers.delete(sessionId);
|
|
733
|
+
sessionMcpLogLevels.delete(sessionId);
|
|
748
734
|
}
|
|
749
735
|
}
|
|
750
736
|
export function resolveMcpSessionIdByServer(server) {
|
|
@@ -764,7 +750,7 @@ export function getRequestId() {
|
|
|
764
750
|
const context = getRequestContext();
|
|
765
751
|
return context?.requestId;
|
|
766
752
|
}
|
|
767
|
-
function getSessionId() {
|
|
753
|
+
export function getSessionId() {
|
|
768
754
|
return getRequestContext()?.sessionId;
|
|
769
755
|
}
|
|
770
756
|
export function getOperationId() {
|
|
@@ -834,22 +820,51 @@ const LEVEL_PRIORITY = {
|
|
|
834
820
|
function shouldLog(level) {
|
|
835
821
|
return LEVEL_PRIORITY[level] >= LEVEL_PRIORITY[config.logging.level];
|
|
836
822
|
}
|
|
837
|
-
function
|
|
838
|
-
switch (level) {
|
|
839
|
-
case 'warn':
|
|
840
|
-
return 'warning';
|
|
841
|
-
case 'error':
|
|
842
|
-
return 'error';
|
|
823
|
+
function normalizeLogLevel(level) {
|
|
824
|
+
switch (level.toLowerCase()) {
|
|
843
825
|
case 'debug':
|
|
844
826
|
return 'debug';
|
|
845
827
|
case 'info':
|
|
846
|
-
|
|
828
|
+
case 'notice':
|
|
847
829
|
return 'info';
|
|
830
|
+
case 'warning':
|
|
831
|
+
case 'warn':
|
|
832
|
+
return 'warn';
|
|
833
|
+
case 'error':
|
|
834
|
+
case 'critical':
|
|
835
|
+
case 'alert':
|
|
836
|
+
case 'emergency':
|
|
837
|
+
return 'error';
|
|
838
|
+
default:
|
|
839
|
+
return undefined;
|
|
840
|
+
}
|
|
841
|
+
}
|
|
842
|
+
function resolveMcpLogLevel(sessionId) {
|
|
843
|
+
if (sessionId) {
|
|
844
|
+
return sessionMcpLogLevels.get(sessionId) ?? config.logging.level;
|
|
848
845
|
}
|
|
846
|
+
return stdioMcpLogLevel ?? config.logging.level;
|
|
847
|
+
}
|
|
848
|
+
function shouldForwardMcpLog(level, sessionId) {
|
|
849
|
+
return LEVEL_PRIORITY[level] >= LEVEL_PRIORITY[resolveMcpLogLevel(sessionId)];
|
|
850
|
+
}
|
|
851
|
+
function mapToMcpLevel(level) {
|
|
852
|
+
return level === 'warn' ? 'warning' : level;
|
|
849
853
|
}
|
|
850
854
|
function resolveErrorText(err) {
|
|
851
|
-
if (err instanceof Error)
|
|
855
|
+
if (err instanceof Error) {
|
|
856
|
+
if ('errno' in err && typeof err.errno === 'number') {
|
|
857
|
+
try {
|
|
858
|
+
const sysMsg = getSystemErrorMessage(err.errno);
|
|
859
|
+
if (sysMsg)
|
|
860
|
+
return `${err.message} (${sysMsg})`;
|
|
861
|
+
}
|
|
862
|
+
catch {
|
|
863
|
+
// ignore
|
|
864
|
+
}
|
|
865
|
+
}
|
|
852
866
|
return err.message;
|
|
867
|
+
}
|
|
853
868
|
if (typeof err === 'string')
|
|
854
869
|
return err;
|
|
855
870
|
return 'unknown error';
|
|
@@ -870,18 +885,18 @@ function safeWriteStderr(line) {
|
|
|
870
885
|
}
|
|
871
886
|
}
|
|
872
887
|
function writeLog(level, message, meta) {
|
|
873
|
-
if (!shouldLog(level))
|
|
874
|
-
return;
|
|
875
|
-
const line = formatLogEntry(level, message, meta);
|
|
876
|
-
safeWriteStderr(`${stripVTControlCharacters(line)}\n`);
|
|
877
888
|
const sessionId = getSessionId();
|
|
878
|
-
|
|
879
|
-
|
|
880
|
-
|
|
889
|
+
if (shouldLog(level)) {
|
|
890
|
+
const line = formatLogEntry(level, message, meta);
|
|
891
|
+
safeWriteStderr(`${stripVTControlCharacters(line)}\n`);
|
|
892
|
+
}
|
|
893
|
+
const server = sessionId ? sessionServers.get(sessionId) : mcpServer;
|
|
881
894
|
if (!server)
|
|
882
895
|
return;
|
|
883
896
|
if (!server.isConnected())
|
|
884
897
|
return;
|
|
898
|
+
if (!shouldForwardMcpLog(level, sessionId))
|
|
899
|
+
return;
|
|
885
900
|
try {
|
|
886
901
|
server.server
|
|
887
902
|
.sendLoggingMessage({
|
|
@@ -913,30 +928,36 @@ export function logDebug(message, meta) {
|
|
|
913
928
|
export function logWarn(message, meta) {
|
|
914
929
|
writeLog('warn', message, meta);
|
|
915
930
|
}
|
|
931
|
+
function formatErrorMeta(error) {
|
|
932
|
+
const meta = { error: error.message, stack: error.stack };
|
|
933
|
+
if ('errno' in error && typeof error.errno === 'number') {
|
|
934
|
+
try {
|
|
935
|
+
const sysMsg = getSystemErrorMessage(error.errno);
|
|
936
|
+
if (sysMsg)
|
|
937
|
+
meta['sysError'] = sysMsg;
|
|
938
|
+
}
|
|
939
|
+
catch {
|
|
940
|
+
// ignore
|
|
941
|
+
}
|
|
942
|
+
}
|
|
943
|
+
return meta;
|
|
944
|
+
}
|
|
916
945
|
export function logError(message, error) {
|
|
917
|
-
const errorMeta = error instanceof Error
|
|
918
|
-
? { error: error.message, stack: error.stack }
|
|
919
|
-
: (error ?? {});
|
|
946
|
+
const errorMeta = error instanceof Error ? formatErrorMeta(error) : (error ?? {});
|
|
920
947
|
writeLog('error', message, errorMeta);
|
|
921
948
|
}
|
|
922
|
-
export function
|
|
923
|
-
|
|
924
|
-
|
|
925
|
-
|
|
926
|
-
|
|
927
|
-
|
|
928
|
-
|
|
929
|
-
|
|
930
|
-
|
|
931
|
-
|
|
932
|
-
config.logging.level = 'warn';
|
|
933
|
-
}
|
|
934
|
-
else if (normalized === 'error' ||
|
|
935
|
-
normalized === 'critical' ||
|
|
936
|
-
normalized === 'alert' ||
|
|
937
|
-
normalized === 'emergency') {
|
|
938
|
-
config.logging.level = 'error';
|
|
949
|
+
export function getMcpLogLevel(sessionId) {
|
|
950
|
+
return resolveMcpLogLevel(sessionId);
|
|
951
|
+
}
|
|
952
|
+
export function setLogLevel(level, sessionId) {
|
|
953
|
+
const normalized = normalizeLogLevel(level);
|
|
954
|
+
if (!normalized)
|
|
955
|
+
return;
|
|
956
|
+
if (sessionId) {
|
|
957
|
+
sessionMcpLogLevels.set(sessionId, normalized);
|
|
958
|
+
return;
|
|
939
959
|
}
|
|
960
|
+
stdioMcpLogLevel = normalized;
|
|
940
961
|
}
|
|
941
962
|
export function redactUrl(rawUrl) {
|
|
942
963
|
try {
|
|
@@ -976,15 +997,11 @@ function handleSessionCleanupError(error) {
|
|
|
976
997
|
return;
|
|
977
998
|
logWarn('Session cleanup loop failed', { error: getErrorMessage(error) });
|
|
978
999
|
}
|
|
979
|
-
function getRejectedSettledResult(result) {
|
|
980
|
-
return result.status === 'rejected' ? result : undefined;
|
|
981
|
-
}
|
|
982
1000
|
function logRejectedSettledResults(results, message) {
|
|
983
1001
|
for (const result of results) {
|
|
984
|
-
|
|
985
|
-
|
|
986
|
-
|
|
987
|
-
logWarn(message, { error: getErrorMessage(rejected.reason) });
|
|
1002
|
+
if (result.status === 'rejected') {
|
|
1003
|
+
logWarn(message, { error: getErrorMessage(result.reason) });
|
|
1004
|
+
}
|
|
988
1005
|
}
|
|
989
1006
|
}
|
|
990
1007
|
function isSessionExpired(session, now, sessionTtlMs) {
|
|
@@ -992,13 +1009,6 @@ function isSessionExpired(session, now, sessionTtlMs) {
|
|
|
992
1009
|
return false;
|
|
993
1010
|
return now - session.lastSeen > sessionTtlMs;
|
|
994
1011
|
}
|
|
995
|
-
function chunkArray(items, size) {
|
|
996
|
-
const chunks = [];
|
|
997
|
-
for (let index = 0; index < items.length; index += size) {
|
|
998
|
-
chunks.push(items.slice(index, index + size));
|
|
999
|
-
}
|
|
1000
|
-
return chunks;
|
|
1001
|
-
}
|
|
1002
1012
|
class SessionCleanupLoop {
|
|
1003
1013
|
store;
|
|
1004
1014
|
sessionTtlMs;
|
|
@@ -1024,8 +1034,9 @@ class SessionCleanupLoop {
|
|
|
1024
1034
|
}
|
|
1025
1035
|
async handleTick(now, signal) {
|
|
1026
1036
|
const evicted = this.store.evictExpired();
|
|
1027
|
-
for (
|
|
1028
|
-
const
|
|
1037
|
+
for (let i = 0; i < evicted.length; i += SESSION_CLOSE_BATCH_SIZE) {
|
|
1038
|
+
const batch = evicted.slice(i, i + SESSION_CLOSE_BATCH_SIZE);
|
|
1039
|
+
const results = await Promise.allSettled(batch.map((session) => this.closeExpiredSession(session)));
|
|
1029
1040
|
logRejectedSettledResults(results, 'Failed to process expired session cleanup task');
|
|
1030
1041
|
if (signal.aborted)
|
|
1031
1042
|
return;
|
|
@@ -1060,10 +1071,12 @@ class SessionCleanupLoop {
|
|
|
1060
1071
|
session.transport.close(),
|
|
1061
1072
|
session.server.close(),
|
|
1062
1073
|
]);
|
|
1063
|
-
|
|
1064
|
-
|
|
1065
|
-
|
|
1066
|
-
|
|
1074
|
+
if (transportResult.status === 'rejected') {
|
|
1075
|
+
this.logCloseFailure('transport', transportResult.reason);
|
|
1076
|
+
}
|
|
1077
|
+
if (serverResult.status === 'rejected') {
|
|
1078
|
+
this.logCloseFailure('server', serverResult.reason);
|
|
1079
|
+
}
|
|
1067
1080
|
}
|
|
1068
1081
|
logCloseFailure(target, error) {
|
|
1069
1082
|
if (error == null)
|
|
@@ -1,6 +1,5 @@
|
|
|
1
1
|
import { type MarkdownTransformResult } from '../transform/types.js';
|
|
2
2
|
type JsonRecord = Record<string, unknown>;
|
|
3
|
-
export declare function readString(obj: unknown, key: string): string | undefined;
|
|
4
3
|
export declare function readNestedRecord(obj: unknown, keys: readonly string[]): JsonRecord | undefined;
|
|
5
4
|
export declare function withSignal(signal?: AbortSignal): {
|
|
6
5
|
signal: AbortSignal;
|
|
@@ -68,3 +67,4 @@ export declare function performSharedFetch(options: SharedFetchOptions, deps?: S
|
|
|
68
67
|
inlineResult: InlineContentResult;
|
|
69
68
|
}>;
|
|
70
69
|
export {};
|
|
70
|
+
//# sourceMappingURL=fetch-pipeline.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"fetch-pipeline.d.ts","sourceRoot":"","sources":["../../src/lib/fetch-pipeline.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,KAAK,uBAAuB,EAAE,MAAM,uBAAuB,CAAC;AAqBrE,KAAK,UAAU,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;AAY1C,wBAAgB,gBAAgB,CAC9B,GAAG,EAAE,OAAO,EACZ,IAAI,EAAE,SAAS,MAAM,EAAE,GACtB,UAAU,GAAG,SAAS,CAOxB;AAQD,wBAAgB,UAAU,CACxB,MAAM,CAAC,EAAE,WAAW,GACnB;IAAE,MAAM,EAAE,WAAW,CAAA;CAAE,GAAG,MAAM,CAAC,MAAM,EAAE,KAAK,CAAC,CAEjD;AAMD,eAAO,MAAM,iBAAiB,mBAAmB,CAAC;AAClD,MAAM,WAAW,mBAAmB;IAClC,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,WAAW,EAAE,MAAM,CAAC;IACpB,SAAS,CAAC,EAAE,OAAO,CAAC;CACrB;AAuED,wBAAgB,sBAAsB,CACpC,OAAO,EAAE,MAAM,EACf,MAAM,EAAE,MAAM,GACb,MAAM,CAkBR;AAoDD,UAAU,oBAAoB,CAAC,CAAC;IAC9B,GAAG,EAAE,MAAM,CAAC;IACZ,cAAc,EAAE,MAAM,CAAC;IACvB,MAAM,CAAC,EAAE,WAAW,CAAC;IACrB,SAAS,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,MAAM,CAAC;IAC7C,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB,SAAS,EAAE,CACT,KAAK,EAAE;QAAE,MAAM,EAAE,UAAU,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAC;QAAC,SAAS,CAAC,EAAE,OAAO,CAAA;KAAE,EACpE,GAAG,EAAE,MAAM,KACR,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;IACpB,SAAS,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,KAAK,MAAM,CAAC;IAClC,WAAW,CAAC,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK,CAAC,GAAG,SAAS,CAAC;CACjD;AACD,MAAM,WAAW,cAAc,CAAC,CAAC;IAC/B,IAAI,EAAE,CAAC,CAAC;IACR,SAAS,EAAE,OAAO,CAAC;IACnB,GAAG,EAAE,MAAM,CAAC;IACZ,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CAC1B;AAmMD,wBAAsB,oBAAoB,CAAC,CAAC,EAC1C,OAAO,EAAE,oBAAoB,CAAC,CAAC,CAAC,GAC/B,OAAO,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC,CA4D5B;AAMD,MAAM,MAAM,sBAAsB,GAAG,uBAAuB,GAAG;IAC7D,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;CAC1B,CAAC;AAsBF,wBAAgB,yBAAyB,CACvC,MAAM,EAAE,MAAM,GACb,sBAAsB,GAAG,SAAS,CAsBpC;AACD,eAAO,MAAM,iBAAiB,GAC5B,OAAO;IAAE,MAAM,EAAE,UAAU,CAAC;IAAC,QAAQ,EAAE,MAAM,CAAC;IAAC,SAAS,CAAC,EAAE,OAAO,CAAA;CAAE,EACpE,KAAK,MAAM,EACX,SAAS,WAAW,EACpB,mBAAmB,OAAO,KACzB,OAAO,CAAC,sBAAsB,CAUhC,CAAC;AACF,wBAAgB,uBAAuB,CACrC,MAAM,EAAE,sBAAsB,GAC7B,MAAM,CAaR;AAMD,UAAU,kBAAkB;IAC1B,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC;IACrB,QAAQ,CAAC,MAAM,CAAC,EAAE,WAAW,CAAC;IAC9B,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,MAAM,CAAC;IACtD,QAAQ,CAAC,YAAY,CAAC,EAAE,OAAO,CAAC;IAChC,QAAQ,CAAC,cAAc,CAAC,EAAE,MAAM,CAAC;IACjC,QAAQ,CAAC,SAAS,EAAE,CAClB,KAAK,EAAE;QAAE,MAAM,EAAE,UAAU,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAC;QAAC,SAAS,CAAC,EAAE,OAAO,CAAA;KAAE,EACpE,aAAa,EAAE,MAAM,KAClB,sBAAsB,GAAG,OAAO,CAAC,sBAAsB,CAAC,CAAC;IAC9D,QAAQ,CAAC,SAAS,CAAC,EAAE,CAAC,MAAM,EAAE,sBAAsB,KAAK,MAAM,CAAC;IAChE,QAAQ,CAAC,WAAW,CAAC,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK,sBAAsB,GAAG,SAAS,CAAC;CAC/E;AACD,UAAU,eAAe;IACvB,QAAQ,CAAC,oBAAoB,CAAC,EAAE,OAAO,oBAAoB,CAAC;CAC7D;AAeD,wBAAsB,kBAAkB,CACtC,OAAO,EAAE,kBAAkB,EAC3B,IAAI,GAAE,eAAoB,GACzB,OAAO,CAAC;IACT,QAAQ,EAAE,cAAc,CAAC,sBAAsB,CAAC,CAAC;IACjD,YAAY,EAAE,mBAAmB,CAAC;CACnC,CAAC,CAWD"}
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { cachedPayloadSchema } from '../schemas/cache.js';
|
|
2
2
|
import { transformBufferToMarkdown } from '../transform/transform.js';
|
|
3
3
|
import {} from '../transform/types.js';
|
|
4
4
|
import { config, createCacheKey, get, isEnabled, logDebug, logWarn, set, } from './core.js';
|
|
@@ -11,7 +11,7 @@ function readUnknown(obj, key) {
|
|
|
11
11
|
const record = asRecord(obj);
|
|
12
12
|
return record ? record[key] : undefined;
|
|
13
13
|
}
|
|
14
|
-
|
|
14
|
+
function readString(obj, key) {
|
|
15
15
|
const value = readUnknown(obj, key);
|
|
16
16
|
return typeof value === 'string' ? value : undefined;
|
|
17
17
|
}
|
|
@@ -244,6 +244,40 @@ function persistCache(params) {
|
|
|
244
244
|
});
|
|
245
245
|
}
|
|
246
246
|
}
|
|
247
|
+
function buildCacheWriteTargets(params) {
|
|
248
|
+
const { primaryCacheKey, cacheNamespace, cacheVary, requestedUrl, finalUrl } = params;
|
|
249
|
+
const targets = [];
|
|
250
|
+
if (primaryCacheKey) {
|
|
251
|
+
targets.push({
|
|
252
|
+
cacheKey: primaryCacheKey,
|
|
253
|
+
normalizedUrl: finalUrl ?? requestedUrl,
|
|
254
|
+
});
|
|
255
|
+
}
|
|
256
|
+
if (!finalUrl || finalUrl === requestedUrl) {
|
|
257
|
+
return targets;
|
|
258
|
+
}
|
|
259
|
+
const finalCacheKey = createCacheKey(cacheNamespace, finalUrl, cacheVary);
|
|
260
|
+
if (!finalCacheKey || finalCacheKey === primaryCacheKey) {
|
|
261
|
+
return targets;
|
|
262
|
+
}
|
|
263
|
+
targets.push({
|
|
264
|
+
cacheKey: finalCacheKey,
|
|
265
|
+
normalizedUrl: finalUrl,
|
|
266
|
+
});
|
|
267
|
+
return targets;
|
|
268
|
+
}
|
|
269
|
+
function persistCacheTargets(params) {
|
|
270
|
+
const { targets, data, serialize, cacheNamespace } = params;
|
|
271
|
+
for (const target of targets) {
|
|
272
|
+
persistCache({
|
|
273
|
+
cacheKey: target.cacheKey,
|
|
274
|
+
data,
|
|
275
|
+
serialize,
|
|
276
|
+
normalizedUrl: target.normalizedUrl,
|
|
277
|
+
cacheNamespace,
|
|
278
|
+
});
|
|
279
|
+
}
|
|
280
|
+
}
|
|
247
281
|
export async function executeFetchPipeline(options) {
|
|
248
282
|
const resolvedUrl = resolveNormalizedUrl(options.url);
|
|
249
283
|
logRawUrlTransformation(resolvedUrl);
|
|
@@ -265,25 +299,18 @@ export async function executeFetchPipeline(options) {
|
|
|
265
299
|
const transformUrl = resolvedFinalUrl;
|
|
266
300
|
const data = await options.transform({ buffer, encoding, ...(truncated ? { truncated: true } : {}) }, transformUrl);
|
|
267
301
|
if (isEnabled()) {
|
|
268
|
-
|
|
269
|
-
|
|
302
|
+
persistCacheTargets({
|
|
303
|
+
targets: buildCacheWriteTargets({
|
|
304
|
+
primaryCacheKey: cacheKey,
|
|
305
|
+
cacheNamespace: options.cacheNamespace,
|
|
306
|
+
cacheVary: options.cacheVary,
|
|
307
|
+
requestedUrl: resolvedUrl.normalizedUrl,
|
|
308
|
+
finalUrl,
|
|
309
|
+
}),
|
|
270
310
|
data,
|
|
271
311
|
serialize: options.serialize,
|
|
272
|
-
normalizedUrl: resolvedFinalUrl,
|
|
273
312
|
cacheNamespace: options.cacheNamespace,
|
|
274
313
|
});
|
|
275
|
-
if (finalUrl && finalUrl !== resolvedUrl.normalizedUrl) {
|
|
276
|
-
const finalCacheKey = createCacheKey(options.cacheNamespace, finalUrl, options.cacheVary);
|
|
277
|
-
if (finalCacheKey && finalCacheKey !== cacheKey) {
|
|
278
|
-
persistCache({
|
|
279
|
-
cacheKey: finalCacheKey,
|
|
280
|
-
data,
|
|
281
|
-
serialize: options.serialize,
|
|
282
|
-
normalizedUrl: finalUrl,
|
|
283
|
-
cacheNamespace: options.cacheNamespace,
|
|
284
|
-
});
|
|
285
|
-
}
|
|
286
|
-
}
|
|
287
314
|
}
|
|
288
315
|
return {
|
|
289
316
|
data,
|
|
@@ -301,29 +328,9 @@ function normalizeExtractedMetadata(metadata) {
|
|
|
301
328
|
const normalized = Object.fromEntries(Object.entries(metadata).filter(([, v]) => Boolean(v)));
|
|
302
329
|
return Object.keys(normalized).length > 0 ? normalized : undefined;
|
|
303
330
|
}
|
|
304
|
-
const cachedMarkdownSchema = z
|
|
305
|
-
.object({
|
|
306
|
-
markdown: z.string().optional(),
|
|
307
|
-
content: z.string().optional(),
|
|
308
|
-
title: z.string().optional(),
|
|
309
|
-
metadata: z
|
|
310
|
-
.strictObject({
|
|
311
|
-
title: z.string().optional(),
|
|
312
|
-
description: z.string().optional(),
|
|
313
|
-
author: z.string().optional(),
|
|
314
|
-
image: z.string().optional(),
|
|
315
|
-
favicon: z.string().optional(),
|
|
316
|
-
publishedAt: z.string().optional(),
|
|
317
|
-
modifiedAt: z.string().optional(),
|
|
318
|
-
})
|
|
319
|
-
.optional(),
|
|
320
|
-
truncated: z.boolean().optional(),
|
|
321
|
-
})
|
|
322
|
-
.catchall(z.unknown())
|
|
323
|
-
.refine((value) => typeof value.markdown === 'string' || typeof value.content === 'string', { message: 'Missing markdown/content' });
|
|
324
331
|
export function parseCachedMarkdownResult(cached) {
|
|
325
332
|
const parsed = safeJsonParse(cached);
|
|
326
|
-
const result =
|
|
333
|
+
const result = cachedPayloadSchema.safeParse(parsed);
|
|
327
334
|
if (!result.success)
|
|
328
335
|
return undefined;
|
|
329
336
|
const markdown = result.data.markdown ?? result.data.content;
|
|
@@ -357,16 +364,16 @@ export function serializeMarkdownResult(result) {
|
|
|
357
364
|
const persistedMarkdown = result.truncated
|
|
358
365
|
? appendTruncationMarker(result.markdown, TRUNCATION_MARKER)
|
|
359
366
|
: result.markdown;
|
|
360
|
-
|
|
367
|
+
const payload = {
|
|
361
368
|
markdown: persistedMarkdown,
|
|
362
369
|
title: result.title,
|
|
363
370
|
metadata: result.metadata,
|
|
364
371
|
truncated: result.truncated,
|
|
365
|
-
}
|
|
372
|
+
};
|
|
373
|
+
return JSON.stringify(payload);
|
|
366
374
|
}
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
const pipelineOptions = {
|
|
375
|
+
function buildSharedFetchPipelineOptions(options) {
|
|
376
|
+
return {
|
|
370
377
|
url: options.url,
|
|
371
378
|
cacheNamespace: 'markdown',
|
|
372
379
|
...withSignal(options.signal),
|
|
@@ -376,7 +383,10 @@ export async function performSharedFetch(options, deps = {}) {
|
|
|
376
383
|
...(options.serialize ? { serialize: options.serialize } : {}),
|
|
377
384
|
...(options.deserialize ? { deserialize: options.deserialize } : {}),
|
|
378
385
|
};
|
|
379
|
-
|
|
386
|
+
}
|
|
387
|
+
export async function performSharedFetch(options, deps = {}) {
|
|
388
|
+
const executePipeline = deps.executeFetchPipeline ?? executeFetchPipeline;
|
|
389
|
+
const pipeline = await executePipeline(buildSharedFetchPipelineOptions(options));
|
|
380
390
|
const inlineResult = applyInlineContentLimit(pipeline.data.content, options.maxInlineChars);
|
|
381
391
|
return { pipeline, inlineResult };
|
|
382
392
|
}
|
package/dist/lib/http.d.ts
CHANGED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"http.d.ts","sourceRoot":"","sources":["../../src/lib/http.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,KAAK,cAAc,EAAE,MAAM,WAAW,CAAC;AAWhD,OAAO,EAAE,KAAK,EAAmB,MAAM,QAAQ,CAAC;AAehD,OAAO,EAOL,KAAK,eAAe,EAGrB,MAAM,UAAU,CAAC;AAgElB,wBAAgB,oBAAoB,CAClC,GAAG,EAAE,MAAM,EACX,KAAK,CAAC,EAAE,MAAM,EACd,YAAY,CAAC,EAAE,MAAM,EACrB,SAAS,SAAQ,GAChB,MAAM,CAIR;AAkBD,wBAAgB,cAAc,CAC5B,GAAG,EAAE,cAAc,EACnB,SAAS,EAAE,MAAM,EACjB,IAAI,EAAE,MAAM,GACX,IAAI,CAwCN;AAmoCD,UAAU,qBAAqB;IAC7B,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,GAAG,EAAE,MAAM,CAAC;IACZ,MAAM,EAAE,MAAM,CAAC;IACf,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AAuJD,UAAU,YAAY;IACpB,MAAM,CAAC,EAAE,WAAW,CAAC;CACtB;AAsMD,wBAAgB,WAAW,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAE/C;AACD,wBAAgB,YAAY,CAAC,SAAS,EAAE,MAAM,GAAG;IAC/C,aAAa,EAAE,MAAM,CAAC;IACtB,QAAQ,EAAE,MAAM,CAAC;CAClB,CAEA;AACD,wBAAgB,uBAAuB,CAAC,SAAS,EAAE,MAAM,GAAG,MAAM,CAEjE;AACD,wBAAgB,iBAAiB,CAAC,GAAG,EAAE,MAAM,GAAG,eAAe,CAE9D;AACD,wBAAgB,mBAAmB,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAExD;AACD,wBAAgB,mBAAmB,CACjC,GAAG,EAAE,MAAM,EACX,MAAM,EAAE,MAAM,GACb,qBAAqB,CAEvB;AACD,wBAAgB,mBAAmB,CACjC,OAAO,EAAE,qBAAqB,EAC9B,QAAQ,EAAE,QAAQ,EAClB,WAAW,CAAC,EAAE,MAAM,GACnB,IAAI,CAEN;AACD,wBAAgB,gBAAgB,CAC9B,OAAO,EAAE,qBAAqB,EAC9B,KAAK,EAAE,OAAO,EACd,MAAM,CAAC,EAAE,MAAM,GACd,IAAI,CAEN;AACD,wBAAsB,kBAAkB,CACtC,GAAG,EAAE,MAAM,EACX,IAAI,EAAE,WAAW,EACjB,YAAY,EAAE,MAAM,GACnB,OAAO,CAAC;IAAE,QAAQ,EAAE,QAAQ,CAAC;IAAC,GAAG,EAAE,MAAM,CAAC;IAAC,KAAK,CAAC,EAAE,KAAK,CAAA;CAAE,CAAC,CAE7D;AACD,wBAAsB,gBAAgB,CACpC,QAAQ,EAAE,QAAQ,EAClB,GAAG,EAAE,MAAM,EACX,QAAQ,EAAE,MAAM,EAChB,MAAM,CAAC,EAAE,WAAW,EACpB,QAAQ,CAAC,EAAE,MAAM,GAChB,OAAO,CAAC;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,IAAI,EAAE,MAAM,CAAA;CAAE,CAAC,CAUzC;AACD,wBAAsB,kBAAkB,CACtC,aAAa,EAAE,MAAM,EACrB,OAAO,CAAC,EAAE,YAAY,GACrB,OAAO,CAAC,MAAM,CAAC,CAEjB;AACD,wBAAsB,wBAAwB,CAC5C,aAAa,EAAE,MAAM,EACrB,OAAO,CAAC,EAAE,YAAY,GACrB,OAAO,CAAC;IACT,MAAM,EAAE,UAAU,CAAC;IACnB,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,OAAO,CAAC;IACnB,QAAQ,EAAE,MAAM,CAAC;CAClB,CAAC,CAED"}
|
package/dist/lib/mcp-tools.d.ts
CHANGED
|
@@ -1,17 +1,54 @@
|
|
|
1
1
|
import { type CallToolResult } from '@modelcontextprotocol/sdk/types.js';
|
|
2
|
+
import { z } from 'zod';
|
|
2
3
|
export type JsonRpcId = string | number | null;
|
|
3
4
|
interface McpRequestParams {
|
|
4
5
|
_meta?: Record<string, unknown>;
|
|
5
6
|
[key: string]: unknown;
|
|
6
7
|
}
|
|
7
|
-
|
|
8
|
-
jsonrpc:
|
|
9
|
-
method:
|
|
10
|
-
id
|
|
11
|
-
params
|
|
12
|
-
}
|
|
8
|
+
declare const jsonRpcRequestSchema: z.ZodObject<{
|
|
9
|
+
jsonrpc: z.ZodLiteral<"2.0">;
|
|
10
|
+
method: z.ZodString;
|
|
11
|
+
id: z.ZodOptional<z.ZodUnion<readonly [z.ZodString, z.ZodNumber]>>;
|
|
12
|
+
params: z.ZodOptional<z.ZodType<McpRequestParams, unknown, z.core.$ZodTypeInternals<McpRequestParams, unknown>>>;
|
|
13
|
+
}, z.core.$strict>;
|
|
14
|
+
declare const jsonRpcResponseSchema: z.ZodUnion<readonly [z.ZodObject<{
|
|
15
|
+
jsonrpc: z.ZodLiteral<"2.0">;
|
|
16
|
+
id: z.ZodUnion<readonly [z.ZodString, z.ZodNumber]>;
|
|
17
|
+
result: z.ZodRecord<z.ZodString, z.ZodUnknown>;
|
|
18
|
+
}, z.core.$strict>, z.ZodObject<{
|
|
19
|
+
jsonrpc: z.ZodLiteral<"2.0">;
|
|
20
|
+
id: z.ZodOptional<z.ZodUnion<readonly [z.ZodString, z.ZodNumber, z.ZodNull]>>;
|
|
21
|
+
error: z.ZodObject<{
|
|
22
|
+
code: z.ZodNumber;
|
|
23
|
+
message: z.ZodString;
|
|
24
|
+
data: z.ZodOptional<z.ZodUnknown>;
|
|
25
|
+
}, z.core.$strict>;
|
|
26
|
+
}, z.core.$strict>]>;
|
|
27
|
+
declare const jsonRpcMessageSchema: z.ZodUnion<readonly [z.ZodObject<{
|
|
28
|
+
jsonrpc: z.ZodLiteral<"2.0">;
|
|
29
|
+
method: z.ZodString;
|
|
30
|
+
id: z.ZodOptional<z.ZodUnion<readonly [z.ZodString, z.ZodNumber]>>;
|
|
31
|
+
params: z.ZodOptional<z.ZodType<McpRequestParams, unknown, z.core.$ZodTypeInternals<McpRequestParams, unknown>>>;
|
|
32
|
+
}, z.core.$strict>, z.ZodUnion<readonly [z.ZodObject<{
|
|
33
|
+
jsonrpc: z.ZodLiteral<"2.0">;
|
|
34
|
+
id: z.ZodUnion<readonly [z.ZodString, z.ZodNumber]>;
|
|
35
|
+
result: z.ZodRecord<z.ZodString, z.ZodUnknown>;
|
|
36
|
+
}, z.core.$strict>, z.ZodObject<{
|
|
37
|
+
jsonrpc: z.ZodLiteral<"2.0">;
|
|
38
|
+
id: z.ZodOptional<z.ZodUnion<readonly [z.ZodString, z.ZodNumber, z.ZodNull]>>;
|
|
39
|
+
error: z.ZodObject<{
|
|
40
|
+
code: z.ZodNumber;
|
|
41
|
+
message: z.ZodString;
|
|
42
|
+
data: z.ZodOptional<z.ZodUnknown>;
|
|
43
|
+
}, z.core.$strict>;
|
|
44
|
+
}, z.core.$strict>]>]>;
|
|
45
|
+
type McpRequestBody = z.infer<typeof jsonRpcRequestSchema>;
|
|
46
|
+
type JsonRpcResponseBody = z.infer<typeof jsonRpcResponseSchema>;
|
|
47
|
+
type JsonRpcMessageBody = z.infer<typeof jsonRpcMessageSchema>;
|
|
13
48
|
export declare function isJsonRpcBatchRequest(body: unknown): boolean;
|
|
14
49
|
export declare function isMcpRequestBody(body: unknown): body is McpRequestBody;
|
|
50
|
+
export declare function isJsonRpcResponseBody(body: unknown): body is JsonRpcResponseBody;
|
|
51
|
+
export declare function isMcpMessageBody(body: unknown): body is JsonRpcMessageBody;
|
|
15
52
|
export declare function acceptsEventStream(header: string | null | undefined): boolean;
|
|
16
53
|
export declare function acceptsJsonAndEventStream(header: string | null | undefined): boolean;
|
|
17
54
|
type ToolErrorResponse = CallToolResult & {
|
|
@@ -24,5 +61,6 @@ export declare function createToolErrorResponse(message: string, url: string, ex
|
|
|
24
61
|
}): ToolErrorResponse;
|
|
25
62
|
export declare function handleToolError(error: unknown, url: string, fallbackMessage?: string): ToolErrorResponse;
|
|
26
63
|
export { registerServerLifecycleCleanup, registerTaskHandlers, cancelTasksForOwner, abortAllTaskExecutions, } from './task-handlers.js';
|
|
27
|
-
export {
|
|
64
|
+
export { readNestedRecord, withSignal, TRUNCATION_MARKER, type InlineContentResult, appendTruncationMarker, type PipelineResult, executeFetchPipeline, type MarkdownPipelineResult, parseCachedMarkdownResult, markdownTransform, serializeMarkdownResult, performSharedFetch, } from './fetch-pipeline.js';
|
|
28
65
|
export { type ProgressNotificationParams, type ProgressNotification, type ToolHandlerExtra, type ProgressReporter, createProgressReporter, } from './progress.js';
|
|
66
|
+
//# sourceMappingURL=mcp-tools.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"mcp-tools.d.ts","sourceRoot":"","sources":["../../src/lib/mcp-tools.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,cAAc,EAAE,MAAM,oCAAoC,CAAC;AACzE,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAQxB,MAAM,MAAM,SAAS,GAAG,MAAM,GAAG,MAAM,GAAG,IAAI,CAAC;AAC/C,UAAU,gBAAgB;IACxB,KAAK,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAChC,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC;CACxB;AAGD,QAAA,MAAM,oBAAoB;;;;;kBAKxB,CAAC;AAeH,QAAA,MAAM,qBAAqB;;;;;;;;;;;;oBAGzB,CAAC;AACH,QAAA,MAAM,oBAAoB;;;;;;;;;;;;;;;;;sBAGxB,CAAC;AACH,KAAK,cAAc,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,oBAAoB,CAAC,CAAC;AAC3D,KAAK,mBAAmB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,qBAAqB,CAAC,CAAC;AACjE,KAAK,kBAAkB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,oBAAoB,CAAC,CAAC;AAC/D,wBAAgB,qBAAqB,CAAC,IAAI,EAAE,OAAO,GAAG,OAAO,CAE5D;AACD,wBAAgB,gBAAgB,CAAC,IAAI,EAAE,OAAO,GAAG,IAAI,IAAI,cAAc,CAEtE;AACD,wBAAgB,qBAAqB,CACnC,IAAI,EAAE,OAAO,GACZ,IAAI,IAAI,mBAAmB,CAE7B;AACD,wBAAgB,gBAAgB,CAAC,IAAI,EAAE,OAAO,GAAG,IAAI,IAAI,kBAAkB,CAE1E;AAaD,wBAAgB,kBAAkB,CAAC,MAAM,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,GAAG,OAAO,CAG7E;AAeD,wBAAgB,yBAAyB,CACvC,MAAM,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,GAChC,OAAO,CAUT;AAMD,KAAK,iBAAiB,GAAG,cAAc,GAAG;IACxC,OAAO,EAAE,IAAI,CAAC;CACf,CAAC;AACF,wBAAgB,uBAAuB,CACrC,OAAO,EAAE,MAAM,EACf,GAAG,EAAE,MAAM,EACX,KAAK,CAAC,EAAE;IACN,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACnC,GACA,iBAAiB,CAenB;AAkCD,wBAAgB,eAAe,CAC7B,KAAK,EAAE,OAAO,EACd,GAAG,EAAE,MAAM,EACX,eAAe,SAAqB,GACnC,iBAAiB,CAWnB;AASD,OAAO,EACL,8BAA8B,EAC9B,oBAAoB,EACpB,mBAAmB,EACnB,sBAAsB,GACvB,MAAM,oBAAoB,CAAC;AAE5B,OAAO,EACL,gBAAgB,EAChB,UAAU,EACV,iBAAiB,EACjB,KAAK,mBAAmB,EACxB,sBAAsB,EACtB,KAAK,cAAc,EACnB,oBAAoB,EACpB,KAAK,sBAAsB,EAC3B,yBAAyB,EACzB,iBAAiB,EACjB,uBAAuB,EACvB,kBAAkB,GACnB,MAAM,qBAAqB,CAAC;AAE7B,OAAO,EACL,KAAK,0BAA0B,EAC/B,KAAK,oBAAoB,EACzB,KAAK,gBAAgB,EACrB,KAAK,gBAAgB,EACrB,sBAAsB,GACvB,MAAM,eAAe,CAAC"}
|