@j0hanz/fetch-url-mcp 1.12.0 → 1.12.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +34 -17
- package/dist/http/auth.d.ts.map +1 -1
- package/dist/http/auth.js +61 -20
- package/dist/http/helpers.d.ts +1 -1
- package/dist/http/helpers.d.ts.map +1 -1
- package/dist/http/helpers.js +7 -9
- package/dist/http/native.d.ts.map +1 -1
- package/dist/http/native.js +271 -54
- package/dist/http/rate-limit.d.ts.map +1 -1
- package/dist/http/rate-limit.js +2 -1
- package/dist/index.js +5 -4
- package/dist/lib/config.d.ts +1 -1
- package/dist/lib/config.d.ts.map +1 -1
- package/dist/lib/config.js +8 -1
- package/dist/lib/core.d.ts +8 -4
- package/dist/lib/core.d.ts.map +1 -1
- package/dist/lib/core.js +240 -73
- package/dist/lib/fetch-pipeline.d.ts.map +1 -1
- package/dist/lib/fetch-pipeline.js +15 -2
- package/dist/lib/http.d.ts.map +1 -1
- package/dist/lib/http.js +1 -1
- package/dist/lib/mcp-interop.d.ts +15 -3
- package/dist/lib/mcp-interop.d.ts.map +1 -1
- package/dist/lib/mcp-interop.js +92 -23
- package/dist/lib/url.d.ts.map +1 -1
- package/dist/lib/url.js +1 -1
- package/dist/lib/utils.d.ts.map +1 -1
- package/dist/lib/utils.js +2 -2
- package/dist/resources/index.d.ts +4 -0
- package/dist/resources/index.d.ts.map +1 -1
- package/dist/resources/index.js +39 -4
- package/dist/schemas.d.ts +5 -5
- package/dist/schemas.d.ts.map +1 -1
- package/dist/schemas.js +7 -9
- package/dist/server.d.ts +3 -1
- package/dist/server.d.ts.map +1 -1
- package/dist/server.js +20 -11
- package/dist/tasks/execution.d.ts +1 -1
- package/dist/tasks/execution.d.ts.map +1 -1
- package/dist/tasks/execution.js +72 -25
- package/dist/tasks/handlers.d.ts.map +1 -1
- package/dist/tasks/handlers.js +31 -24
- package/dist/tasks/manager.d.ts +5 -2
- package/dist/tasks/manager.d.ts.map +1 -1
- package/dist/tasks/manager.js +58 -19
- package/dist/tasks/owner.d.ts +5 -0
- package/dist/tasks/owner.d.ts.map +1 -1
- package/dist/tasks/owner.js +15 -7
- package/dist/tasks/registry.d.ts +10 -8
- package/dist/tasks/registry.d.ts.map +1 -1
- package/dist/tasks/registry.js +27 -15
- package/dist/tools/fetch-url.d.ts +2 -0
- package/dist/tools/fetch-url.d.ts.map +1 -1
- package/dist/tools/fetch-url.js +76 -21
- package/dist/transform/dom-prep.d.ts.map +1 -1
- package/dist/transform/dom-prep.js +6 -6
- package/dist/transform/transform.d.ts.map +1 -1
- package/dist/transform/transform.js +17 -14
- package/dist/transform/worker-pool.d.ts.map +1 -1
- package/dist/transform/worker-pool.js +43 -3
- package/package.json +2 -2
package/dist/index.js
CHANGED
|
@@ -36,7 +36,7 @@ const isStdioMode = !values.http;
|
|
|
36
36
|
let isShuttingDown = false;
|
|
37
37
|
const shutdownHandlerRef = {};
|
|
38
38
|
function shouldAttemptShutdown() {
|
|
39
|
-
return !isShuttingDown &&
|
|
39
|
+
return !isShuttingDown && Boolean(shutdownHandlerRef.current);
|
|
40
40
|
}
|
|
41
41
|
function attemptShutdown(signal) {
|
|
42
42
|
if (!shutdownHandlerRef.current)
|
|
@@ -60,13 +60,13 @@ function registerHttpSignalHandlers() {
|
|
|
60
60
|
registerSignalHandlers(['SIGINT', 'SIGTERM'], tryShutdown);
|
|
61
61
|
}
|
|
62
62
|
function writeStartupError(error) {
|
|
63
|
-
logError('Failed to start server', error);
|
|
63
|
+
logError('Failed to start server', error, 'server');
|
|
64
64
|
process.stderr.write(`Failed to start server: ${error.message}\n`);
|
|
65
65
|
process.exitCode = 1;
|
|
66
66
|
scheduleForcedExit('Startup failure');
|
|
67
67
|
}
|
|
68
68
|
function handleFatalError(label, error, signal) {
|
|
69
|
-
logError(label, error);
|
|
69
|
+
logError(label, error, 'server');
|
|
70
70
|
process.stderr.write(`${label}: ${error.message}\n`);
|
|
71
71
|
process.exitCode = 1;
|
|
72
72
|
if (shouldAttemptShutdown()) {
|
|
@@ -84,7 +84,8 @@ process.on('unhandledRejection', (reason) => {
|
|
|
84
84
|
});
|
|
85
85
|
try {
|
|
86
86
|
if (isStdioMode) {
|
|
87
|
-
await startStdioServer();
|
|
87
|
+
const { shutdown } = await startStdioServer();
|
|
88
|
+
shutdownHandlerRef.current = shutdown;
|
|
88
89
|
}
|
|
89
90
|
else {
|
|
90
91
|
const { shutdown } = await startHttpServer();
|
package/dist/lib/config.d.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
export declare const serverVersion: string;
|
|
2
|
-
export type LogLevel = 'debug' | 'info' | 'warn' | 'error';
|
|
2
|
+
export type LogLevel = 'debug' | 'info' | 'notice' | 'warn' | 'error' | 'critical';
|
|
3
3
|
type TransformWorkerMode = 'threads' | 'process';
|
|
4
4
|
type AuthMode = 'oauth' | 'static';
|
|
5
5
|
interface WorkerResourceLimits {
|
package/dist/lib/config.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"config.d.ts","sourceRoot":"","sources":["../../src/lib/config.ts"],"names":[],"mappings":"AAgDA,eAAO,MAAM,aAAa,EAAE,MAA2C,CAAC;AAIxE,MAAM,MAAM,QAAQ,
|
|
1
|
+
{"version":3,"file":"config.d.ts","sourceRoot":"","sources":["../../src/lib/config.ts"],"names":[],"mappings":"AAgDA,eAAO,MAAM,aAAa,EAAE,MAA2C,CAAC;AAIxE,MAAM,MAAM,QAAQ,GAChB,OAAO,GACP,MAAM,GACN,QAAQ,GACR,MAAM,GACN,OAAO,GACP,UAAU,CAAC;AAkCf,KAAK,mBAAmB,GAAG,SAAS,GAAG,SAAS,CAAC;AACjD,KAAK,QAAQ,GAAG,OAAO,GAAG,QAAQ,CAAC;AA4OnC,UAAU,oBAAoB;IAC5B,sBAAsB,CAAC,EAAE,MAAM,CAAC;IAChC,wBAAwB,CAAC,EAAE,MAAM,CAAC;IAClC,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AAyCD,UAAU,UAAU;IAClB,IAAI,EAAE,QAAQ,CAAC;IACf,SAAS,EAAE,GAAG,GAAG,SAAS,CAAC;IAC3B,gBAAgB,EAAE,GAAG,GAAG,SAAS,CAAC;IAClC,QAAQ,EAAE,GAAG,GAAG,SAAS,CAAC;IAC1B,aAAa,EAAE,GAAG,GAAG,SAAS,CAAC;IAC/B,eAAe,EAAE,GAAG,GAAG,SAAS,CAAC;IACjC,gBAAgB,EAAE,GAAG,GAAG,SAAS,CAAC;IAClC,WAAW,EAAE,GAAG,CAAC;IACjB,cAAc,EAAE,MAAM,EAAE,CAAC;IACzB,QAAQ,EAAE,MAAM,GAAG,SAAS,CAAC;IAC7B,YAAY,EAAE,MAAM,GAAG,SAAS,CAAC;IACjC,sBAAsB,EAAE,MAAM,CAAC;IAC/B,YAAY,EAAE,MAAM,EAAE,CAAC;CACxB;AAED,UAAU,WAAW;IACnB,OAAO,EAAE,OAAO,CAAC;IACjB,OAAO,EAAE,MAAM,GAAG,SAAS,CAAC;IAC5B,QAAQ,EAAE,MAAM,GAAG,SAAS,CAAC;IAC7B,MAAM,EAAE,MAAM,GAAG,SAAS,CAAC;CAC5B;AAoGD,UAAU,YAAY;IACpB,QAAQ,EAAE,OAAO,CAAC;CACnB;AAQD,UAAU,mBAAmB;IAC3B,gBAAgB,EAAE,MAAM,GAAG,SAAS,CAAC;IACrC,gBAAgB,EAAE,MAAM,GAAG,SAAS,CAAC;IACrC,kBAAkB,EAAE,MAAM,GAAG,SAAS,CAAC;IACvC,wBAAwB,EAAE,MAAM,GAAG,SAAS,CAAC;IAC7C,eAAe,EAAE,MAAM,GAAG,SAAS,CAAC;IACpC,cAAc,EAAE,MAAM,CAAC;IACvB,uBAAuB,EAAE,OAAO,CAAC;IACjC,4BAA4B,EAAE,OAAO,CAAC;IACtC,2BAA2B,EAAE,OAAO,CAAC;CACtC;AAED,UAAU,eAAe;IACvB,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,MAAM,CAAC;IAChB,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,WAAW,CAAC;IACnB,YAAY,EAAE,MAAM,CAAC;IACrB,oBAAoB,EAAE,MAAM,CAAC;IAC7B,WAAW,EAAE,MAAM,CAAC;IACpB,IAAI,EAAE,mBAAmB,CAAC;CAC3B;AAuCD,UAAU,gBAAgB;IACxB,OAAO,EAAE,MAAM,CAAC;IAChB,YAAY,EAAE,MAAM,CAAC;IACrB,SAAS,EAAE,MAAM,CAAC;IAClB,gBAAgB,EAAE,MAAM,CAAC;CAC1B;AAWD,UAAU,kBAAkB;IAC1B,SAAS,EAAE,MAAM,CAAC;IAClB,cAAc,EAAE,MAAM,CAAC;IACvB,cAAc,EAAE,MAAM,CAAC;IACvB,cAAc,EAAE,MAAM,CAAC;IACvB,kBAAkB,EAAE,MAAM,CAAC;IAC3B,UAAU,EAAE,mBAAmB,CAAC;IAChC,oBAAoB,EAAE,oBAAoB,GAAG,SAAS,CAAC;CACxD;AAkBD,UAAU,cAAc;IACtB,QAAQ,EAAE,MAAM,CAAC;IACjB,WAAW,EAAE,MAAM,CAAC;IACpB,uBAAuB,EAAE,OAAO,CAAC;IACjC,mBAAmB,EAAE,OAAO,CAAC;CAC9B;AAmBD,UAAU,qBAAqB;IAC7B,WAAW,EAAE,MAAM,EAAE,CAAC;IACtB,cAAc,EAAE,MAAM,EAAE,CAAC;IACzB,iBAAiB,EAAE,MAAM,EAAE,CAAC;IAC5B,KAAK,EAAE,OAAO,CAAC;IACf,cAAc,EAAE,OAAO,CAAC;IACxB,iBAAiB,EAAE,OAAO,CAAC;IAC3B,OAAO,EAAE;QACP,MAAM,EAAE,MAAM,CAAC;QACf,UAAU,EAAE,MAAM,CAAC;QACnB,KAAK,EAAE,MAAM,CAAC;QACd,WAAW,EAAE,MAAM,CAAC;QACpB,SAAS,EAAE,MAAM,CAAC;KACnB,CAAC;CACH;AA2BD,UAAU,wBAAwB;IAChC,qBAAqB,EAAE,OAAO,CAAC;IAC/B,eAAe,EAAE,OAAO,CAAC;IACzB,eAAe,EAAE,OAAO,CAAC;IACzB,qBAAqB,EAAE,OAAO,CAAC;IAC/B,eAAe,EAAE,MAAM,EAAE,CAAC;CAC3B;AAiBD,eAAO,MAAM,MAAM;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CA8ClB,CAAC;AAEF,wBAAgB,cAAc,IAAI,IAAI,CAErC"}
|
package/dist/lib/config.js
CHANGED
|
@@ -30,7 +30,14 @@ function readServerVersion(moduleUrl) {
|
|
|
30
30
|
return packageJson.version;
|
|
31
31
|
}
|
|
32
32
|
export const serverVersion = readServerVersion(import.meta.url);
|
|
33
|
-
const LOG_LEVELS = [
|
|
33
|
+
const LOG_LEVELS = [
|
|
34
|
+
'debug',
|
|
35
|
+
'info',
|
|
36
|
+
'notice',
|
|
37
|
+
'warn',
|
|
38
|
+
'error',
|
|
39
|
+
'critical',
|
|
40
|
+
];
|
|
34
41
|
const ALLOWED_LOG_LEVELS = new Set(LOG_LEVELS);
|
|
35
42
|
const DEFAULT_HEADING_KEYWORDS = [
|
|
36
43
|
'overview',
|
package/dist/lib/core.d.ts
CHANGED
|
@@ -10,17 +10,21 @@ interface RequestContext {
|
|
|
10
10
|
}
|
|
11
11
|
export declare function setMcpServer(server: McpServer): void;
|
|
12
12
|
export declare function registerMcpSessionServer(sessionId: string, server: McpServer): void;
|
|
13
|
+
export declare function registerMcpSessionOwnerKey(sessionId: string, ownerKey: string): void;
|
|
13
14
|
export declare function unregisterMcpSessionServer(sessionId: string): void;
|
|
14
15
|
export declare function unregisterMcpSessionServerByServer(server: McpServer): void;
|
|
16
|
+
export declare function resolveMcpSessionOwnerKey(sessionId: string): string | undefined;
|
|
15
17
|
export declare function resolveMcpSessionIdByServer(server: McpServer): string | undefined;
|
|
16
18
|
export declare function runWithRequestContext<T>(context: RequestContext, fn: () => T): T;
|
|
17
19
|
export declare function getRequestId(): string | undefined;
|
|
18
20
|
export declare function getSessionId(): string | undefined;
|
|
19
21
|
export declare function getOperationId(): string | undefined;
|
|
20
|
-
export declare function logInfo(message: string, meta?: LogMetadata): void;
|
|
21
|
-
export declare function logDebug(message: string, meta?: LogMetadata): void;
|
|
22
|
-
export declare function
|
|
23
|
-
export declare function
|
|
22
|
+
export declare function logInfo(message: string, meta?: LogMetadata, logger?: string): void;
|
|
23
|
+
export declare function logDebug(message: string, meta?: LogMetadata, logger?: string): void;
|
|
24
|
+
export declare function logNotice(message: string, meta?: LogMetadata, logger?: string): void;
|
|
25
|
+
export declare function logWarn(message: string, meta?: LogMetadata, logger?: string): void;
|
|
26
|
+
export declare function logError(message: string, error?: Error | LogMetadata, logger?: string): void;
|
|
27
|
+
export declare function logCritical(message: string, error?: Error | LogMetadata, logger?: string): void;
|
|
24
28
|
export declare function setLogLevel(level: string, sessionId?: string): void;
|
|
25
29
|
export declare function redactUrl(rawUrl: string): string;
|
|
26
30
|
export type { SessionEntry, SessionStore } from './session.js';
|
package/dist/lib/core.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"core.d.ts","sourceRoot":"","sources":["../../src/lib/core.ts"],"names":[],"mappings":"AAQA,OAAO,EAAE,KAAK,SAAS,EAAE,MAAM,yCAAyC,CAAC;AAGzE,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AACjD,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAOjD,OAAO,EAAE,MAAM,EAAE,cAAc,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAYpE,KAAK,WAAW,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;AAC3C,UAAU,cAAc;IACtB,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,CAAC;CAC/B;
|
|
1
|
+
{"version":3,"file":"core.d.ts","sourceRoot":"","sources":["../../src/lib/core.ts"],"names":[],"mappings":"AAQA,OAAO,EAAE,KAAK,SAAS,EAAE,MAAM,yCAAyC,CAAC;AAGzE,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AACjD,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAOjD,OAAO,EAAE,MAAM,EAAE,cAAc,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAYpE,KAAK,WAAW,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;AAC3C,UAAU,cAAc;IACtB,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,CAAC;CAC/B;AA6BD,wBAAgB,YAAY,CAAC,MAAM,EAAE,SAAS,GAAG,IAAI,CAKpD;AACD,wBAAgB,wBAAwB,CACtC,SAAS,EAAE,MAAM,EACjB,MAAM,EAAE,SAAS,GAChB,IAAI,CAGN;AACD,wBAAgB,0BAA0B,CACxC,SAAS,EAAE,MAAM,EACjB,QAAQ,EAAE,MAAM,GACf,IAAI,CAGN;AACD,wBAAgB,0BAA0B,CAAC,SAAS,EAAE,MAAM,GAAG,IAAI,CAKlE;AACD,wBAAgB,kCAAkC,CAAC,MAAM,EAAE,SAAS,GAAG,IAAI,CAO1E;AACD,wBAAgB,yBAAyB,CACvC,SAAS,EAAE,MAAM,GAChB,MAAM,GAAG,SAAS,CAEpB;AACD,wBAAgB,2BAA2B,CACzC,MAAM,EAAE,SAAS,GAChB,MAAM,GAAG,SAAS,CAKpB;AACD,wBAAgB,qBAAqB,CAAC,CAAC,EACrC,OAAO,EAAE,cAAc,EACvB,EAAE,EAAE,MAAM,CAAC,GACV,CAAC,CAEH;AAID,wBAAgB,YAAY,IAAI,MAAM,GAAG,SAAS,CAGjD;AACD,wBAAgB,YAAY,IAAI,MAAM,GAAG,SAAS,CAEjD;AACD,wBAAgB,cAAc,IAAI,MAAM,GAAG,SAAS,CAEnD;AAqcD,wBAAgB,OAAO,CACrB,OAAO,EAAE,MAAM,EACf,IAAI,CAAC,EAAE,WAAW,EAClB,MAAM,CAAC,EAAE,MAAM,GACd,IAAI,CAEN;AACD,wBAAgB,QAAQ,CACtB,OAAO,EAAE,MAAM,EACf,IAAI,CAAC,EAAE,WAAW,EAClB,MAAM,CAAC,EAAE,MAAM,GACd,IAAI,CAEN;AACD,wBAAgB,SAAS,CACvB,OAAO,EAAE,MAAM,EACf,IAAI,CAAC,EAAE,WAAW,EAClB,MAAM,CAAC,EAAE,MAAM,GACd,IAAI,CAEN;AACD,wBAAgB,OAAO,CACrB,OAAO,EAAE,MAAM,EACf,IAAI,CAAC,EAAE,WAAW,EAClB,MAAM,CAAC,EAAE,MAAM,GACd,IAAI,CAEN;AAcD,wBAAgB,QAAQ,CACtB,OAAO,EAAE,MAAM,EACf,KAAK,CAAC,EAAE,KAAK,GAAG,WAAW,EAC3B,MAAM,CAAC,EAAE,MAAM,GACd,IAAI,CAIN;AACD,wBAAgB,WAAW,CACzB,OAAO,EAAE,MAAM,EACf,KAAK,CAAC,EAAE,KAAK,GAAG,WAAW,EAC3B,MAAM,CAAC,EAAE,MAAM,GACd,IAAI,CAIN;AACD,wBAAgB,WAAW,CAAC,KAAK,EAAE,MAAM,EAAE,SAAS,CAAC,EAAE,MAAM,GAAG,IAAI,CAUnE;AACD,wBAAgB,SAAS,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,CAQhD;AACD,YAAY,EAAE,YAAY,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAC/D,OAAO,EACL,oBAAoB,EACpB,kBAAkB,EAClB,iBAAiB,EACjB,qBAAqB,EACrB,kBAAkB,GACnB,MAAM,cAAc,CAAC;AA2KtB,wBAAgB,uBAAuB,CACrC,KAAK,EAAE,YAAY,EACnB,YAAY,EAAE,MAAM,EACpB,OAAO,CAAC,EAAE;IACR,cAAc,CAAC,EAAE,CAAC,OAAO,EAAE,YAAY,KAAK,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC;IACjE,iBAAiB,CAAC,EAAE,MAAM,CAAC;CAC5B,GACA,eAAe,CAOjB"}
|
package/dist/lib/core.js
CHANGED
|
@@ -8,8 +8,19 @@ export { config, enableHttpMode, serverVersion } from './config.js';
|
|
|
8
8
|
const requestContext = new AsyncLocalStorage({
|
|
9
9
|
name: 'requestContext',
|
|
10
10
|
});
|
|
11
|
+
const LOG_METADATA_MAX_DEPTH = 5;
|
|
12
|
+
const URL_METADATA_KEY_SUFFIXES = [
|
|
13
|
+
'url',
|
|
14
|
+
'uri',
|
|
15
|
+
'href',
|
|
16
|
+
'origin',
|
|
17
|
+
'location',
|
|
18
|
+
'referer',
|
|
19
|
+
'referrer',
|
|
20
|
+
];
|
|
11
21
|
let mcpServer;
|
|
12
22
|
const sessionServers = new Map();
|
|
23
|
+
const sessionOwnerKeys = new Map();
|
|
13
24
|
const sessionMcpLogLevels = new Map();
|
|
14
25
|
let stdioMcpLogLevel;
|
|
15
26
|
let stderrAvailable = true;
|
|
@@ -33,10 +44,16 @@ export function registerMcpSessionServer(sessionId, server) {
|
|
|
33
44
|
return;
|
|
34
45
|
sessionServers.set(sessionId, server);
|
|
35
46
|
}
|
|
47
|
+
export function registerMcpSessionOwnerKey(sessionId, ownerKey) {
|
|
48
|
+
if (!sessionId || !ownerKey)
|
|
49
|
+
return;
|
|
50
|
+
sessionOwnerKeys.set(sessionId, ownerKey);
|
|
51
|
+
}
|
|
36
52
|
export function unregisterMcpSessionServer(sessionId) {
|
|
37
53
|
if (!sessionId)
|
|
38
54
|
return;
|
|
39
55
|
sessionServers.delete(sessionId);
|
|
56
|
+
sessionOwnerKeys.delete(sessionId);
|
|
40
57
|
sessionMcpLogLevels.delete(sessionId);
|
|
41
58
|
}
|
|
42
59
|
export function unregisterMcpSessionServerByServer(server) {
|
|
@@ -44,9 +61,13 @@ export function unregisterMcpSessionServerByServer(server) {
|
|
|
44
61
|
if (mappedServer !== server)
|
|
45
62
|
continue;
|
|
46
63
|
sessionServers.delete(sessionId);
|
|
64
|
+
sessionOwnerKeys.delete(sessionId);
|
|
47
65
|
sessionMcpLogLevels.delete(sessionId);
|
|
48
66
|
}
|
|
49
67
|
}
|
|
68
|
+
export function resolveMcpSessionOwnerKey(sessionId) {
|
|
69
|
+
return sessionOwnerKeys.get(sessionId);
|
|
70
|
+
}
|
|
50
71
|
export function resolveMcpSessionIdByServer(server) {
|
|
51
72
|
for (const [sessionId, mappedServer] of sessionServers.entries()) {
|
|
52
73
|
if (mappedServer === server)
|
|
@@ -91,8 +112,114 @@ function mergeMetadata(meta) {
|
|
|
91
112
|
contextMeta['sessionId'] = sessionId;
|
|
92
113
|
return hasMeta ? { ...contextMeta, ...meta } : contextMeta;
|
|
93
114
|
}
|
|
115
|
+
function isUrlLikeKey(key) {
|
|
116
|
+
const normalized = key
|
|
117
|
+
.replaceAll(/([a-z0-9])([A-Z])/g, '$1_$2')
|
|
118
|
+
.toLowerCase();
|
|
119
|
+
return URL_METADATA_KEY_SUFFIXES.some((suffix) => normalized === suffix ||
|
|
120
|
+
normalized.endsWith(`_${suffix}`) ||
|
|
121
|
+
normalized.endsWith(`-${suffix}`));
|
|
122
|
+
}
|
|
123
|
+
function redactUrlValue(value) {
|
|
124
|
+
if (!URL.canParse(value))
|
|
125
|
+
return value;
|
|
126
|
+
const parsed = URL.parse(value);
|
|
127
|
+
if (!parsed)
|
|
128
|
+
return value;
|
|
129
|
+
if (parsed.username.length === 0 &&
|
|
130
|
+
parsed.password.length === 0 &&
|
|
131
|
+
parsed.search.length === 0 &&
|
|
132
|
+
parsed.hash.length === 0) {
|
|
133
|
+
return value;
|
|
134
|
+
}
|
|
135
|
+
return redactUrl(value);
|
|
136
|
+
}
|
|
137
|
+
function sanitizeLogValue(value, options) {
|
|
138
|
+
const { includeStack, depth = 0, seen = new WeakSet(), key, } = options;
|
|
139
|
+
if (depth >= LOG_METADATA_MAX_DEPTH) {
|
|
140
|
+
return '[truncated]';
|
|
141
|
+
}
|
|
142
|
+
if (value === null ||
|
|
143
|
+
typeof value === 'number' ||
|
|
144
|
+
typeof value === 'boolean') {
|
|
145
|
+
return value;
|
|
146
|
+
}
|
|
147
|
+
if (typeof value === 'string') {
|
|
148
|
+
return key && isUrlLikeKey(key) ? redactUrlValue(value) : value;
|
|
149
|
+
}
|
|
150
|
+
if (typeof value === 'bigint')
|
|
151
|
+
return value.toString();
|
|
152
|
+
if (value instanceof URL)
|
|
153
|
+
return redactUrl(value.toString());
|
|
154
|
+
if (value instanceof Error) {
|
|
155
|
+
const sanitized = {
|
|
156
|
+
error: value.message,
|
|
157
|
+
...(value.name && value.name !== 'Error'
|
|
158
|
+
? { errorName: value.name }
|
|
159
|
+
: {}),
|
|
160
|
+
};
|
|
161
|
+
if ('code' in value) {
|
|
162
|
+
const errorCode = value.code;
|
|
163
|
+
if (typeof errorCode === 'string' || typeof errorCode === 'number') {
|
|
164
|
+
sanitized['code'] = errorCode;
|
|
165
|
+
}
|
|
166
|
+
}
|
|
167
|
+
if ('errno' in value && typeof value.errno === 'number') {
|
|
168
|
+
try {
|
|
169
|
+
const sysMsg = getSystemErrorMessage(value.errno);
|
|
170
|
+
if (sysMsg)
|
|
171
|
+
sanitized['sysError'] = sysMsg;
|
|
172
|
+
}
|
|
173
|
+
catch {
|
|
174
|
+
// ignore
|
|
175
|
+
}
|
|
176
|
+
}
|
|
177
|
+
if (includeStack && value.stack) {
|
|
178
|
+
sanitized['stack'] = value.stack;
|
|
179
|
+
}
|
|
180
|
+
return sanitized;
|
|
181
|
+
}
|
|
182
|
+
if (Array.isArray(value)) {
|
|
183
|
+
return value
|
|
184
|
+
.map((entry) => sanitizeLogValue(entry, { includeStack, depth: depth + 1, seen }))
|
|
185
|
+
.filter((entry) => entry !== undefined);
|
|
186
|
+
}
|
|
187
|
+
if (isPlainLogObject(value)) {
|
|
188
|
+
if (seen.has(value))
|
|
189
|
+
return '[circular]';
|
|
190
|
+
seen.add(value);
|
|
191
|
+
const sanitized = {};
|
|
192
|
+
for (const [entryKey, entryValue] of Object.entries(value)) {
|
|
193
|
+
if (isSensitiveKey(entryKey) &&
|
|
194
|
+
(!includeStack || entryKey.toLowerCase() !== 'stack')) {
|
|
195
|
+
continue;
|
|
196
|
+
}
|
|
197
|
+
const normalized = sanitizeLogValue(entryValue, {
|
|
198
|
+
includeStack,
|
|
199
|
+
depth: depth + 1,
|
|
200
|
+
seen,
|
|
201
|
+
key: entryKey,
|
|
202
|
+
});
|
|
203
|
+
if (normalized !== undefined) {
|
|
204
|
+
sanitized[entryKey] = normalized;
|
|
205
|
+
}
|
|
206
|
+
}
|
|
207
|
+
return sanitized;
|
|
208
|
+
}
|
|
209
|
+
return undefined;
|
|
210
|
+
}
|
|
211
|
+
function sanitizeLogMetadata(meta, options) {
|
|
212
|
+
if (!meta || Object.keys(meta).length === 0)
|
|
213
|
+
return undefined;
|
|
214
|
+
const sanitized = sanitizeLogValue(meta, options);
|
|
215
|
+
return isPlainLogObject(sanitized) && Object.keys(sanitized).length > 0
|
|
216
|
+
? sanitized
|
|
217
|
+
: undefined;
|
|
218
|
+
}
|
|
94
219
|
function formatMetadata(meta) {
|
|
95
|
-
const merged = mergeMetadata(meta)
|
|
220
|
+
const merged = sanitizeLogMetadata(mergeMetadata(meta), {
|
|
221
|
+
includeStack: true,
|
|
222
|
+
});
|
|
96
223
|
if (!merged)
|
|
97
224
|
return '';
|
|
98
225
|
return ` ${inspect(merged, { breakLength: Infinity, colors: false, compact: true, sorted: true })}`;
|
|
@@ -100,12 +227,15 @@ function formatMetadata(meta) {
|
|
|
100
227
|
function createTimestamp() {
|
|
101
228
|
return new Date().toISOString();
|
|
102
229
|
}
|
|
103
|
-
function formatLogEntry(level, message, meta) {
|
|
230
|
+
function formatLogEntry(level, message, meta, logger) {
|
|
104
231
|
if (config.logging.format === 'json') {
|
|
105
|
-
const merged = mergeMetadata(meta)
|
|
232
|
+
const merged = sanitizeLogMetadata(mergeMetadata(meta), {
|
|
233
|
+
includeStack: true,
|
|
234
|
+
});
|
|
106
235
|
const entry = {
|
|
107
236
|
timestamp: createTimestamp(),
|
|
108
237
|
level: level.toUpperCase(),
|
|
238
|
+
...(logger ? { logger } : {}),
|
|
109
239
|
message,
|
|
110
240
|
};
|
|
111
241
|
if (merged) {
|
|
@@ -113,13 +243,16 @@ function formatLogEntry(level, message, meta) {
|
|
|
113
243
|
}
|
|
114
244
|
return JSON.stringify(entry);
|
|
115
245
|
}
|
|
116
|
-
|
|
246
|
+
const loggerTag = logger ? ` [${logger}]` : '';
|
|
247
|
+
return `[${createTimestamp()}] ${level.toUpperCase()}${loggerTag}: ${message}${formatMetadata(meta)}`;
|
|
117
248
|
}
|
|
118
249
|
const LEVEL_PRIORITY = {
|
|
119
250
|
debug: 0,
|
|
120
251
|
info: 1,
|
|
121
|
-
|
|
122
|
-
|
|
252
|
+
notice: 2,
|
|
253
|
+
warn: 3,
|
|
254
|
+
error: 4,
|
|
255
|
+
critical: 5,
|
|
123
256
|
};
|
|
124
257
|
function shouldLog(level) {
|
|
125
258
|
return LEVEL_PRIORITY[level] >= LEVEL_PRIORITY[config.logging.level];
|
|
@@ -150,7 +283,9 @@ function normalizeLogLevel(level) {
|
|
|
150
283
|
}
|
|
151
284
|
// Map internal log levels to standard RFC 5424 severities
|
|
152
285
|
function toMcpLogLevel(level) {
|
|
153
|
-
|
|
286
|
+
if (level === 'warn')
|
|
287
|
+
return 'warning';
|
|
288
|
+
return level;
|
|
154
289
|
}
|
|
155
290
|
function shouldForwardMcpLog(level, sessionId) {
|
|
156
291
|
const emittedLevel = toMcpLogLevel(level);
|
|
@@ -179,56 +314,36 @@ function resolveErrorText(err) {
|
|
|
179
314
|
return err;
|
|
180
315
|
return 'unknown error';
|
|
181
316
|
}
|
|
182
|
-
const
|
|
183
|
-
|
|
317
|
+
const MCP_LOG_SENSITIVE_PATTERNS = [
|
|
318
|
+
'password',
|
|
319
|
+
'secret',
|
|
320
|
+
'token',
|
|
321
|
+
'authorization',
|
|
322
|
+
'credential',
|
|
323
|
+
'key',
|
|
324
|
+
'cookie',
|
|
325
|
+
'stack',
|
|
326
|
+
];
|
|
327
|
+
function isSensitiveKey(key) {
|
|
328
|
+
const lowerKey = key.toLowerCase();
|
|
329
|
+
return MCP_LOG_SENSITIVE_PATTERNS.some((p) => lowerKey.includes(p));
|
|
330
|
+
}
|
|
184
331
|
function isPlainLogObject(value) {
|
|
185
332
|
return typeof value === 'object' && value !== null && !Array.isArray(value);
|
|
186
333
|
}
|
|
187
|
-
function sanitizeMcpLogValue(value, depth = 0) {
|
|
188
|
-
if (depth >= MCP_LOG_MAX_DEPTH) {
|
|
189
|
-
return '[truncated]';
|
|
190
|
-
}
|
|
191
|
-
if (value === null ||
|
|
192
|
-
typeof value === 'string' ||
|
|
193
|
-
typeof value === 'number' ||
|
|
194
|
-
typeof value === 'boolean') {
|
|
195
|
-
return value;
|
|
196
|
-
}
|
|
197
|
-
if (typeof value === 'bigint')
|
|
198
|
-
return value.toString();
|
|
199
|
-
if (value instanceof Error)
|
|
200
|
-
return value.message;
|
|
201
|
-
if (Array.isArray(value)) {
|
|
202
|
-
return value
|
|
203
|
-
.map((entry) => sanitizeMcpLogValue(entry, depth + 1))
|
|
204
|
-
.filter((entry) => entry !== undefined);
|
|
205
|
-
}
|
|
206
|
-
if (isPlainLogObject(value)) {
|
|
207
|
-
const sanitized = {};
|
|
208
|
-
for (const [key, entry] of Object.entries(value)) {
|
|
209
|
-
if (MCP_LOG_META_BLOCKLIST.has(key))
|
|
210
|
-
continue;
|
|
211
|
-
const normalized = sanitizeMcpLogValue(entry, depth + 1);
|
|
212
|
-
if (normalized !== undefined) {
|
|
213
|
-
sanitized[key] = normalized;
|
|
214
|
-
}
|
|
215
|
-
}
|
|
216
|
-
return sanitized;
|
|
217
|
-
}
|
|
218
|
-
return undefined;
|
|
219
|
-
}
|
|
220
334
|
function buildMcpLogData(message, meta) {
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
}
|
|
224
|
-
const sanitized = sanitizeMcpLogValue(meta);
|
|
225
|
-
if (!isPlainLogObject(sanitized) || Object.keys(sanitized).length === 0) {
|
|
335
|
+
const sanitized = sanitizeLogMetadata(meta, { includeStack: false });
|
|
336
|
+
if (!sanitized) {
|
|
226
337
|
return { message };
|
|
227
338
|
}
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
339
|
+
// Ensure `message` from system isn't overwritten by `meta.message`,
|
|
340
|
+
// but keep the original meta message around if present.
|
|
341
|
+
const payload = { ...sanitized };
|
|
342
|
+
if ('message' in payload) {
|
|
343
|
+
payload['_message'] = payload['message'];
|
|
344
|
+
}
|
|
345
|
+
payload['message'] = message;
|
|
346
|
+
return payload;
|
|
232
347
|
}
|
|
233
348
|
function safeWriteStderr(line) {
|
|
234
349
|
if (!stderrAvailable)
|
|
@@ -245,13 +360,42 @@ function safeWriteStderr(line) {
|
|
|
245
360
|
stderrAvailable = false;
|
|
246
361
|
}
|
|
247
362
|
}
|
|
248
|
-
|
|
363
|
+
const DEFAULT_LOGGER = 'fetch-url-mcp';
|
|
364
|
+
const MCP_RATE_LIMIT_CAPACITY = 200;
|
|
365
|
+
const MCP_RATE_LIMIT_REFILL_RATE = 50; // per second
|
|
366
|
+
const mcpRateLimiters = new Map();
|
|
367
|
+
// Returns true if allowed, false if dropped.
|
|
368
|
+
function checkMcpRateLimit(sessionId = '') {
|
|
369
|
+
const now = Date.now();
|
|
370
|
+
let limit = mcpRateLimiters.get(sessionId);
|
|
371
|
+
if (!limit) {
|
|
372
|
+
limit = { tokens: MCP_RATE_LIMIT_CAPACITY, lastRefill: now, dropped: 0 };
|
|
373
|
+
mcpRateLimiters.set(sessionId, limit);
|
|
374
|
+
}
|
|
375
|
+
const elapsed = now - limit.lastRefill;
|
|
376
|
+
if (elapsed >= 1000) {
|
|
377
|
+
const refillTokens = Math.floor((elapsed / 1000) * MCP_RATE_LIMIT_REFILL_RATE);
|
|
378
|
+
limit.tokens = Math.min(MCP_RATE_LIMIT_CAPACITY, limit.tokens + refillTokens);
|
|
379
|
+
limit.lastRefill = now;
|
|
380
|
+
}
|
|
381
|
+
if (limit.tokens > 0) {
|
|
382
|
+
limit.tokens--;
|
|
383
|
+
const droppedCount = limit.dropped;
|
|
384
|
+
if (droppedCount > 0) {
|
|
385
|
+
limit.dropped = 0;
|
|
386
|
+
}
|
|
387
|
+
return { allowed: true, droppedCount };
|
|
388
|
+
}
|
|
389
|
+
limit.dropped++;
|
|
390
|
+
return { allowed: false, droppedCount: 0 };
|
|
391
|
+
}
|
|
392
|
+
function writeLog(level, message, meta, logger) {
|
|
249
393
|
const sessionId = getSessionId();
|
|
250
394
|
if (shouldLog(level)) {
|
|
251
|
-
const line = formatLogEntry(level, message, meta);
|
|
395
|
+
const line = formatLogEntry(level, message, meta, logger);
|
|
252
396
|
safeWriteStderr(`${stripVTControlCharacters(line)}\n`);
|
|
253
397
|
}
|
|
254
|
-
forwardMcpLog(level, message, meta, sessionId);
|
|
398
|
+
forwardMcpLog(level, message, meta, sessionId, logger);
|
|
255
399
|
}
|
|
256
400
|
function resolveLogServer(sessionId) {
|
|
257
401
|
const server = sessionId ? sessionServers.get(sessionId) : mcpServer;
|
|
@@ -259,18 +403,34 @@ function resolveLogServer(sessionId) {
|
|
|
259
403
|
return undefined;
|
|
260
404
|
return server.isConnected() ? server : undefined;
|
|
261
405
|
}
|
|
262
|
-
function forwardMcpLog(level, message, meta, sessionId) {
|
|
406
|
+
function forwardMcpLog(level, message, meta, sessionId, logger) {
|
|
263
407
|
const server = resolveLogServer(sessionId);
|
|
264
408
|
if (!server)
|
|
265
409
|
return;
|
|
266
410
|
if (!shouldForwardMcpLog(level, sessionId))
|
|
267
411
|
return;
|
|
412
|
+
const { allowed, droppedCount } = checkMcpRateLimit(sessionId);
|
|
413
|
+
if (!allowed)
|
|
414
|
+
return;
|
|
268
415
|
try {
|
|
416
|
+
const safeLogger = logger ?? DEFAULT_LOGGER;
|
|
417
|
+
if (droppedCount > 0) {
|
|
418
|
+
server.server
|
|
419
|
+
.sendLoggingMessage({
|
|
420
|
+
level: 'warning',
|
|
421
|
+
logger: safeLogger,
|
|
422
|
+
data: {
|
|
423
|
+
message: 'MCP log messages dropped due to high volume',
|
|
424
|
+
droppedCount,
|
|
425
|
+
},
|
|
426
|
+
}, sessionId)
|
|
427
|
+
.catch(() => { });
|
|
428
|
+
}
|
|
269
429
|
server.server
|
|
270
430
|
.sendLoggingMessage({
|
|
271
|
-
level: level
|
|
272
|
-
logger:
|
|
273
|
-
data: buildMcpLogData(message, meta),
|
|
431
|
+
level: toMcpLogLevel(level),
|
|
432
|
+
logger: safeLogger,
|
|
433
|
+
data: buildMcpLogData(message, mergeMetadata(meta)),
|
|
274
434
|
}, sessionId)
|
|
275
435
|
.catch((err) => {
|
|
276
436
|
if (!isDebugEnabled())
|
|
@@ -286,14 +446,17 @@ function forwardMcpLog(level, message, meta, sessionId) {
|
|
|
286
446
|
safeWriteStderr(`[${createTimestamp()}] WARN: Failed to forward log to MCP (sync error): ${errorText}\n`);
|
|
287
447
|
}
|
|
288
448
|
}
|
|
289
|
-
export function logInfo(message, meta) {
|
|
290
|
-
writeLog('info', message, meta);
|
|
449
|
+
export function logInfo(message, meta, logger) {
|
|
450
|
+
writeLog('info', message, meta, logger);
|
|
451
|
+
}
|
|
452
|
+
export function logDebug(message, meta, logger) {
|
|
453
|
+
writeLog('debug', message, meta, logger);
|
|
291
454
|
}
|
|
292
|
-
export function
|
|
293
|
-
writeLog('
|
|
455
|
+
export function logNotice(message, meta, logger) {
|
|
456
|
+
writeLog('notice', message, meta, logger);
|
|
294
457
|
}
|
|
295
|
-
export function logWarn(message, meta) {
|
|
296
|
-
writeLog('warn', message, meta);
|
|
458
|
+
export function logWarn(message, meta, logger) {
|
|
459
|
+
writeLog('warn', message, meta, logger);
|
|
297
460
|
}
|
|
298
461
|
function formatErrorMeta(error) {
|
|
299
462
|
const meta = { error: error.message, stack: error.stack };
|
|
@@ -309,9 +472,13 @@ function formatErrorMeta(error) {
|
|
|
309
472
|
}
|
|
310
473
|
return meta;
|
|
311
474
|
}
|
|
312
|
-
export function logError(message, error) {
|
|
475
|
+
export function logError(message, error, logger) {
|
|
313
476
|
const errorMeta = error instanceof Error ? formatErrorMeta(error) : (error ?? {});
|
|
314
|
-
writeLog('error', message, errorMeta);
|
|
477
|
+
writeLog('error', message, errorMeta, logger);
|
|
478
|
+
}
|
|
479
|
+
export function logCritical(message, error, logger) {
|
|
480
|
+
const errorMeta = error instanceof Error ? formatErrorMeta(error) : (error ?? {});
|
|
481
|
+
writeLog('critical', message, errorMeta, logger);
|
|
315
482
|
}
|
|
316
483
|
export function setLogLevel(level, sessionId) {
|
|
317
484
|
const normalized = normalizeLogLevel(level);
|
|
@@ -344,12 +511,12 @@ function getCleanupIntervalMs(sessionTtlMs) {
|
|
|
344
511
|
function handleSessionCleanupError(error) {
|
|
345
512
|
if (isAbortError(error))
|
|
346
513
|
return;
|
|
347
|
-
logWarn('Session cleanup loop failed', { error: getErrorMessage(error) });
|
|
514
|
+
logWarn('Session cleanup loop failed', { error: getErrorMessage(error) }, 'session');
|
|
348
515
|
}
|
|
349
516
|
function logRejectedSettledResults(results, message) {
|
|
350
517
|
for (const result of results) {
|
|
351
518
|
if (result.status === 'rejected') {
|
|
352
|
-
logWarn(message, { error: getErrorMessage(result.reason) });
|
|
519
|
+
logWarn(message, { error: getErrorMessage(result.reason) }, 'session');
|
|
353
520
|
}
|
|
354
521
|
}
|
|
355
522
|
}
|
|
@@ -389,7 +556,7 @@ class SessionCleanupLoop {
|
|
|
389
556
|
logInfo('Expired sessions evicted', {
|
|
390
557
|
evicted: evicted.length,
|
|
391
558
|
timestamp: new Date(now).toISOString(),
|
|
392
|
-
});
|
|
559
|
+
}, 'session');
|
|
393
560
|
}
|
|
394
561
|
}
|
|
395
562
|
async closeExpiredSession(sessionId, session) {
|
|
@@ -400,7 +567,7 @@ class SessionCleanupLoop {
|
|
|
400
567
|
catch (error) {
|
|
401
568
|
logWarn('Expired session pre-close hook failed', {
|
|
402
569
|
error: getErrorMessage(error),
|
|
403
|
-
});
|
|
570
|
+
}, 'session');
|
|
404
571
|
}
|
|
405
572
|
}
|
|
406
573
|
const closePromise = Promise.allSettled([
|
|
@@ -429,7 +596,7 @@ class SessionCleanupLoop {
|
|
|
429
596
|
catch (error) {
|
|
430
597
|
logWarn('Session close operation failed or timed out', {
|
|
431
598
|
error: getErrorMessage(error),
|
|
432
|
-
});
|
|
599
|
+
}, 'session');
|
|
433
600
|
}
|
|
434
601
|
finally {
|
|
435
602
|
if (timeoutId) {
|
|
@@ -442,7 +609,7 @@ class SessionCleanupLoop {
|
|
|
442
609
|
catch (error) {
|
|
443
610
|
logWarn('Failed to unregister session server', {
|
|
444
611
|
error: getErrorMessage(error),
|
|
445
|
-
});
|
|
612
|
+
}, 'session');
|
|
446
613
|
}
|
|
447
614
|
}
|
|
448
615
|
logCloseFailure(target, error) {
|
|
@@ -450,7 +617,7 @@ class SessionCleanupLoop {
|
|
|
450
617
|
return;
|
|
451
618
|
logWarn(`Failed to close expired session ${target}`, {
|
|
452
619
|
error: getErrorMessage(error),
|
|
453
|
-
});
|
|
620
|
+
}, 'session');
|
|
454
621
|
}
|
|
455
622
|
}
|
|
456
623
|
export function startSessionCleanupLoop(store, sessionTtlMs, options) {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"fetch-pipeline.d.ts","sourceRoot":"","sources":["../../src/lib/fetch-pipeline.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,KAAK,uBAAuB,EAAE,MAAM,uBAAuB,CAAC;AAOrE,OAAO,EAAE,UAAU,EAAE,MAAM,YAAY,CAAC;AAExC,OAAO,EAAE,UAAU,EAAE,CAAC;AAGtB,MAAM,WAAW,mBAAmB;IAClC,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,WAAW,EAAE,MAAM,CAAC;IACpB,SAAS,CAAC,EAAE,OAAO,CAAC;CACrB;AACD,UAAU,mBAAmB;IAC3B,MAAM,EAAE,UAAU,CAAC;IACnB,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,CAAC,EAAE,OAAO,CAAC;CACrB;AAmGD,wBAAgB,sBAAsB,CACpC,QAAQ,EAAE,MAAM,GAAG,SAAS,EAC5B,OAAO,GAAE;IAAE,QAAQ,CAAC,EAAE,MAAM,CAAA;CAAO,GAClC,MAAM,GAAG,SAAS,CAQpB;AAyBD,UAAU,oBAAoB,CAAC,CAAC;IAC9B,GAAG,EAAE,MAAM,CAAC;IACZ,MAAM,CAAC,EAAE,WAAW,CAAC;IACrB,OAAO,CAAC,EAAE,CAAC,KAAK,EAAE,gBAAgB,KAAK,IAAI,CAAC;IAC5C,SAAS,EAAE,CAAC,KAAK,EAAE,mBAAmB,EAAE,GAAG,EAAE,MAAM,KAAK,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;CACxE;AACD,MAAM,WAAW,cAAc,CAAC,CAAC;IAC/B,IAAI,EAAE,CAAC,CAAC;IACR,GAAG,EAAE,MAAM,CAAC;IACZ,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;CACnB;AACD,MAAM,MAAM,gBAAgB,GACxB,aAAa,GACb,cAAc,GACd,gBAAgB,GAChB,iBAAiB,GACjB,gBAAgB,GAChB,iBAAiB,CAAC;AAmBtB,wBAAsB,oBAAoB,CAAC,CAAC,EAC1C,OAAO,EAAE,oBAAoB,CAAC,CAAC,CAAC,GAC/B,OAAO,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC,
|
|
1
|
+
{"version":3,"file":"fetch-pipeline.d.ts","sourceRoot":"","sources":["../../src/lib/fetch-pipeline.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,KAAK,uBAAuB,EAAE,MAAM,uBAAuB,CAAC;AAOrE,OAAO,EAAE,UAAU,EAAE,MAAM,YAAY,CAAC;AAExC,OAAO,EAAE,UAAU,EAAE,CAAC;AAGtB,MAAM,WAAW,mBAAmB;IAClC,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,WAAW,EAAE,MAAM,CAAC;IACpB,SAAS,CAAC,EAAE,OAAO,CAAC;CACrB;AACD,UAAU,mBAAmB;IAC3B,MAAM,EAAE,UAAU,CAAC;IACnB,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,CAAC,EAAE,OAAO,CAAC;CACrB;AAmGD,wBAAgB,sBAAsB,CACpC,QAAQ,EAAE,MAAM,GAAG,SAAS,EAC5B,OAAO,GAAE;IAAE,QAAQ,CAAC,EAAE,MAAM,CAAA;CAAO,GAClC,MAAM,GAAG,SAAS,CAQpB;AAyBD,UAAU,oBAAoB,CAAC,CAAC;IAC9B,GAAG,EAAE,MAAM,CAAC;IACZ,MAAM,CAAC,EAAE,WAAW,CAAC;IACrB,OAAO,CAAC,EAAE,CAAC,KAAK,EAAE,gBAAgB,KAAK,IAAI,CAAC;IAC5C,SAAS,EAAE,CAAC,KAAK,EAAE,mBAAmB,EAAE,GAAG,EAAE,MAAM,KAAK,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;CACxE;AACD,MAAM,WAAW,cAAc,CAAC,CAAC;IAC/B,IAAI,EAAE,CAAC,CAAC;IACR,GAAG,EAAE,MAAM,CAAC;IACZ,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;CACnB;AACD,MAAM,MAAM,gBAAgB,GACxB,aAAa,GACb,cAAc,GACd,gBAAgB,GAChB,iBAAiB,GACjB,gBAAgB,GAChB,iBAAiB,CAAC;AAmBtB,wBAAsB,oBAAoB,CAAC,CAAC,EAC1C,OAAO,EAAE,oBAAoB,CAAC,CAAC,CAAC,GAC/B,OAAO,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC,CAiD5B;AAED,MAAM,MAAM,sBAAsB,GAAG,uBAAuB,GAAG;IAC7D,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;CAC1B,CAAC;AAwBF,eAAO,MAAM,iBAAiB,GAC5B,OAAO,mBAAmB,EAC1B,KAAK,MAAM,EACX,SAAS,WAAW,KACnB,OAAO,CAAC,sBAAsB,CAchC,CAAC;AAEF,UAAU,oBAAoB;IAC5B,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC;IACrB,QAAQ,CAAC,MAAM,CAAC,EAAE,WAAW,CAAC;IAC9B,QAAQ,CAAC,OAAO,CAAC,EAAE,CAAC,KAAK,EAAE,gBAAgB,KAAK,IAAI,CAAC;IACrD,QAAQ,CAAC,SAAS,EAAE,CAClB,KAAK,EAAE,mBAAmB,EAC1B,aAAa,EAAE,MAAM,KAClB,sBAAsB,GAAG,OAAO,CAAC,sBAAsB,CAAC,CAAC;CAC/D;AACD,UAAU,eAAe;IACvB,QAAQ,CAAC,oBAAoB,CAAC,EAAE,OAAO,oBAAoB,CAAC;CAC7D;AAMD,wBAAsB,kBAAkB,CACtC,OAAO,EAAE,oBAAoB,EAC7B,IAAI,GAAE,eAAoB,GACzB,OAAO,CAAC;IACT,QAAQ,EAAE,cAAc,CAAC,sBAAsB,CAAC,CAAC;IACjD,YAAY,EAAE,mBAAmB,CAAC;CACnC,CAAC,CAyBD"}
|
|
@@ -116,11 +116,17 @@ export async function executeFetchPipeline(options) {
|
|
|
116
116
|
if (resolvedUrl.transformed) {
|
|
117
117
|
logDebug('Using transformed raw content URL', {
|
|
118
118
|
original: resolvedUrl.originalUrl,
|
|
119
|
-
});
|
|
119
|
+
}, 'fetch');
|
|
120
120
|
}
|
|
121
121
|
options.onStage?.('fetch_remote');
|
|
122
|
-
logDebug('Fetching URL', { url: resolvedUrl.normalizedUrl });
|
|
122
|
+
logDebug('Fetching URL', { url: resolvedUrl.normalizedUrl }, 'fetch');
|
|
123
123
|
const { buffer, encoding, truncated, finalUrl } = await fetchNormalizedUrlBuffer(resolvedUrl.normalizedUrl, withSignal(options.signal));
|
|
124
|
+
if (finalUrl && finalUrl !== resolvedUrl.normalizedUrl) {
|
|
125
|
+
logDebug('Fetch redirected', {
|
|
126
|
+
fromUrl: resolvedUrl.normalizedUrl,
|
|
127
|
+
toUrl: finalUrl,
|
|
128
|
+
}, 'fetch');
|
|
129
|
+
}
|
|
124
130
|
options.onStage?.('response_ready');
|
|
125
131
|
options.onStage?.('transform_start');
|
|
126
132
|
const resolvedFinalUrl = finalUrl || resolvedUrl.normalizedUrl;
|
|
@@ -166,5 +172,12 @@ export async function performSharedFetch(options, deps = {}) {
|
|
|
166
172
|
options.onStage?.('prepare_output');
|
|
167
173
|
options.onStage?.('finalize_output');
|
|
168
174
|
const inlineResult = applyInlineContentLimit(pipeline.data.content, pipeline.data.truncated);
|
|
175
|
+
if (inlineResult.truncated) {
|
|
176
|
+
logDebug('Inline markdown truncated for response payload', {
|
|
177
|
+
url: pipeline.finalUrl ?? pipeline.url,
|
|
178
|
+
contentSize: inlineResult.contentSize,
|
|
179
|
+
maxInlineChars: config.constants.maxInlineContentChars,
|
|
180
|
+
}, 'fetch');
|
|
181
|
+
}
|
|
169
182
|
return { pipeline, inlineResult };
|
|
170
183
|
}
|
package/dist/lib/http.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"http.d.ts","sourceRoot":"","sources":["../../src/lib/http.ts"],"names":[],"mappings":"AASA,OAAO,EAAE,KAAK,EAAmB,MAAM,QAAQ,CAAC;AAYhD,OAAO,EAOL,KAAK,eAAe,EAGrB,MAAM,UAAU,CAAC;
|
|
1
|
+
{"version":3,"file":"http.d.ts","sourceRoot":"","sources":["../../src/lib/http.ts"],"names":[],"mappings":"AASA,OAAO,EAAE,KAAK,EAAmB,MAAM,QAAQ,CAAC;AAYhD,OAAO,EAOL,KAAK,eAAe,EAGrB,MAAM,UAAU,CAAC;AA+vClB,UAAU,qBAAqB;IAC7B,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,GAAG,EAAE,MAAM,CAAC;IACZ,MAAM,EAAE,MAAM,CAAC;IACf,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AAoKD,UAAU,YAAY;IACpB,MAAM,CAAC,EAAE,WAAW,CAAC;CACtB;AA6ND,wBAAgB,WAAW,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAE/C;AACD,wBAAgB,YAAY,CAAC,SAAS,EAAE,MAAM,GAAG;IAC/C,aAAa,EAAE,MAAM,CAAC;IACtB,QAAQ,EAAE,MAAM,CAAC;CAClB,CAEA;AACD,wBAAgB,uBAAuB,CAAC,SAAS,EAAE,MAAM,GAAG,MAAM,CAEjE;AACD,wBAAgB,iBAAiB,CAAC,GAAG,EAAE,MAAM,GAAG,eAAe,CAE9D;AACD,wBAAgB,mBAAmB,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAExD;AACD,wBAAgB,mBAAmB,CACjC,GAAG,EAAE,MAAM,EACX,MAAM,EAAE,MAAM,GACb,qBAAqB,CAEvB;AACD,wBAAgB,mBAAmB,CACjC,OAAO,EAAE,qBAAqB,EAC9B,QAAQ,EAAE,QAAQ,EAClB,WAAW,CAAC,EAAE,MAAM,GACnB,IAAI,CAEN;AACD,wBAAgB,gBAAgB,CAC9B,OAAO,EAAE,qBAAqB,EAC9B,KAAK,EAAE,OAAO,EACd,MAAM,CAAC,EAAE,MAAM,GACd,IAAI,CAEN;AACD,wBAAsB,kBAAkB,CACtC,GAAG,EAAE,MAAM,EACX,IAAI,EAAE,WAAW,EACjB,YAAY,EAAE,MAAM,GACnB,OAAO,CAAC;IAAE,QAAQ,EAAE,QAAQ,CAAC;IAAC,GAAG,EAAE,MAAM,CAAC;IAAC,KAAK,CAAC,EAAE,KAAK,CAAA;CAAE,CAAC,CAE7D;AACD,wBAAsB,gBAAgB,CACpC,QAAQ,EAAE,QAAQ,EAClB,GAAG,EAAE,MAAM,EACX,QAAQ,EAAE,MAAM,EAChB,MAAM,CAAC,EAAE,WAAW,EACpB,QAAQ,CAAC,EAAE,MAAM,GAChB,OAAO,CAAC;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,IAAI,EAAE,MAAM,CAAA;CAAE,CAAC,CASzC;AACD,wBAAsB,kBAAkB,CACtC,aAAa,EAAE,MAAM,EACrB,OAAO,CAAC,EAAE,YAAY,GACrB,OAAO,CAAC,MAAM,CAAC,CAEjB;AACD,wBAAsB,wBAAwB,CAC5C,aAAa,EAAE,MAAM,EACrB,OAAO,CAAC,EAAE,YAAY,GACrB,OAAO,CAAC;IACT,MAAM,EAAE,UAAU,CAAC;IACnB,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,OAAO,CAAC;IACnB,QAAQ,EAAE,MAAM,CAAC;CAClB,CAAC,CAED"}
|
package/dist/lib/http.js
CHANGED
|
@@ -536,7 +536,7 @@ function assertSupportedContentType(contentType, url) {
|
|
|
536
536
|
if (!mediaType) {
|
|
537
537
|
logDebug('No Content-Type header; relying on binary-content detection', {
|
|
538
538
|
url: redactUrl(url),
|
|
539
|
-
});
|
|
539
|
+
}, 'fetch');
|
|
540
540
|
return;
|
|
541
541
|
}
|
|
542
542
|
if (!isTextLikeMediaType(mediaType)) {
|