@j0hanz/fetch-url-mcp 1.10.15 → 1.10.17
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/http/native.d.ts.map +1 -1
- package/dist/http/native.js +36 -24
- package/dist/lib/core.d.ts.map +1 -1
- package/dist/lib/core.js +54 -34
- package/dist/lib/dom-prep.d.ts +4 -0
- package/dist/lib/dom-prep.d.ts.map +1 -1
- package/dist/lib/dom-prep.js +318 -71
- package/dist/lib/http.d.ts.map +1 -1
- package/dist/lib/http.js +84 -53
- package/dist/lib/md-cleanup.d.ts.map +1 -1
- package/dist/lib/md-cleanup.js +38 -37
- package/dist/lib/progress.d.ts.map +1 -1
- package/dist/lib/progress.js +3 -6
- package/dist/lib/url.d.ts +7 -2
- package/dist/lib/url.d.ts.map +1 -1
- package/dist/lib/url.js +79 -81
- package/dist/lib/utils.d.ts +26 -3
- package/dist/lib/utils.d.ts.map +1 -1
- package/dist/lib/utils.js +102 -4
- package/dist/tools/fetch-url-progress.d.ts +1 -1
- package/dist/tools/fetch-url-progress.d.ts.map +1 -1
- package/dist/tools/fetch-url-progress.js +35 -19
- package/dist/tools/fetch-url.js +4 -4
- package/dist/transform/html-translators.d.ts.map +1 -1
- package/dist/transform/html-translators.js +116 -81
- package/dist/transform/transform.d.ts +3 -3
- package/dist/transform/transform.d.ts.map +1 -1
- package/dist/transform/transform.js +103 -386
- package/dist/transform/worker-pool.d.ts +1 -0
- package/dist/transform/worker-pool.d.ts.map +1 -1
- package/dist/transform/worker-pool.js +16 -6
- package/package.json +1 -1
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"native.d.ts","sourceRoot":"","sources":["../../src/http/native.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"native.d.ts","sourceRoot":"","sources":["../../src/http/native.ts"],"names":[],"mappings":"AAokCA,wBAAsB,eAAe,IAAI,OAAO,CAAC;IAC/C,QAAQ,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;IAC5C,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;CACd,CAAC,CA0DD"}
|
package/dist/http/native.js
CHANGED
|
@@ -335,7 +335,9 @@ class McpSessionGateway {
|
|
|
335
335
|
void teardownUnregisteredSessionResources(unpublishedSession, 'session-init-timeout');
|
|
336
336
|
}, config.server.sessionInitTimeoutMs);
|
|
337
337
|
initTimeout.unref();
|
|
338
|
+
const connectState = { transportClosed: false };
|
|
338
339
|
transportImpl.onclose = () => {
|
|
340
|
+
connectState.transportClosed = true;
|
|
339
341
|
clearTimeout(initTimeout);
|
|
340
342
|
this.sessionInitTimeouts.delete(newSessionId);
|
|
341
343
|
tracker.releaseSlot();
|
|
@@ -351,6 +353,10 @@ class McpSessionGateway {
|
|
|
351
353
|
throw err;
|
|
352
354
|
}
|
|
353
355
|
tracker.releaseSlot();
|
|
356
|
+
if (connectState.transportClosed) {
|
|
357
|
+
void teardownUnregisteredSessionResources(unpublishedSession, 'session-closed-during-connect');
|
|
358
|
+
return null;
|
|
359
|
+
}
|
|
354
360
|
this.store.set(newSessionId, {
|
|
355
361
|
server: sessionServer,
|
|
356
362
|
transport: transportImpl,
|
|
@@ -524,6 +530,35 @@ class HttpDispatcher {
|
|
|
524
530
|
}
|
|
525
531
|
}
|
|
526
532
|
}
|
|
533
|
+
// ---------------------------------------------------------------------------
|
|
534
|
+
// Body parse error responses
|
|
535
|
+
// ---------------------------------------------------------------------------
|
|
536
|
+
function sendBodyParseError(ctx, bodyErrorKind, rawReq) {
|
|
537
|
+
if (ctx.url.pathname === '/mcp') {
|
|
538
|
+
switch (bodyErrorKind) {
|
|
539
|
+
case 'payload-too-large':
|
|
540
|
+
sendError(ctx.res, -32600, 'Request body too large', 413, null);
|
|
541
|
+
break;
|
|
542
|
+
case 'read-failed':
|
|
543
|
+
if (!rawReq.destroyed) {
|
|
544
|
+
sendError(ctx.res, -32600, 'Request body read failed', 400, null);
|
|
545
|
+
}
|
|
546
|
+
break;
|
|
547
|
+
case 'invalid-json':
|
|
548
|
+
default:
|
|
549
|
+
sendError(ctx.res, -32700, 'Parse error', 400, null);
|
|
550
|
+
break;
|
|
551
|
+
}
|
|
552
|
+
}
|
|
553
|
+
else {
|
|
554
|
+
sendJson(ctx.res, bodyErrorKind === 'payload-too-large' ? 413 : 400, {
|
|
555
|
+
error: bodyErrorKind === 'payload-too-large'
|
|
556
|
+
? 'Payload too large'
|
|
557
|
+
: 'Invalid JSON',
|
|
558
|
+
});
|
|
559
|
+
}
|
|
560
|
+
drainRequest(rawReq);
|
|
561
|
+
}
|
|
527
562
|
class HttpRequestPipeline {
|
|
528
563
|
rateLimiter;
|
|
529
564
|
dispatcher;
|
|
@@ -600,30 +635,7 @@ class HttpRequestPipeline {
|
|
|
600
635
|
}
|
|
601
636
|
catch (error) {
|
|
602
637
|
const bodyErrorKind = isJsonBodyError(error) ? error.kind : null;
|
|
603
|
-
|
|
604
|
-
switch (bodyErrorKind) {
|
|
605
|
-
case 'payload-too-large':
|
|
606
|
-
sendError(ctx.res, -32600, 'Request body too large', 413, null);
|
|
607
|
-
break;
|
|
608
|
-
case 'read-failed':
|
|
609
|
-
if (!rawReq.destroyed) {
|
|
610
|
-
sendError(ctx.res, -32600, 'Request body read failed', 400, null);
|
|
611
|
-
}
|
|
612
|
-
break;
|
|
613
|
-
case 'invalid-json':
|
|
614
|
-
default:
|
|
615
|
-
sendError(ctx.res, -32700, 'Parse error', 400, null);
|
|
616
|
-
break;
|
|
617
|
-
}
|
|
618
|
-
}
|
|
619
|
-
else {
|
|
620
|
-
sendJson(ctx.res, bodyErrorKind === 'payload-too-large' ? 413 : 400, {
|
|
621
|
-
error: bodyErrorKind === 'payload-too-large'
|
|
622
|
-
? 'Payload too large'
|
|
623
|
-
: 'Invalid JSON',
|
|
624
|
-
});
|
|
625
|
-
}
|
|
626
|
-
drainRequest(rawReq);
|
|
638
|
+
sendBodyParseError(ctx, bodyErrorKind, rawReq);
|
|
627
639
|
return false;
|
|
628
640
|
}
|
|
629
641
|
}
|
package/dist/lib/core.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"core.d.ts","sourceRoot":"","sources":["../../src/lib/core.ts"],"names":[],"mappings":"AAWA,OAAO,EAAE,KAAK,SAAS,EAAE,MAAM,yCAAyC,CAAC;AASzE,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AACjD,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AASjD,eAAO,MAAM,aAAa,EAAE,MAA2C,CAAC;AACxE,KAAK,QAAQ,GAAG,OAAO,GAAG,MAAM,GAAG,MAAM,GAAG,OAAO,CAAC;AACpD,KAAK,WAAW,GACZ,OAAO,GACP,MAAM,GACN,QAAQ,GACR,SAAS,GACT,OAAO,GACP,UAAU,GACV,OAAO,GACP,WAAW,CAAC;AAwBhB,KAAK,mBAAmB,GAAG,SAAS,GAAG,SAAS,CAAC;AACjD,KAAK,QAAQ,GAAG,OAAO,GAAG,QAAQ,CAAC;
|
|
1
|
+
{"version":3,"file":"core.d.ts","sourceRoot":"","sources":["../../src/lib/core.ts"],"names":[],"mappings":"AAWA,OAAO,EAAE,KAAK,SAAS,EAAE,MAAM,yCAAyC,CAAC;AASzE,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AACjD,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AASjD,eAAO,MAAM,aAAa,EAAE,MAA2C,CAAC;AACxE,KAAK,QAAQ,GAAG,OAAO,GAAG,MAAM,GAAG,MAAM,GAAG,OAAO,CAAC;AACpD,KAAK,WAAW,GACZ,OAAO,GACP,MAAM,GACN,QAAQ,GACR,SAAS,GACT,OAAO,GACP,UAAU,GACV,OAAO,GACP,WAAW,CAAC;AAwBhB,KAAK,mBAAmB,GAAG,SAAS,GAAG,SAAS,CAAC;AACjD,KAAK,QAAQ,GAAG,OAAO,GAAG,QAAQ,CAAC;AAoNnC,UAAU,oBAAoB;IAC5B,sBAAsB,CAAC,EAAE,MAAM,CAAC;IAChC,wBAAwB,CAAC,EAAE,MAAM,CAAC;IAClC,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AA0BD,UAAU,UAAU;IAClB,IAAI,EAAE,QAAQ,CAAC;IACf,SAAS,EAAE,GAAG,GAAG,SAAS,CAAC;IAC3B,gBAAgB,EAAE,GAAG,GAAG,SAAS,CAAC;IAClC,QAAQ,EAAE,GAAG,GAAG,SAAS,CAAC;IAC1B,aAAa,EAAE,GAAG,GAAG,SAAS,CAAC;IAC/B,eAAe,EAAE,GAAG,GAAG,SAAS,CAAC;IACjC,gBAAgB,EAAE,GAAG,GAAG,SAAS,CAAC;IAClC,WAAW,EAAE,GAAG,CAAC;IACjB,cAAc,EAAE,MAAM,EAAE,CAAC;IACzB,QAAQ,EAAE,MAAM,GAAG,SAAS,CAAC;IAC7B,YAAY,EAAE,MAAM,GAAG,SAAS,CAAC;IACjC,sBAAsB,EAAE,MAAM,CAAC;IAC/B,YAAY,EAAE,MAAM,EAAE,CAAC;CACxB;AAED,UAAU,WAAW;IACnB,OAAO,EAAE,OAAO,CAAC;IACjB,OAAO,EAAE,MAAM,GAAG,SAAS,CAAC;IAC5B,QAAQ,EAAE,MAAM,GAAG,SAAS,CAAC;IAC7B,MAAM,EAAE,MAAM,GAAG,SAAS,CAAC;CAC5B;AA4FD,UAAU,YAAY;IACpB,QAAQ,EAAE,OAAO,CAAC;CACnB;AAKD,UAAU,mBAAmB;IAC3B,gBAAgB,EAAE,MAAM,GAAG,SAAS,CAAC;IACrC,gBAAgB,EAAE,MAAM,GAAG,SAAS,CAAC;IACrC,kBAAkB,EAAE,MAAM,GAAG,SAAS,CAAC;IACvC,wBAAwB,EAAE,MAAM,GAAG,SAAS,CAAC;IAC7C,eAAe,EAAE,MAAM,GAAG,SAAS,CAAC;IACpC,cAAc,EAAE,MAAM,CAAC;IACvB,uBAAuB,EAAE,OAAO,CAAC;IACjC,4BAA4B,EAAE,OAAO,CAAC;IACtC,2BAA2B,EAAE,OAAO,CAAC;CACtC;AAED,UAAU,eAAe;IACvB,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,MAAM,CAAC;IAChB,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,WAAW,CAAC;IACnB,YAAY,EAAE,MAAM,CAAC;IACrB,oBAAoB,EAAE,MAAM,CAAC;IAC7B,WAAW,EAAE,MAAM,CAAC;IACpB,IAAI,EAAE,mBAAmB,CAAC;CAC3B;AAuCD,UAAU,gBAAgB;IACxB,OAAO,EAAE,MAAM,CAAC;IAChB,YAAY,EAAE,MAAM,CAAC;IACrB,SAAS,EAAE,MAAM,CAAC;IAClB,gBAAgB,EAAE,MAAM,CAAC;CAC1B;AAWD,UAAU,kBAAkB;IAC1B,SAAS,EAAE,MAAM,CAAC;IAClB,cAAc,EAAE,MAAM,CAAC;IACvB,cAAc,EAAE,MAAM,CAAC;IACvB,cAAc,EAAE,MAAM,CAAC;IACvB,kBAAkB,EAAE,MAAM,CAAC;IAC3B,UAAU,EAAE,mBAAmB,CAAC;IAChC,oBAAoB,EAAE,oBAAoB,GAAG,SAAS,CAAC;CACxD;AAkBD,UAAU,cAAc;IACtB,QAAQ,EAAE,MAAM,CAAC;IACjB,WAAW,EAAE,MAAM,CAAC;IACpB,uBAAuB,EAAE,OAAO,CAAC;IACjC,mBAAmB,EAAE,OAAO,CAAC;CAC9B;AAmBD,UAAU,cAAc;IACtB,OAAO,EAAE,OAAO,CAAC;IACjB,GAAG,EAAE,MAAM,CAAC;IACZ,OAAO,EAAE,MAAM,CAAC;IAChB,YAAY,EAAE,MAAM,CAAC;CACtB;AAWD,UAAU,qBAAqB;IAC7B,WAAW,EAAE,MAAM,EAAE,CAAC;IACtB,cAAc,EAAE,MAAM,EAAE,CAAC;IACzB,iBAAiB,EAAE,MAAM,EAAE,CAAC;IAC5B,KAAK,EAAE,OAAO,CAAC;IACf,cAAc,EAAE,OAAO,CAAC;IACxB,iBAAiB,EAAE,OAAO,CAAC;IAC3B,OAAO,EAAE;QACP,MAAM,EAAE,MAAM,CAAC;QACf,UAAU,EAAE,MAAM,CAAC;QACnB,KAAK,EAAE,MAAM,CAAC;QACd,WAAW,EAAE,MAAM,CAAC;QACpB,SAAS,EAAE,MAAM,CAAC;KACnB,CAAC;CACH;AA2BD,UAAU,wBAAwB;IAChC,qBAAqB,EAAE,OAAO,CAAC;IAC/B,eAAe,EAAE,OAAO,CAAC;IACzB,eAAe,EAAE,OAAO,CAAC;IACzB,qBAAqB,EAAE,OAAO,CAAC;IAC/B,eAAe,EAAE,MAAM,EAAE,CAAC;CAC3B;AAeD,eAAO,MAAM,MAAM;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CA+ClB,CAAC;AACF,wBAAgB,cAAc,IAAI,IAAI,CAErC;AAED,UAAU,UAAU;IAClB,GAAG,EAAE,MAAM,CAAC;IACZ,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,MAAM,CAAC;IAChB,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;CACnB;AACD,UAAU,aAAa;IACrB,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;CACjB;AACD,UAAU,eAAe;IACvB,KAAK,CAAC,EAAE,OAAO,CAAC;CACjB;AACD,UAAU,eAAe;IACvB,KAAK,CAAC,EAAE,OAAO,CAAC;CACjB;AACD,UAAU,kBAAkB;IAC1B,GAAG,EAAE,MAAM,CAAC;IACZ,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAID,UAAU,gBAAgB;IACxB,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;IAChB,WAAW,EAAE,OAAO,CAAC;CACtB;AACD,KAAK,mBAAmB,GAAG,CAAC,KAAK,EAAE,gBAAgB,KAAK,OAAO,CAAC;AAChE,wBAAgB,cAAc,CAC5B,SAAS,EAAE,MAAM,EACjB,GAAG,EAAE,MAAM,EACX,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,MAAM,GACtC,MAAM,GAAG,IAAI,CAyBf;AACD,wBAAgB,aAAa,CAAC,QAAQ,EAAE,MAAM,GAAG,aAAa,GAAG,IAAI,CASpE;AA4LD,wBAAgB,aAAa,CAAC,QAAQ,EAAE,mBAAmB,GAAG,MAAM,IAAI,CAEvE;AACD,wBAAgB,GAAG,CACjB,QAAQ,EAAE,MAAM,GAAG,IAAI,EACvB,OAAO,CAAC,EAAE,eAAe,GACxB,UAAU,GAAG,SAAS,CAExB;AACD,wBAAgB,GAAG,CACjB,QAAQ,EAAE,MAAM,GAAG,IAAI,EACvB,OAAO,EAAE,MAAM,EACf,QAAQ,EAAE,kBAAkB,EAC5B,OAAO,CAAC,EAAE,eAAe,GACxB,IAAI,CAEN;AACD,wBAAgB,IAAI,IAAI,SAAS,MAAM,EAAE,CAExC;AACD,wBAAgB,YAAY,CAC1B,QAAQ,EAAE,MAAM,GACf;IAAE,GAAG,EAAE,MAAM,CAAC;IAAC,KAAK,CAAC,EAAE,MAAM,CAAC;IAAC,SAAS,CAAC,EAAE,MAAM,CAAA;CAAE,GAAG,SAAS,CAQjE;AACD,wBAAgB,SAAS,IAAI,OAAO,CAEnC;AA4BD,KAAK,WAAW,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;AAC3C,UAAU,cAAc;IACtB,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,CAAC;CAC/B;AAYD,wBAAgB,YAAY,CAAC,MAAM,EAAE,SAAS,GAAG,IAAI,CAKpD;AACD,wBAAgB,wBAAwB,CACtC,SAAS,EAAE,MAAM,EACjB,MAAM,EAAE,SAAS,GAChB,IAAI,CAGN;AACD,wBAAgB,0BAA0B,CAAC,SAAS,EAAE,MAAM,GAAG,IAAI,CAIlE;AACD,wBAAgB,kCAAkC,CAAC,MAAM,EAAE,SAAS,GAAG,IAAI,CAM1E;AACD,wBAAgB,2BAA2B,CACzC,MAAM,EAAE,SAAS,GAChB,MAAM,GAAG,SAAS,CAKpB;AACD,wBAAgB,qBAAqB,CAAC,CAAC,EACrC,OAAO,EAAE,cAAc,EACvB,EAAE,EAAE,MAAM,CAAC,GACV,CAAC,CAEH;AAID,wBAAgB,YAAY,IAAI,MAAM,GAAG,SAAS,CAGjD;AACD,wBAAgB,YAAY,IAAI,MAAM,GAAG,SAAS,CAEjD;AACD,wBAAgB,cAAc,IAAI,MAAM,GAAG,SAAS,CAEnD;AAiPD,wBAAgB,OAAO,CAAC,OAAO,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,WAAW,GAAG,IAAI,CAEjE;AACD,wBAAgB,QAAQ,CAAC,OAAO,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,WAAW,GAAG,IAAI,CAElE;AACD,wBAAgB,OAAO,CAAC,OAAO,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,WAAW,GAAG,IAAI,CAEjE;AAcD,wBAAgB,QAAQ,CAAC,OAAO,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,KAAK,GAAG,WAAW,GAAG,IAAI,CAI3E;AACD,wBAAgB,cAAc,CAAC,SAAS,CAAC,EAAE,MAAM,GAAG,WAAW,CAO9D;AACD,wBAAgB,WAAW,CAAC,KAAK,EAAE,MAAM,EAAE,SAAS,CAAC,EAAE,MAAM,GAAG,IAAI,CAUnE;AACD,wBAAgB,SAAS,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,CAQhD;AACD,YAAY,EAAE,YAAY,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAC/D,OAAO,EACL,oBAAoB,EACpB,kBAAkB,EAClB,iBAAiB,EACjB,qBAAqB,EACrB,kBAAkB,GACnB,MAAM,cAAc,CAAC;AAwHtB,wBAAgB,uBAAuB,CACrC,KAAK,EAAE,YAAY,EACnB,YAAY,EAAE,MAAM,EACpB,OAAO,CAAC,EAAE;IACR,cAAc,CAAC,EAAE,CAAC,OAAO,EAAE,YAAY,KAAK,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC;IACjE,iBAAiB,CAAC,EAAE,MAAM,CAAC;CAC5B,GACA,eAAe,CAOjB"}
|
package/dist/lib/core.js
CHANGED
|
@@ -60,28 +60,28 @@ const ENV_BOOLEAN_SCHEMA = z.stringbool({
|
|
|
60
60
|
falsy: ['false', '0', 'no', 'off'],
|
|
61
61
|
});
|
|
62
62
|
const EnvParser = {
|
|
63
|
-
integerValue(envValue,
|
|
63
|
+
integerValue(envValue, opts) {
|
|
64
64
|
if (!envValue)
|
|
65
65
|
return null;
|
|
66
66
|
const parsed = Number.parseInt(envValue, 10);
|
|
67
67
|
if (Number.isNaN(parsed)) {
|
|
68
|
-
if (envName)
|
|
69
|
-
process.stderr.write(`Warning: ignoring invalid ${envName} value "${envValue}" (not an integer)\n`);
|
|
68
|
+
if (opts?.envName)
|
|
69
|
+
process.stderr.write(`Warning: ignoring invalid ${opts.envName} value "${envValue}" (not an integer)\n`);
|
|
70
70
|
return null;
|
|
71
71
|
}
|
|
72
|
-
if ((min !== undefined && parsed < min) ||
|
|
73
|
-
(max !== undefined && parsed > max)) {
|
|
74
|
-
if (envName)
|
|
75
|
-
process.stderr.write(`Warning: ignoring out-of-range ${envName} value ${parsed} (allowed: ${min ?? '-∞'}..${max ?? '∞'})\n`);
|
|
72
|
+
if ((opts?.min !== undefined && parsed < opts.min) ||
|
|
73
|
+
(opts?.max !== undefined && parsed > opts.max)) {
|
|
74
|
+
if (opts.envName)
|
|
75
|
+
process.stderr.write(`Warning: ignoring out-of-range ${opts.envName} value ${parsed} (allowed: ${opts.min ?? '-∞'}..${opts.max ?? '∞'})\n`);
|
|
76
76
|
return null;
|
|
77
77
|
}
|
|
78
78
|
return parsed;
|
|
79
79
|
},
|
|
80
|
-
optionalInteger(envValue,
|
|
81
|
-
return EnvParser.integerValue(envValue,
|
|
80
|
+
optionalInteger(envValue, opts) {
|
|
81
|
+
return EnvParser.integerValue(envValue, opts) ?? undefined;
|
|
82
82
|
},
|
|
83
|
-
integer(envValue, defaultValue,
|
|
84
|
-
return EnvParser.integerValue(envValue,
|
|
83
|
+
integer(envValue, defaultValue, opts) {
|
|
84
|
+
return EnvParser.integerValue(envValue, opts) ?? defaultValue;
|
|
85
85
|
},
|
|
86
86
|
boolean(envValue, defaultValue, envName) {
|
|
87
87
|
const trimmed = envValue?.trim();
|
|
@@ -177,20 +177,22 @@ const EnvParser = {
|
|
|
177
177
|
return hostname;
|
|
178
178
|
},
|
|
179
179
|
};
|
|
180
|
+
const PRIMARY_HASH_LENGTH = 32;
|
|
181
|
+
const VARY_HASH_LENGTH = 16;
|
|
180
182
|
const MAX_HTML_BYTES = 10 * 1024 * 1024;
|
|
181
|
-
const MAX_INLINE_CONTENT_CHARS = EnvParser.integer(env['MAX_INLINE_CONTENT_CHARS'], 0, 0, MAX_HTML_BYTES, 'MAX_INLINE_CONTENT_CHARS');
|
|
183
|
+
const MAX_INLINE_CONTENT_CHARS = EnvParser.integer(env['MAX_INLINE_CONTENT_CHARS'], 0, { min: 0, max: MAX_HTML_BYTES, envName: 'MAX_INLINE_CONTENT_CHARS' });
|
|
182
184
|
const DEFAULT_SESSION_TTL_MS = 30 * 60 * 1000;
|
|
183
185
|
const DEFAULT_SESSION_INIT_TIMEOUT_MS = 10000;
|
|
184
186
|
const DEFAULT_MAX_SESSIONS = 200;
|
|
185
187
|
const DEFAULT_USER_AGENT = `fetch-url-mcp/${serverVersion}`;
|
|
186
188
|
const DEFAULT_TOOL_TIMEOUT_PADDING_MS = 5000;
|
|
187
189
|
const DEFAULT_TRANSFORM_TIMEOUT_MS = 30000;
|
|
188
|
-
const DEFAULT_FETCH_TIMEOUT_MS = EnvParser.integer(env['FETCH_TIMEOUT_MS'], 15000, 1000, 60000, 'FETCH_TIMEOUT_MS');
|
|
190
|
+
const DEFAULT_FETCH_TIMEOUT_MS = EnvParser.integer(env['FETCH_TIMEOUT_MS'], 15000, { min: 1000, max: 60000, envName: 'FETCH_TIMEOUT_MS' });
|
|
189
191
|
const DEFAULT_TOOL_TIMEOUT_MS = DEFAULT_FETCH_TIMEOUT_MS +
|
|
190
192
|
DEFAULT_TRANSFORM_TIMEOUT_MS +
|
|
191
193
|
DEFAULT_TOOL_TIMEOUT_PADDING_MS;
|
|
192
|
-
const DEFAULT_TASKS_MAX_TOTAL = EnvParser.integer(env['TASKS_MAX_TOTAL'], 5000, 1,
|
|
193
|
-
const DEFAULT_TASKS_MAX_PER_OWNER = EnvParser.integer(env['TASKS_MAX_PER_OWNER'], 1000, 1,
|
|
194
|
+
const DEFAULT_TASKS_MAX_TOTAL = EnvParser.integer(env['TASKS_MAX_TOTAL'], 5000, { min: 1, envName: 'TASKS_MAX_TOTAL' });
|
|
195
|
+
const DEFAULT_TASKS_MAX_PER_OWNER = EnvParser.integer(env['TASKS_MAX_PER_OWNER'], 1000, { min: 1, envName: 'TASKS_MAX_PER_OWNER' });
|
|
194
196
|
const RESOLVED_TASKS_MAX_PER_OWNER = Math.min(DEFAULT_TASKS_MAX_PER_OWNER, DEFAULT_TASKS_MAX_TOTAL);
|
|
195
197
|
function resolveWorkerResourceLimits() {
|
|
196
198
|
const limits = {};
|
|
@@ -202,7 +204,10 @@ function resolveWorkerResourceLimits() {
|
|
|
202
204
|
};
|
|
203
205
|
let hasLimits = false;
|
|
204
206
|
for (const [prop, envKey] of Object.entries(keys)) {
|
|
205
|
-
const val = EnvParser.optionalInteger(env[envKey],
|
|
207
|
+
const val = EnvParser.optionalInteger(env[envKey], {
|
|
208
|
+
min: 1,
|
|
209
|
+
envName: envKey,
|
|
210
|
+
});
|
|
206
211
|
if (val !== undefined) {
|
|
207
212
|
limits[prop] = val;
|
|
208
213
|
hasLimits = true;
|
|
@@ -274,7 +279,11 @@ const BLOCKED_HOSTS = new Set([
|
|
|
274
279
|
const host = (env['HOST'] ?? LOOPBACK_V4).trim();
|
|
275
280
|
const port = env['PORT']?.trim() === '0'
|
|
276
281
|
? 0
|
|
277
|
-
: EnvParser.integer(env['PORT'], 3000,
|
|
282
|
+
: EnvParser.integer(env['PORT'], 3000, {
|
|
283
|
+
min: 1024,
|
|
284
|
+
max: 65535,
|
|
285
|
+
envName: 'PORT',
|
|
286
|
+
});
|
|
278
287
|
const httpsConfig = buildHttpsConfig();
|
|
279
288
|
const allowRemote = EnvParser.boolean(env['ALLOW_REMOTE'], false, 'ALLOW_REMOTE');
|
|
280
289
|
const baseUrl = new URL(`${httpsConfig.enabled ? 'https' : 'http'}://${EnvParser.formatHostForUrl(host)}:${port}`);
|
|
@@ -282,7 +291,7 @@ const runtimeState = {
|
|
|
282
291
|
httpMode: false,
|
|
283
292
|
};
|
|
284
293
|
function buildServerConfig() {
|
|
285
|
-
const parseOptInt = (key, min = 1) => EnvParser.optionalInteger(env[key], min,
|
|
294
|
+
const parseOptInt = (key, min = 1) => EnvParser.optionalInteger(env[key], { min, envName: key });
|
|
286
295
|
return {
|
|
287
296
|
name: 'fetch-url-mcp',
|
|
288
297
|
version: serverVersion,
|
|
@@ -298,7 +307,10 @@ function buildServerConfig() {
|
|
|
298
307
|
keepAliveTimeoutMs: parseOptInt('SERVER_KEEP_ALIVE_TIMEOUT_MS'),
|
|
299
308
|
keepAliveTimeoutBufferMs: parseOptInt('SERVER_KEEP_ALIVE_TIMEOUT_BUFFER_MS', 0),
|
|
300
309
|
maxHeadersCount: parseOptInt('SERVER_MAX_HEADERS_COUNT'),
|
|
301
|
-
maxConnections: EnvParser.integer(env['SERVER_MAX_CONNECTIONS'], 0,
|
|
310
|
+
maxConnections: EnvParser.integer(env['SERVER_MAX_CONNECTIONS'], 0, {
|
|
311
|
+
min: 0,
|
|
312
|
+
envName: 'SERVER_MAX_CONNECTIONS',
|
|
313
|
+
}),
|
|
302
314
|
blockPrivateConnections: EnvParser.boolean(env['SERVER_BLOCK_PRIVATE_CONNECTIONS'], false, 'SERVER_BLOCK_PRIVATE_CONNECTIONS'),
|
|
303
315
|
shutdownCloseIdleConnections: true,
|
|
304
316
|
shutdownCloseAllConnections: false,
|
|
@@ -319,7 +331,7 @@ function buildTransformConfig() {
|
|
|
319
331
|
stageWarnRatio: 0.5,
|
|
320
332
|
metadataFormat: 'markdown',
|
|
321
333
|
maxWorkerScale: 4,
|
|
322
|
-
cancelAckTimeoutMs: EnvParser.integer(env['TRANSFORM_CANCEL_ACK_TIMEOUT_MS'], 200, 50, 5000, 'TRANSFORM_CANCEL_ACK_TIMEOUT_MS'),
|
|
334
|
+
cancelAckTimeoutMs: EnvParser.integer(env['TRANSFORM_CANCEL_ACK_TIMEOUT_MS'], 200, { min: 50, max: 5000, envName: 'TRANSFORM_CANCEL_ACK_TIMEOUT_MS' }),
|
|
323
335
|
workerMode: EnvParser.transformWorkerMode(env['TRANSFORM_WORKER_MODE']),
|
|
324
336
|
workerResourceLimits: resolveWorkerResourceLimits(),
|
|
325
337
|
};
|
|
@@ -423,7 +435,7 @@ export function enableHttpMode() {
|
|
|
423
435
|
export function createCacheKey(namespace, url, vary) {
|
|
424
436
|
if (!namespace || !url)
|
|
425
437
|
return null;
|
|
426
|
-
const urlHash = sha256Hex(url).substring(0,
|
|
438
|
+
const urlHash = sha256Hex(url).substring(0, PRIMARY_HASH_LENGTH);
|
|
427
439
|
if (!vary)
|
|
428
440
|
return `${namespace}:${urlHash}`;
|
|
429
441
|
const varyString = typeof vary === 'string'
|
|
@@ -439,7 +451,7 @@ export function createCacheKey(namespace, url, vary) {
|
|
|
439
451
|
if (varyString === null)
|
|
440
452
|
return null;
|
|
441
453
|
const varyHash = varyString
|
|
442
|
-
? sha256Hex(varyString).substring(0,
|
|
454
|
+
? sha256Hex(varyString).substring(0, VARY_HASH_LENGTH)
|
|
443
455
|
: undefined;
|
|
444
456
|
return varyHash
|
|
445
457
|
? `${namespace}:${urlHash}.${varyHash}`
|
|
@@ -532,25 +544,16 @@ class InMemoryCacheStore {
|
|
|
532
544
|
const firstKey = this.entries.keys().next();
|
|
533
545
|
return !firstKey.done && this.delete(firstKey.value);
|
|
534
546
|
}
|
|
535
|
-
|
|
536
|
-
if (!cacheKey || !content)
|
|
537
|
-
return;
|
|
538
|
-
if (!this.isEnabled() && !options?.force)
|
|
539
|
-
return;
|
|
540
|
-
const now = Date.now();
|
|
541
|
-
const expiresAtMs = now + this.ttlMs;
|
|
542
|
-
// Check size limit before insertion
|
|
543
|
-
const entrySize = content.length;
|
|
547
|
+
ensureCapacity(cacheKey, entrySize) {
|
|
544
548
|
if (entrySize > this.maxBytes) {
|
|
545
549
|
logWarn('Cache entry exceeds max size', {
|
|
546
550
|
key: cacheKey,
|
|
547
551
|
size: entrySize,
|
|
548
552
|
max: this.maxBytes,
|
|
549
553
|
});
|
|
550
|
-
return;
|
|
554
|
+
return { ok: false, listChanged: false };
|
|
551
555
|
}
|
|
552
|
-
let listChanged =
|
|
553
|
-
// Evict if needed (size-based)
|
|
556
|
+
let listChanged = false;
|
|
554
557
|
while (this.currentBytes + entrySize > this.maxBytes) {
|
|
555
558
|
if (this.evictOldestEntry()) {
|
|
556
559
|
listChanged = true;
|
|
@@ -559,6 +562,20 @@ class InMemoryCacheStore {
|
|
|
559
562
|
break;
|
|
560
563
|
}
|
|
561
564
|
}
|
|
565
|
+
return { ok: true, listChanged };
|
|
566
|
+
}
|
|
567
|
+
set(cacheKey, content, metadata, options) {
|
|
568
|
+
if (!cacheKey || !content)
|
|
569
|
+
return;
|
|
570
|
+
if (!this.isEnabled() && !options?.force)
|
|
571
|
+
return;
|
|
572
|
+
const now = Date.now();
|
|
573
|
+
const expiresAtMs = now + this.ttlMs;
|
|
574
|
+
const entrySize = content.length;
|
|
575
|
+
const capacity = this.ensureCapacity(cacheKey, entrySize);
|
|
576
|
+
if (!capacity.ok)
|
|
577
|
+
return;
|
|
578
|
+
let listChanged = !this.entries.has(cacheKey) || capacity.listChanged;
|
|
562
579
|
const entry = {
|
|
563
580
|
url: metadata.url,
|
|
564
581
|
content,
|
|
@@ -894,6 +911,9 @@ function writeLog(level, message, meta) {
|
|
|
894
911
|
const line = formatLogEntry(level, message, meta);
|
|
895
912
|
safeWriteStderr(`${stripVTControlCharacters(line)}\n`);
|
|
896
913
|
}
|
|
914
|
+
forwardMcpLog(level, message, meta, sessionId);
|
|
915
|
+
}
|
|
916
|
+
function forwardMcpLog(level, message, meta, sessionId) {
|
|
897
917
|
const server = sessionId ? sessionServers.get(sessionId) : mcpServer;
|
|
898
918
|
if (!server)
|
|
899
919
|
return;
|
package/dist/lib/dom-prep.d.ts
CHANGED
|
@@ -1,7 +1,11 @@
|
|
|
1
|
+
import type { ExtractedArticle } from '../transform/types.js';
|
|
2
|
+
export declare function extractNoscriptImages(document: Document): void;
|
|
1
3
|
export declare function resolveDocumentBody(document: Document): Element;
|
|
2
4
|
export declare function serializeDocumentForMarkdown(document: Document, fallback: string): string;
|
|
3
5
|
/** Surface hidden tab panels, then strip unselected tab triggers. */
|
|
4
6
|
export declare function normalizeTabContent(document: Document): void;
|
|
5
7
|
export declare function prepareDocumentForMarkdown(document: Document, baseUrl?: string, signal?: AbortSignal): void;
|
|
6
8
|
export declare function removeNoiseFromHtml(html: string, document?: Document, baseUrl?: string, signal?: AbortSignal): string;
|
|
9
|
+
export declare function getVisibleTextLength(htmlOrDocument: string | Document): number;
|
|
10
|
+
export declare function evaluateArticleContent(article: ExtractedArticle, document: Document): Document | null;
|
|
7
11
|
//# sourceMappingURL=dom-prep.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"dom-prep.d.ts","sourceRoot":"","sources":["../../src/lib/dom-prep.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"dom-prep.d.ts","sourceRoot":"","sources":["../../src/lib/dom-prep.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;AAypB9D,wBAAgB,qBAAqB,CAAC,QAAQ,EAAE,QAAQ,GAAG,IAAI,CA6B9D;AAuBD,wBAAgB,mBAAmB,CAAC,QAAQ,EAAE,QAAQ,GAAG,OAAO,CAY/D;AAED,wBAAgB,4BAA4B,CAC1C,QAAQ,EAAE,QAAQ,EAClB,QAAQ,EAAE,MAAM,GACf,MAAM,CASR;AA0CD,qEAAqE;AACrE,wBAAgB,mBAAmB,CAAC,QAAQ,EAAE,QAAQ,GAAG,IAAI,CAG5D;AAyTD,wBAAgB,0BAA0B,CACxC,QAAQ,EAAE,QAAQ,EAClB,OAAO,CAAC,EAAE,MAAM,EAChB,MAAM,CAAC,EAAE,WAAW,GACnB,IAAI,CAON;AA4BD,wBAAgB,mBAAmB,CACjC,IAAI,EAAE,MAAM,EACZ,QAAQ,CAAC,EAAE,QAAQ,EACnB,OAAO,CAAC,EAAE,MAAM,EAChB,MAAM,CAAC,EAAE,WAAW,GACnB,MAAM,CAcR;AA0ED,wBAAgB,oBAAoB,CAClC,cAAc,EAAE,MAAM,GAAG,QAAQ,GAChC,MAAM,CAaR;AA6ID,wBAAgB,sBAAsB,CACpC,OAAO,EAAE,gBAAgB,EACzB,QAAQ,EAAE,QAAQ,GACjB,QAAQ,GAAG,IAAI,CAsBjB"}
|