@j0hanz/fetch-url-mcp 1.10.24 → 1.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/http/native.d.ts.map +1 -1
- package/dist/http/native.js +7 -4
- package/dist/lib/core.d.ts.map +1 -1
- package/dist/lib/core.js +11 -1
- package/dist/lib/dom-prep.d.ts +1 -0
- package/dist/lib/dom-prep.d.ts.map +1 -1
- package/dist/lib/dom-prep.js +102 -58
- package/dist/lib/fetch-pipeline.d.ts.map +1 -1
- package/dist/lib/fetch-pipeline.js +5 -2
- package/dist/schemas.js +1 -1
- package/dist/transform/transform.d.ts.map +1 -1
- package/dist/transform/transform.js +2 -1
- package/package.json +2 -3
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"native.d.ts","sourceRoot":"","sources":["../../src/http/native.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"native.d.ts","sourceRoot":"","sources":["../../src/http/native.ts"],"names":[],"mappings":"AAooCA,wBAAsB,eAAe,IAAI,OAAO,CAAC;IAC/C,QAAQ,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;IAC5C,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;CACd,CAAC,CA0DD"}
|
package/dist/http/native.js
CHANGED
|
@@ -51,6 +51,9 @@ function isInitializedNotification(method) {
|
|
|
51
51
|
function isPingRequest(method) {
|
|
52
52
|
return method === 'ping';
|
|
53
53
|
}
|
|
54
|
+
function isMcpRoute(pathname) {
|
|
55
|
+
return pathname === '/mcp' || pathname === '/mcp/';
|
|
56
|
+
}
|
|
54
57
|
function createSessionTeardownOptions(mode, context) {
|
|
55
58
|
switch (mode) {
|
|
56
59
|
case 'ended':
|
|
@@ -136,8 +139,8 @@ class McpSessionGateway {
|
|
|
136
139
|
}
|
|
137
140
|
validatePostRequest(ctx) {
|
|
138
141
|
if (!acceptsJsonAndEventStream(getHeaderValue(ctx.req, 'accept'))) {
|
|
139
|
-
sendJson(ctx.res,
|
|
140
|
-
error: '
|
|
142
|
+
sendJson(ctx.res, 406, {
|
|
143
|
+
error: 'Not Acceptable: expected application/json and text/event-stream',
|
|
141
144
|
});
|
|
142
145
|
return null;
|
|
143
146
|
}
|
|
@@ -494,7 +497,7 @@ class HttpDispatcher {
|
|
|
494
497
|
const authCtx = { ...ctx, auth };
|
|
495
498
|
if (this.tryHandleDownloadRoute(authCtx))
|
|
496
499
|
return;
|
|
497
|
-
if (ctx.url.pathname
|
|
500
|
+
if (isMcpRoute(ctx.url.pathname)) {
|
|
498
501
|
const handled = await this.handleMcpRoutes(authCtx);
|
|
499
502
|
if (handled)
|
|
500
503
|
return;
|
|
@@ -568,7 +571,7 @@ const BODY_PARSE_ERRORS = {
|
|
|
568
571
|
function sendBodyParseError(ctx, bodyErrorKind, rawReq) {
|
|
569
572
|
const errorDef = BODY_PARSE_ERRORS[bodyErrorKind ?? 'default'] ?? DEFAULT_BODY_ERROR;
|
|
570
573
|
if (bodyErrorKind !== 'read-failed' || !rawReq.destroyed) {
|
|
571
|
-
if (ctx.url.pathname
|
|
574
|
+
if (isMcpRoute(ctx.url.pathname)) {
|
|
572
575
|
sendError(ctx.res, errorDef.mcpCode, errorDef.mcpMsg, errorDef.statusCode, null);
|
|
573
576
|
}
|
|
574
577
|
else {
|
package/dist/lib/core.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"core.d.ts","sourceRoot":"","sources":["../../src/lib/core.ts"],"names":[],"mappings":"AAWA,OAAO,EAAE,KAAK,SAAS,EAAE,MAAM,yCAAyC,CAAC;AAGzE,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AACjD,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAejD,eAAO,MAAM,aAAa,EAAE,MAA2C,CAAC;AACxE,KAAK,QAAQ,GAAG,OAAO,GAAG,MAAM,GAAG,MAAM,GAAG,OAAO,CAAC;AACpD,KAAK,WAAW,GACZ,OAAO,GACP,MAAM,GACN,QAAQ,GACR,SAAS,GACT,OAAO,GACP,UAAU,GACV,OAAO,GACP,WAAW,CAAC;AAwBhB,KAAK,mBAAmB,GAAG,SAAS,GAAG,SAAS,CAAC;AACjD,KAAK,QAAQ,GAAG,OAAO,GAAG,QAAQ,CAAC;AAgOnC,UAAU,oBAAoB;IAC5B,sBAAsB,CAAC,EAAE,MAAM,CAAC;IAChC,wBAAwB,CAAC,EAAE,MAAM,CAAC;IAClC,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AA0BD,UAAU,UAAU;IAClB,IAAI,EAAE,QAAQ,CAAC;IACf,SAAS,EAAE,GAAG,GAAG,SAAS,CAAC;IAC3B,gBAAgB,EAAE,GAAG,GAAG,SAAS,CAAC;IAClC,QAAQ,EAAE,GAAG,GAAG,SAAS,CAAC;IAC1B,aAAa,EAAE,GAAG,GAAG,SAAS,CAAC;IAC/B,eAAe,EAAE,GAAG,GAAG,SAAS,CAAC;IACjC,gBAAgB,EAAE,GAAG,GAAG,SAAS,CAAC;IAClC,WAAW,EAAE,GAAG,CAAC;IACjB,cAAc,EAAE,MAAM,EAAE,CAAC;IACzB,QAAQ,EAAE,MAAM,GAAG,SAAS,CAAC;IAC7B,YAAY,EAAE,MAAM,GAAG,SAAS,CAAC;IACjC,sBAAsB,EAAE,MAAM,CAAC;IAC/B,YAAY,EAAE,MAAM,EAAE,CAAC;CACxB;AAED,UAAU,WAAW;IACnB,OAAO,EAAE,OAAO,CAAC;IACjB,OAAO,EAAE,MAAM,GAAG,SAAS,CAAC;IAC5B,QAAQ,EAAE,MAAM,GAAG,SAAS,CAAC;IAC7B,MAAM,EAAE,MAAM,GAAG,SAAS,CAAC;CAC5B;AA4FD,UAAU,YAAY;IACpB,QAAQ,EAAE,OAAO,CAAC;CACnB;AAKD,UAAU,mBAAmB;IAC3B,gBAAgB,EAAE,MAAM,GAAG,SAAS,CAAC;IACrC,gBAAgB,EAAE,MAAM,GAAG,SAAS,CAAC;IACrC,kBAAkB,EAAE,MAAM,GAAG,SAAS,CAAC;IACvC,wBAAwB,EAAE,MAAM,GAAG,SAAS,CAAC;IAC7C,eAAe,EAAE,MAAM,GAAG,SAAS,CAAC;IACpC,cAAc,EAAE,MAAM,CAAC;IACvB,uBAAuB,EAAE,OAAO,CAAC;IACjC,4BAA4B,EAAE,OAAO,CAAC;IACtC,2BAA2B,EAAE,OAAO,CAAC;CACtC;AAED,UAAU,eAAe;IACvB,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,MAAM,CAAC;IAChB,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,WAAW,CAAC;IACnB,YAAY,EAAE,MAAM,CAAC;IACrB,oBAAoB,EAAE,MAAM,CAAC;IAC7B,WAAW,EAAE,MAAM,CAAC;IACpB,IAAI,EAAE,mBAAmB,CAAC;CAC3B;AAuCD,UAAU,gBAAgB;IACxB,OAAO,EAAE,MAAM,CAAC;IAChB,YAAY,EAAE,MAAM,CAAC;IACrB,SAAS,EAAE,MAAM,CAAC;IAClB,gBAAgB,EAAE,MAAM,CAAC;CAC1B;AAWD,UAAU,kBAAkB;IAC1B,SAAS,EAAE,MAAM,CAAC;IAClB,cAAc,EAAE,MAAM,CAAC;IACvB,cAAc,EAAE,MAAM,CAAC;IACvB,cAAc,EAAE,MAAM,CAAC;IACvB,kBAAkB,EAAE,MAAM,CAAC;IAC3B,UAAU,EAAE,mBAAmB,CAAC;IAChC,oBAAoB,EAAE,oBAAoB,GAAG,SAAS,CAAC;CACxD;AAkBD,UAAU,cAAc;IACtB,QAAQ,EAAE,MAAM,CAAC;IACjB,WAAW,EAAE,MAAM,CAAC;IACpB,uBAAuB,EAAE,OAAO,CAAC;IACjC,mBAAmB,EAAE,OAAO,CAAC;CAC9B;AAmBD,UAAU,cAAc;IACtB,OAAO,EAAE,OAAO,CAAC;IACjB,GAAG,EAAE,MAAM,CAAC;IACZ,OAAO,EAAE,MAAM,CAAC;IAChB,YAAY,EAAE,MAAM,CAAC;CACtB;AAWD,UAAU,qBAAqB;IAC7B,WAAW,EAAE,MAAM,EAAE,CAAC;IACtB,cAAc,EAAE,MAAM,EAAE,CAAC;IACzB,iBAAiB,EAAE,MAAM,EAAE,CAAC;IAC5B,KAAK,EAAE,OAAO,CAAC;IACf,cAAc,EAAE,OAAO,CAAC;IACxB,iBAAiB,EAAE,OAAO,CAAC;IAC3B,OAAO,EAAE;QACP,MAAM,EAAE,MAAM,CAAC;QACf,UAAU,EAAE,MAAM,CAAC;QACnB,KAAK,EAAE,MAAM,CAAC;QACd,WAAW,EAAE,MAAM,CAAC;QACpB,SAAS,EAAE,MAAM,CAAC;KACnB,CAAC;CACH;AA2BD,UAAU,wBAAwB;IAChC,qBAAqB,EAAE,OAAO,CAAC;IAC/B,eAAe,EAAE,OAAO,CAAC;IACzB,eAAe,EAAE,OAAO,CAAC;IACzB,qBAAqB,EAAE,OAAO,CAAC;IAC/B,eAAe,EAAE,MAAM,EAAE,CAAC;CAC3B;AAeD,eAAO,MAAM,MAAM;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CA+ClB,CAAC;AACF,wBAAgB,cAAc,IAAI,IAAI,CAErC;AAED,UAAU,UAAU;IAClB,GAAG,EAAE,MAAM,CAAC;IACZ,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,MAAM,CAAC;IAChB,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;CACnB;AACD,UAAU,aAAa;IACrB,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;CACjB;AACD,UAAU,eAAe;IACvB,KAAK,CAAC,EAAE,OAAO,CAAC;CACjB;AACD,UAAU,eAAe;IACvB,KAAK,CAAC,EAAE,OAAO,CAAC;CACjB;AACD,UAAU,kBAAkB;IAC1B,GAAG,EAAE,MAAM,CAAC;IACZ,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAID,UAAU,gBAAgB;IACxB,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;IAChB,WAAW,EAAE,OAAO,CAAC;CACtB;AACD,KAAK,mBAAmB,GAAG,CAAC,KAAK,EAAE,gBAAgB,KAAK,OAAO,CAAC;AAChE,wBAAgB,cAAc,CAC5B,SAAS,EAAE,MAAM,EACjB,GAAG,EAAE,MAAM,EACX,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,MAAM,GACtC,MAAM,GAAG,IAAI,CAyBf;AACD,wBAAgB,aAAa,CAAC,QAAQ,EAAE,MAAM,GAAG,aAAa,GAAG,IAAI,CASpE;
|
|
1
|
+
{"version":3,"file":"core.d.ts","sourceRoot":"","sources":["../../src/lib/core.ts"],"names":[],"mappings":"AAWA,OAAO,EAAE,KAAK,SAAS,EAAE,MAAM,yCAAyC,CAAC;AAGzE,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AACjD,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAejD,eAAO,MAAM,aAAa,EAAE,MAA2C,CAAC;AACxE,KAAK,QAAQ,GAAG,OAAO,GAAG,MAAM,GAAG,MAAM,GAAG,OAAO,CAAC;AACpD,KAAK,WAAW,GACZ,OAAO,GACP,MAAM,GACN,QAAQ,GACR,SAAS,GACT,OAAO,GACP,UAAU,GACV,OAAO,GACP,WAAW,CAAC;AAwBhB,KAAK,mBAAmB,GAAG,SAAS,GAAG,SAAS,CAAC;AACjD,KAAK,QAAQ,GAAG,OAAO,GAAG,QAAQ,CAAC;AAgOnC,UAAU,oBAAoB;IAC5B,sBAAsB,CAAC,EAAE,MAAM,CAAC;IAChC,wBAAwB,CAAC,EAAE,MAAM,CAAC;IAClC,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AA0BD,UAAU,UAAU;IAClB,IAAI,EAAE,QAAQ,CAAC;IACf,SAAS,EAAE,GAAG,GAAG,SAAS,CAAC;IAC3B,gBAAgB,EAAE,GAAG,GAAG,SAAS,CAAC;IAClC,QAAQ,EAAE,GAAG,GAAG,SAAS,CAAC;IAC1B,aAAa,EAAE,GAAG,GAAG,SAAS,CAAC;IAC/B,eAAe,EAAE,GAAG,GAAG,SAAS,CAAC;IACjC,gBAAgB,EAAE,GAAG,GAAG,SAAS,CAAC;IAClC,WAAW,EAAE,GAAG,CAAC;IACjB,cAAc,EAAE,MAAM,EAAE,CAAC;IACzB,QAAQ,EAAE,MAAM,GAAG,SAAS,CAAC;IAC7B,YAAY,EAAE,MAAM,GAAG,SAAS,CAAC;IACjC,sBAAsB,EAAE,MAAM,CAAC;IAC/B,YAAY,EAAE,MAAM,EAAE,CAAC;CACxB;AAED,UAAU,WAAW;IACnB,OAAO,EAAE,OAAO,CAAC;IACjB,OAAO,EAAE,MAAM,GAAG,SAAS,CAAC;IAC5B,QAAQ,EAAE,MAAM,GAAG,SAAS,CAAC;IAC7B,MAAM,EAAE,MAAM,GAAG,SAAS,CAAC;CAC5B;AA4FD,UAAU,YAAY;IACpB,QAAQ,EAAE,OAAO,CAAC;CACnB;AAKD,UAAU,mBAAmB;IAC3B,gBAAgB,EAAE,MAAM,GAAG,SAAS,CAAC;IACrC,gBAAgB,EAAE,MAAM,GAAG,SAAS,CAAC;IACrC,kBAAkB,EAAE,MAAM,GAAG,SAAS,CAAC;IACvC,wBAAwB,EAAE,MAAM,GAAG,SAAS,CAAC;IAC7C,eAAe,EAAE,MAAM,GAAG,SAAS,CAAC;IACpC,cAAc,EAAE,MAAM,CAAC;IACvB,uBAAuB,EAAE,OAAO,CAAC;IACjC,4BAA4B,EAAE,OAAO,CAAC;IACtC,2BAA2B,EAAE,OAAO,CAAC;CACtC;AAED,UAAU,eAAe;IACvB,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,MAAM,CAAC;IAChB,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,WAAW,CAAC;IACnB,YAAY,EAAE,MAAM,CAAC;IACrB,oBAAoB,EAAE,MAAM,CAAC;IAC7B,WAAW,EAAE,MAAM,CAAC;IACpB,IAAI,EAAE,mBAAmB,CAAC;CAC3B;AAuCD,UAAU,gBAAgB;IACxB,OAAO,EAAE,MAAM,CAAC;IAChB,YAAY,EAAE,MAAM,CAAC;IACrB,SAAS,EAAE,MAAM,CAAC;IAClB,gBAAgB,EAAE,MAAM,CAAC;CAC1B;AAWD,UAAU,kBAAkB;IAC1B,SAAS,EAAE,MAAM,CAAC;IAClB,cAAc,EAAE,MAAM,CAAC;IACvB,cAAc,EAAE,MAAM,CAAC;IACvB,cAAc,EAAE,MAAM,CAAC;IACvB,kBAAkB,EAAE,MAAM,CAAC;IAC3B,UAAU,EAAE,mBAAmB,CAAC;IAChC,oBAAoB,EAAE,oBAAoB,GAAG,SAAS,CAAC;CACxD;AAkBD,UAAU,cAAc;IACtB,QAAQ,EAAE,MAAM,CAAC;IACjB,WAAW,EAAE,MAAM,CAAC;IACpB,uBAAuB,EAAE,OAAO,CAAC;IACjC,mBAAmB,EAAE,OAAO,CAAC;CAC9B;AAmBD,UAAU,cAAc;IACtB,OAAO,EAAE,OAAO,CAAC;IACjB,GAAG,EAAE,MAAM,CAAC;IACZ,OAAO,EAAE,MAAM,CAAC;IAChB,YAAY,EAAE,MAAM,CAAC;CACtB;AAWD,UAAU,qBAAqB;IAC7B,WAAW,EAAE,MAAM,EAAE,CAAC;IACtB,cAAc,EAAE,MAAM,EAAE,CAAC;IACzB,iBAAiB,EAAE,MAAM,EAAE,CAAC;IAC5B,KAAK,EAAE,OAAO,CAAC;IACf,cAAc,EAAE,OAAO,CAAC;IACxB,iBAAiB,EAAE,OAAO,CAAC;IAC3B,OAAO,EAAE;QACP,MAAM,EAAE,MAAM,CAAC;QACf,UAAU,EAAE,MAAM,CAAC;QACnB,KAAK,EAAE,MAAM,CAAC;QACd,WAAW,EAAE,MAAM,CAAC;QACpB,SAAS,EAAE,MAAM,CAAC;KACnB,CAAC;CACH;AA2BD,UAAU,wBAAwB;IAChC,qBAAqB,EAAE,OAAO,CAAC;IAC/B,eAAe,EAAE,OAAO,CAAC;IACzB,eAAe,EAAE,OAAO,CAAC;IACzB,qBAAqB,EAAE,OAAO,CAAC;IAC/B,eAAe,EAAE,MAAM,EAAE,CAAC;CAC3B;AAeD,eAAO,MAAM,MAAM;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CA+ClB,CAAC;AACF,wBAAgB,cAAc,IAAI,IAAI,CAErC;AAED,UAAU,UAAU;IAClB,GAAG,EAAE,MAAM,CAAC;IACZ,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,MAAM,CAAC;IAChB,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;CACnB;AACD,UAAU,aAAa;IACrB,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;CACjB;AACD,UAAU,eAAe;IACvB,KAAK,CAAC,EAAE,OAAO,CAAC;CACjB;AACD,UAAU,eAAe;IACvB,KAAK,CAAC,EAAE,OAAO,CAAC;CACjB;AACD,UAAU,kBAAkB;IAC1B,GAAG,EAAE,MAAM,CAAC;IACZ,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAID,UAAU,gBAAgB;IACxB,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;IAChB,WAAW,EAAE,OAAO,CAAC;CACtB;AACD,KAAK,mBAAmB,GAAG,CAAC,KAAK,EAAE,gBAAgB,KAAK,OAAO,CAAC;AAChE,wBAAgB,cAAc,CAC5B,SAAS,EAAE,MAAM,EACjB,GAAG,EAAE,MAAM,EACX,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,MAAM,GACtC,MAAM,GAAG,IAAI,CAyBf;AACD,wBAAgB,aAAa,CAAC,QAAQ,EAAE,MAAM,GAAG,aAAa,GAAG,IAAI,CASpE;AAuMD,wBAAgB,aAAa,CAAC,QAAQ,EAAE,mBAAmB,GAAG,MAAM,IAAI,CAEvE;AACD,wBAAgB,GAAG,CACjB,QAAQ,EAAE,MAAM,GAAG,IAAI,EACvB,OAAO,CAAC,EAAE,eAAe,GACxB,UAAU,GAAG,SAAS,CAExB;AACD,wBAAgB,GAAG,CACjB,QAAQ,EAAE,MAAM,GAAG,IAAI,EACvB,OAAO,EAAE,MAAM,EACf,QAAQ,EAAE,kBAAkB,EAC5B,OAAO,CAAC,EAAE,eAAe,GACxB,IAAI,CAEN;AACD,wBAAgB,IAAI,IAAI,SAAS,MAAM,EAAE,CAExC;AACD,wBAAgB,YAAY,CAC1B,QAAQ,EAAE,MAAM,GACf;IAAE,GAAG,EAAE,MAAM,CAAC;IAAC,KAAK,CAAC,EAAE,MAAM,CAAC;IAAC,SAAS,CAAC,EAAE,MAAM,CAAA;CAAE,GAAG,SAAS,CAQjE;AACD,wBAAgB,SAAS,IAAI,OAAO,CAEnC;AA4BD,KAAK,WAAW,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;AAC3C,UAAU,cAAc;IACtB,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,CAAC;CAC/B;AAYD,wBAAgB,YAAY,CAAC,MAAM,EAAE,SAAS,GAAG,IAAI,CAKpD;AACD,wBAAgB,wBAAwB,CACtC,SAAS,EAAE,MAAM,EACjB,MAAM,EAAE,SAAS,GAChB,IAAI,CAGN;AACD,wBAAgB,0BAA0B,CAAC,SAAS,EAAE,MAAM,GAAG,IAAI,CAIlE;AACD,wBAAgB,kCAAkC,CAAC,MAAM,EAAE,SAAS,GAAG,IAAI,CAM1E;AACD,wBAAgB,2BAA2B,CACzC,MAAM,EAAE,SAAS,GAChB,MAAM,GAAG,SAAS,CAKpB;AACD,wBAAgB,qBAAqB,CAAC,CAAC,EACrC,OAAO,EAAE,cAAc,EACvB,EAAE,EAAE,MAAM,CAAC,GACV,CAAC,CAEH;AAID,wBAAgB,YAAY,IAAI,MAAM,GAAG,SAAS,CAGjD;AACD,wBAAgB,YAAY,IAAI,MAAM,GAAG,SAAS,CAEjD;AACD,wBAAgB,cAAc,IAAI,MAAM,GAAG,SAAS,CAEnD;AAwPD,wBAAgB,OAAO,CAAC,OAAO,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,WAAW,GAAG,IAAI,CAEjE;AACD,wBAAgB,QAAQ,CAAC,OAAO,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,WAAW,GAAG,IAAI,CAElE;AACD,wBAAgB,OAAO,CAAC,OAAO,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,WAAW,GAAG,IAAI,CAEjE;AAcD,wBAAgB,QAAQ,CAAC,OAAO,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,KAAK,GAAG,WAAW,GAAG,IAAI,CAI3E;AACD,wBAAgB,cAAc,CAAC,SAAS,CAAC,EAAE,MAAM,GAAG,WAAW,CAO9D;AACD,wBAAgB,WAAW,CAAC,KAAK,EAAE,MAAM,EAAE,SAAS,CAAC,EAAE,MAAM,GAAG,IAAI,CAUnE;AACD,wBAAgB,SAAS,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,CAQhD;AACD,YAAY,EAAE,YAAY,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAC/D,OAAO,EACL,oBAAoB,EACpB,kBAAkB,EAClB,iBAAiB,EACjB,qBAAqB,EACrB,kBAAkB,GACnB,MAAM,cAAc,CAAC;AAmJtB,wBAAgB,uBAAuB,CACrC,KAAK,EAAE,YAAY,EACnB,YAAY,EAAE,MAAM,EACpB,OAAO,CAAC,EAAE;IACR,cAAc,CAAC,EAAE,CAAC,OAAO,EAAE,YAAY,KAAK,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC;IACjE,iBAAiB,CAAC,EAAE,MAAM,CAAC;CAC5B,GACA,eAAe,CAOjB"}
|
package/dist/lib/core.js
CHANGED
|
@@ -584,6 +584,15 @@ class InMemoryCacheStore {
|
|
|
584
584
|
const now = Date.now();
|
|
585
585
|
const expiresAtMs = now + this.ttlMs;
|
|
586
586
|
const entrySize = content.length;
|
|
587
|
+
// Reject oversized entries before deleting the old one to avoid data loss
|
|
588
|
+
if (entrySize > this.maxBytes) {
|
|
589
|
+
logWarn('Cache entry exceeds max size', {
|
|
590
|
+
key: cacheKey,
|
|
591
|
+
size: entrySize,
|
|
592
|
+
max: this.maxBytes,
|
|
593
|
+
});
|
|
594
|
+
return;
|
|
595
|
+
}
|
|
587
596
|
const isUpdate = this.entries.has(cacheKey);
|
|
588
597
|
if (isUpdate) {
|
|
589
598
|
this.delete(cacheKey);
|
|
@@ -1016,6 +1025,7 @@ export { composeCloseHandlers, createSessionStore, createSlotTracker, ensureSess
|
|
|
1016
1025
|
const MIN_CLEANUP_INTERVAL_MS = 10_000;
|
|
1017
1026
|
const MAX_CLEANUP_INTERVAL_MS = 60_000;
|
|
1018
1027
|
const SESSION_CLOSE_BATCH_SIZE = 10;
|
|
1028
|
+
const SESSION_CLOSE_TIMEOUT_MS = 5_000;
|
|
1019
1029
|
function getCleanupIntervalMs(sessionTtlMs) {
|
|
1020
1030
|
return Math.min(Math.max(Math.floor(sessionTtlMs / 2), MIN_CLEANUP_INTERVAL_MS), MAX_CLEANUP_INTERVAL_MS);
|
|
1021
1031
|
}
|
|
@@ -1089,7 +1099,7 @@ class SessionCleanupLoop {
|
|
|
1089
1099
|
const timeoutPromise = new Promise((_, reject) => {
|
|
1090
1100
|
timeoutId = setTimeout(() => {
|
|
1091
1101
|
reject(new Error('Session close timed out'));
|
|
1092
|
-
},
|
|
1102
|
+
}, SESSION_CLOSE_TIMEOUT_MS);
|
|
1093
1103
|
timeoutId.unref();
|
|
1094
1104
|
});
|
|
1095
1105
|
try {
|
package/dist/lib/dom-prep.d.ts
CHANGED
|
@@ -7,6 +7,7 @@ export declare function serializeDocumentForMarkdown(document: Document, fallbac
|
|
|
7
7
|
export declare function normalizeTabContent(document: Document): void;
|
|
8
8
|
export declare function surfaceCodeEditorContent(document: Document): void;
|
|
9
9
|
export declare function stripDocsControls(document: Document): void;
|
|
10
|
+
export declare function stripScreenReaderText(document: Document): void;
|
|
10
11
|
export declare function runDocsControlPass(document: Document): void;
|
|
11
12
|
export declare function prepareDocumentForMarkdown(document: Document, baseUrl?: string, signal?: AbortSignal): void;
|
|
12
13
|
export declare function removeNoiseFromHtml(html: string, document?: Document, baseUrl?: string, signal?: AbortSignal): string;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"dom-prep.d.ts","sourceRoot":"","sources":["../../src/lib/dom-prep.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;
|
|
1
|
+
{"version":3,"file":"dom-prep.d.ts","sourceRoot":"","sources":["../../src/lib/dom-prep.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;AA4mB9D,eAAO,MAAM,sBAAsB,QAAmB,CAAC;AAyCvD,wBAAgB,qBAAqB,CAAC,QAAQ,EAAE,QAAQ,GAAG,IAAI,CA6B9D;AAuBD,wBAAgB,mBAAmB,CAAC,QAAQ,EAAE,QAAQ,GAAG,OAAO,CAY/D;AAED,wBAAgB,4BAA4B,CAC1C,QAAQ,EAAE,QAAQ,EAClB,QAAQ,EAAE,MAAM,GACf,MAAM,CASR;AA0CD,qEAAqE;AACrE,wBAAgB,mBAAmB,CAAC,QAAQ,EAAE,QAAQ,GAAG,IAAI,CAG5D;AA0RD,wBAAgB,wBAAwB,CAAC,QAAQ,EAAE,QAAQ,GAAG,IAAI,CA2BjE;AAED,wBAAgB,iBAAiB,CAAC,QAAQ,EAAE,QAAQ,GAAG,IAAI,CAE1D;AAED,wBAAgB,qBAAqB,CAAC,QAAQ,EAAE,QAAQ,GAAG,IAAI,CAY9D;AAWD,wBAAgB,kBAAkB,CAAC,QAAQ,EAAE,QAAQ,GAAG,IAAI,CAQ3D;AAuDD,wBAAgB,0BAA0B,CACxC,QAAQ,EAAE,QAAQ,EAClB,OAAO,CAAC,EAAE,MAAM,EAChB,MAAM,CAAC,EAAE,WAAW,GACnB,IAAI,CAON;AA4BD,wBAAgB,mBAAmB,CACjC,IAAI,EAAE,MAAM,EACZ,QAAQ,CAAC,EAAE,QAAQ,EACnB,OAAO,CAAC,EAAE,MAAM,EAChB,MAAM,CAAC,EAAE,WAAW,GACnB,MAAM,CAcR;AA0ED,wBAAgB,oBAAoB,CAClC,cAAc,EAAE,MAAM,GAAG,QAAQ,GAChC,MAAM,CAaR;AAiMD,wBAAgB,sBAAsB,CACpC,OAAO,EAAE,gBAAgB,EACzB,QAAQ,EAAE,QAAQ,GACjB,QAAQ,GAAG,IAAI,CA8BjB"}
|
package/dist/lib/dom-prep.js
CHANGED
|
@@ -181,6 +181,26 @@ function getPromoMatchers(currentConfig, enabledCategories) {
|
|
|
181
181
|
aggressive: buildTokenRegex(aggressiveTokens),
|
|
182
182
|
};
|
|
183
183
|
}
|
|
184
|
+
function buildNoiseSelector(flags) {
|
|
185
|
+
const selectors = [BASE_NOISE_SELECTORS.hidden];
|
|
186
|
+
if (flags.navFooter)
|
|
187
|
+
selectors.push(BASE_NOISE_SELECTORS.navFooter);
|
|
188
|
+
if (flags.cookieBanners)
|
|
189
|
+
selectors.push(BASE_NOISE_SELECTORS.cookieBanners);
|
|
190
|
+
return selectors.join(',');
|
|
191
|
+
}
|
|
192
|
+
function buildCandidateSelector(structuralTags) {
|
|
193
|
+
return [
|
|
194
|
+
...structuralTags,
|
|
195
|
+
...ALWAYS_NOISE_TAGS,
|
|
196
|
+
'aside',
|
|
197
|
+
'header',
|
|
198
|
+
'[class]',
|
|
199
|
+
'[id]',
|
|
200
|
+
'[role]',
|
|
201
|
+
'[style]',
|
|
202
|
+
].join(',');
|
|
203
|
+
}
|
|
184
204
|
function getContext() {
|
|
185
205
|
const currentConfig = config.noiseRemoval;
|
|
186
206
|
const contextKey = JSON.stringify({
|
|
@@ -214,24 +234,9 @@ function getContext() {
|
|
|
214
234
|
const extraSelectors = currentConfig.extraSelectors
|
|
215
235
|
.map((s) => s.trim())
|
|
216
236
|
.filter((s) => s.length > 0);
|
|
217
|
-
|
|
218
|
-
const selectors = [BASE_NOISE_SELECTORS.hidden];
|
|
219
|
-
if (flags.navFooter)
|
|
220
|
-
selectors.push(BASE_NOISE_SELECTORS.navFooter);
|
|
221
|
-
if (flags.cookieBanners)
|
|
222
|
-
selectors.push(BASE_NOISE_SELECTORS.cookieBanners);
|
|
223
|
-
const noiseSelector = selectors.join(',');
|
|
237
|
+
const noiseSelector = buildNoiseSelector(flags);
|
|
224
238
|
const extraSelector = extraSelectors.length > 0 ? extraSelectors.join(',') : null;
|
|
225
|
-
const candidateSelector =
|
|
226
|
-
...structuralTags,
|
|
227
|
-
...ALWAYS_NOISE_TAGS,
|
|
228
|
-
'aside',
|
|
229
|
-
'header',
|
|
230
|
-
'[class]',
|
|
231
|
-
'[id]',
|
|
232
|
-
'[role]',
|
|
233
|
-
'[style]',
|
|
234
|
-
].join(',');
|
|
239
|
+
const candidateSelector = buildCandidateSelector(structuralTags);
|
|
235
240
|
cachedContext = {
|
|
236
241
|
flags,
|
|
237
242
|
structuralTags,
|
|
@@ -313,6 +318,46 @@ function removeNodes(nodes) {
|
|
|
313
318
|
}
|
|
314
319
|
}
|
|
315
320
|
}
|
|
321
|
+
function isStructuralNoise(tagName, interactive, context) {
|
|
322
|
+
return context.structuralTags.has(tagName) && !interactive;
|
|
323
|
+
}
|
|
324
|
+
function isNavigationNoise(tagName, role, className, id, context) {
|
|
325
|
+
if (!context.flags.navFooter)
|
|
326
|
+
return false;
|
|
327
|
+
if (ALWAYS_NOISE_TAGS.has(tagName))
|
|
328
|
+
return true;
|
|
329
|
+
if (tagName === 'header' &&
|
|
330
|
+
((role !== null && NAVIGATION_ROLES.has(role)) ||
|
|
331
|
+
HEADER_NOISE_PATTERN.test(`${className} ${id}`)))
|
|
332
|
+
return true;
|
|
333
|
+
if (tagName === 'aside')
|
|
334
|
+
return true;
|
|
335
|
+
return (role !== null &&
|
|
336
|
+
NAVIGATION_ROLES.has(role) &&
|
|
337
|
+
(tagName !== 'aside' || role !== 'complementary'));
|
|
338
|
+
}
|
|
339
|
+
function isHiddenNoise(hidden, interactive) {
|
|
340
|
+
return hidden && !interactive;
|
|
341
|
+
}
|
|
342
|
+
function isPositionalNoise(className, element) {
|
|
343
|
+
return (FIXED_OR_HIGH_Z_PATTERN.test(className) &&
|
|
344
|
+
(element.textContent || '').trim().length <
|
|
345
|
+
NAV_FOOTER_MIN_CHARS_FOR_PRESERVATION);
|
|
346
|
+
}
|
|
347
|
+
function isPromoNoise(className, id, element, context) {
|
|
348
|
+
if (!context.promoEnabled)
|
|
349
|
+
return false;
|
|
350
|
+
const aggTest = context.promoMatchers.aggressive.test(className) ||
|
|
351
|
+
context.promoMatchers.aggressive.test(id);
|
|
352
|
+
if (aggTest && !isPrimaryContent(element))
|
|
353
|
+
return true;
|
|
354
|
+
if (context.promoMatchers.base.test(className) ||
|
|
355
|
+
context.promoMatchers.base.test(id)) {
|
|
356
|
+
if (!isPrimaryContent(element, true))
|
|
357
|
+
return true;
|
|
358
|
+
}
|
|
359
|
+
return false;
|
|
360
|
+
}
|
|
316
361
|
function isNoiseElement(element, context) {
|
|
317
362
|
const tagName = element.tagName.toLowerCase();
|
|
318
363
|
const role = element.getAttribute('role');
|
|
@@ -323,46 +368,11 @@ function isNoiseElement(element, context) {
|
|
|
323
368
|
const hidden = element.hasAttribute('hidden') ||
|
|
324
369
|
element.getAttribute('aria-hidden') === 'true' ||
|
|
325
370
|
(style !== null && HIDDEN_STYLE_REGEX.test(style));
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
if (ALWAYS_NOISE_TAGS.has(tagName))
|
|
332
|
-
return true;
|
|
333
|
-
if (tagName === 'header' &&
|
|
334
|
-
((role !== null && NAVIGATION_ROLES.has(role)) ||
|
|
335
|
-
HEADER_NOISE_PATTERN.test(`${className} ${id}`)))
|
|
336
|
-
return true;
|
|
337
|
-
if (tagName === 'aside')
|
|
338
|
-
return true;
|
|
339
|
-
if (role !== null &&
|
|
340
|
-
NAVIGATION_ROLES.has(role) &&
|
|
341
|
-
(tagName !== 'aside' || role !== 'complementary'))
|
|
342
|
-
return true;
|
|
343
|
-
}
|
|
344
|
-
// 3. Hidden Noise
|
|
345
|
-
if (hidden && !interactive)
|
|
346
|
-
return true;
|
|
347
|
-
// 4. Positional Noise
|
|
348
|
-
if (FIXED_OR_HIGH_Z_PATTERN.test(className) &&
|
|
349
|
-
(element.textContent || '').trim().length <
|
|
350
|
-
NAV_FOOTER_MIN_CHARS_FOR_PRESERVATION) {
|
|
351
|
-
return true;
|
|
352
|
-
}
|
|
353
|
-
// 5. Promo/Advert Noise
|
|
354
|
-
if (context.promoEnabled) {
|
|
355
|
-
const aggTest = context.promoMatchers.aggressive.test(className) ||
|
|
356
|
-
context.promoMatchers.aggressive.test(id);
|
|
357
|
-
if (aggTest && !isPrimaryContent(element))
|
|
358
|
-
return true;
|
|
359
|
-
if (context.promoMatchers.base.test(className) ||
|
|
360
|
-
context.promoMatchers.base.test(id)) {
|
|
361
|
-
if (!isPrimaryContent(element, true))
|
|
362
|
-
return true;
|
|
363
|
-
}
|
|
364
|
-
}
|
|
365
|
-
return false;
|
|
371
|
+
return (isStructuralNoise(tagName, interactive, context) ||
|
|
372
|
+
isNavigationNoise(tagName, role, className, id, context) ||
|
|
373
|
+
isHiddenNoise(hidden, interactive) ||
|
|
374
|
+
isPositionalNoise(className, element) ||
|
|
375
|
+
isPromoNoise(className, id, element, context));
|
|
366
376
|
}
|
|
367
377
|
function stripHeadingWrapperDivs(h) {
|
|
368
378
|
const divs = h.querySelectorAll('div');
|
|
@@ -905,6 +915,19 @@ export function surfaceCodeEditorContent(document) {
|
|
|
905
915
|
export function stripDocsControls(document) {
|
|
906
916
|
removeNodes(document.querySelectorAll(DOCS_CONTROL_SELECTORS.join(',')));
|
|
907
917
|
}
|
|
918
|
+
export function stripScreenReaderText(document) {
|
|
919
|
+
const selectors = [
|
|
920
|
+
'.sr-only',
|
|
921
|
+
'.screen-reader-text',
|
|
922
|
+
'.visually-hidden',
|
|
923
|
+
'[class*="sr-only"]',
|
|
924
|
+
'[class*="visually-hidden"]',
|
|
925
|
+
'.cdk-visually-hidden',
|
|
926
|
+
'.vh',
|
|
927
|
+
'.hidden-visually',
|
|
928
|
+
];
|
|
929
|
+
removeNodes(document.querySelectorAll(selectors.join(',')));
|
|
930
|
+
}
|
|
908
931
|
function stripAriaLiveInstructions(document) {
|
|
909
932
|
for (const el of document.querySelectorAll('[aria-live]')) {
|
|
910
933
|
const text = (el.textContent || '').trim();
|
|
@@ -1115,6 +1138,13 @@ function countEmptyHeadingSections(root) {
|
|
|
1115
1138
|
let emptyCount = 0;
|
|
1116
1139
|
const headings = root.querySelectorAll('h1,h2,h3,h4,h5,h6');
|
|
1117
1140
|
for (const heading of headings) {
|
|
1141
|
+
// Skip headings that are explicitly hidden or for screen readers
|
|
1142
|
+
const cls = heading.getAttribute('class') ?? '';
|
|
1143
|
+
if (cls.includes('screen-reader-text') ||
|
|
1144
|
+
cls.includes('sr-only') ||
|
|
1145
|
+
cls.includes('visually-hidden')) {
|
|
1146
|
+
continue;
|
|
1147
|
+
}
|
|
1118
1148
|
if (!hasSectionContent(heading))
|
|
1119
1149
|
emptyCount += 1;
|
|
1120
1150
|
}
|
|
@@ -1218,23 +1248,37 @@ function passesRetentionRulesFromHtml(originalDoc, articleHtml) {
|
|
|
1218
1248
|
});
|
|
1219
1249
|
}
|
|
1220
1250
|
function passesEmptySectionRatio(articleDoc) {
|
|
1221
|
-
const
|
|
1251
|
+
const headings = Array.from(articleDoc.querySelectorAll('h1,h2,h3,h4,h5,h6')).filter((h) => {
|
|
1252
|
+
const cls = h.getAttribute('class') ?? '';
|
|
1253
|
+
return (!cls.includes('screen-reader-text') &&
|
|
1254
|
+
!cls.includes('sr-only') &&
|
|
1255
|
+
!cls.includes('visually-hidden'));
|
|
1256
|
+
});
|
|
1257
|
+
const headingCount = headings.length;
|
|
1222
1258
|
return (headingCount < MIN_HEADINGS_FOR_EMPTY_SECTION_GATE ||
|
|
1223
1259
|
countEmptyHeadingSections(articleDoc) / headingCount <=
|
|
1224
1260
|
MAX_EMPTY_SECTION_RATIO);
|
|
1225
1261
|
}
|
|
1226
1262
|
export function evaluateArticleContent(article, document) {
|
|
1227
1263
|
if (!passesContentRatioGate(article.textContent.length, document)) {
|
|
1264
|
+
console.log('FAILED passesContentRatioGate');
|
|
1228
1265
|
return null;
|
|
1229
1266
|
}
|
|
1230
1267
|
if (!passesRetentionRulesFromHtml(document, article.content)) {
|
|
1268
|
+
console.log('FAILED passesRetentionRulesFromHtml');
|
|
1231
1269
|
return null;
|
|
1232
1270
|
}
|
|
1233
1271
|
if (hasTruncatedSentences(article.textContent)) {
|
|
1272
|
+
console.log('FAILED hasTruncatedSentences');
|
|
1234
1273
|
return null;
|
|
1235
1274
|
}
|
|
1236
1275
|
const articleDoc = parseHTML(`<!DOCTYPE html><html><body>${article.content}</body></html>`).document;
|
|
1237
1276
|
if (!passesEmptySectionRatio(articleDoc)) {
|
|
1277
|
+
const headings = articleDoc.querySelectorAll('h1,h2,h3,h4,h5,h6');
|
|
1278
|
+
console.log('FAILED passesEmptySectionRatio:', headings.length, 'headings');
|
|
1279
|
+
for (const h of headings) {
|
|
1280
|
+
console.log('H:', h.textContent, hasSectionContent(h));
|
|
1281
|
+
}
|
|
1238
1282
|
return null;
|
|
1239
1283
|
}
|
|
1240
1284
|
return articleDoc;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"fetch-pipeline.d.ts","sourceRoot":"","sources":["../../src/lib/fetch-pipeline.ts"],"names":[],"mappings":"AAYA,OAAO,EAAE,KAAK,uBAAuB,EAAE,MAAM,uBAAuB,CAAC;AAgBrE,OAAO,EAA6B,UAAU,EAAE,MAAM,YAAY,CAAC;AAEnE,OAAO,EAAE,UAAU,EAAE,CAAC;AAGtB,MAAM,WAAW,mBAAmB;IAClC,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,WAAW,EAAE,MAAM,CAAC;IACpB,SAAS,CAAC,EAAE,OAAO,CAAC;CACrB;AACD,UAAU,mBAAmB;IAC3B,MAAM,EAAE,UAAU,CAAC;IACnB,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,CAAC,EAAE,OAAO,CAAC;CACrB;
|
|
1
|
+
{"version":3,"file":"fetch-pipeline.d.ts","sourceRoot":"","sources":["../../src/lib/fetch-pipeline.ts"],"names":[],"mappings":"AAYA,OAAO,EAAE,KAAK,uBAAuB,EAAE,MAAM,uBAAuB,CAAC;AAgBrE,OAAO,EAA6B,UAAU,EAAE,MAAM,YAAY,CAAC;AAEnE,OAAO,EAAE,UAAU,EAAE,CAAC;AAGtB,MAAM,WAAW,mBAAmB;IAClC,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,WAAW,EAAE,MAAM,CAAC;IACpB,SAAS,CAAC,EAAE,OAAO,CAAC;CACrB;AACD,UAAU,mBAAmB;IAC3B,MAAM,EAAE,UAAU,CAAC;IACnB,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,CAAC,EAAE,OAAO,CAAC;CACrB;AAmGD,wBAAgB,sBAAsB,CACpC,QAAQ,EAAE,MAAM,GAAG,SAAS,EAC5B,OAAO,GAAE;IAAE,QAAQ,CAAC,EAAE,MAAM,CAAA;CAAO,GAClC,MAAM,GAAG,SAAS,CAQpB;AAyBD,UAAU,oBAAoB,CAAC,CAAC;IAC9B,GAAG,EAAE,MAAM,CAAC;IACZ,cAAc,EAAE,MAAM,CAAC;IACvB,MAAM,CAAC,EAAE,WAAW,CAAC;IACrB,SAAS,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,MAAM,CAAC;IAC7C,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB,OAAO,CAAC,EAAE,CAAC,KAAK,EAAE,gBAAgB,KAAK,IAAI,CAAC;IAC5C,SAAS,EAAE,CAAC,KAAK,EAAE,mBAAmB,EAAE,GAAG,EAAE,MAAM,KAAK,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;IACvE,SAAS,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,KAAK,MAAM,CAAC;IAClC,WAAW,CAAC,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK,CAAC,GAAG,SAAS,CAAC;CACjD;AACD,MAAM,WAAW,cAAc,CAAC,CAAC;IAC/B,IAAI,EAAE,CAAC,CAAC;IACR,SAAS,EAAE,OAAO,CAAC;IACnB,GAAG,EAAE,MAAM,CAAC;IACZ,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CAC1B;AACD,MAAM,MAAM,gBAAgB,GACxB,aAAa,GACb,aAAa,GACb,WAAW,GACX,eAAe,GACf,cAAc,GACd,gBAAgB,GAChB,iBAAiB,GACjB,gBAAgB,GAChB,iBAAiB,CAAC;AAqKtB,wBAAsB,oBAAoB,CAAC,CAAC,EAC1C,OAAO,EAAE,oBAAoB,CAAC,CAAC,CAAC,GAC/B,OAAO,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC,CA2D5B;AAED,MAAM,MAAM,sBAAsB,GAAG,uBAAuB,GAAG;IAC7D,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;CAC1B,CAAC;AA4DF,wBAAgB,yBAAyB,CACvC,MAAM,EAAE,MAAM,GACb,sBAAsB,GAAG,SAAS,CAKpC;AAED,eAAO,MAAM,iBAAiB,GAC5B,OAAO,mBAAmB,EAC1B,KAAK,MAAM,EACX,SAAS,WAAW,KACnB,OAAO,CAAC,sBAAsB,CAchC,CAAC;AAEF,wBAAgB,uBAAuB,CACrC,MAAM,EAAE,sBAAsB,GAC7B,MAAM,CAER;AAED,UAAU,oBAAoB;IAC5B,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC;IACrB,QAAQ,CAAC,MAAM,CAAC,EAAE,WAAW,CAAC;IAC9B,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,MAAM,CAAC;IACtD,QAAQ,CAAC,YAAY,CAAC,EAAE,OAAO,CAAC;IAChC,QAAQ,CAAC,OAAO,CAAC,EAAE,CAAC,KAAK,EAAE,gBAAgB,KAAK,IAAI,CAAC;IACrD,QAAQ,CAAC,SAAS,EAAE,CAClB,KAAK,EAAE,mBAAmB,EAC1B,aAAa,EAAE,MAAM,KAClB,sBAAsB,GAAG,OAAO,CAAC,sBAAsB,CAAC,CAAC;IAC9D,QAAQ,CAAC,SAAS,CAAC,EAAE,CAAC,MAAM,EAAE,sBAAsB,KAAK,MAAM,CAAC;IAChE,QAAQ,CAAC,WAAW,CAAC,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK,sBAAsB,GAAG,SAAS,CAAC;CAC/E;AACD,UAAU,eAAe;IACvB,QAAQ,CAAC,oBAAoB,CAAC,EAAE,OAAO,oBAAoB,CAAC;CAC7D;AASD,wBAAsB,kBAAkB,CACtC,OAAO,EAAE,oBAAoB,EAC7B,IAAI,GAAE,eAAoB,GACzB,OAAO,CAAC;IACT,QAAQ,EAAE,cAAc,CAAC,sBAAsB,CAAC,CAAC;IACjD,YAAY,EAAE,mBAAmB,CAAC;CACnC,CAAC,CAaD"}
|
|
@@ -8,20 +8,23 @@ import { getErrorMessage, isObject, withSignal } from './utils.js';
|
|
|
8
8
|
export { withSignal };
|
|
9
9
|
const TRUNCATION_MARKER = '...[truncated]';
|
|
10
10
|
function getOpenCodeFence(content) {
|
|
11
|
-
const FENCE_PATTERN = /^[ \t]*(`{3,}|~{3,})
|
|
11
|
+
const FENCE_PATTERN = /^[ \t]*(`{3,}|~{3,})([^\S\r\n]*|[^\r\n]*)$/gm;
|
|
12
12
|
let inFence = false;
|
|
13
13
|
let fenceChar = null;
|
|
14
14
|
let fenceLength = 0;
|
|
15
15
|
for (const match of content.matchAll(FENCE_PATTERN)) {
|
|
16
16
|
const marker = match[1] ?? '';
|
|
17
|
+
const suffix = match[2] ?? '';
|
|
17
18
|
const char = marker[0] ?? '';
|
|
18
19
|
const { length } = marker;
|
|
19
20
|
if (!inFence) {
|
|
20
21
|
inFence = true;
|
|
21
22
|
fenceChar = char;
|
|
22
23
|
fenceLength = length;
|
|
24
|
+
continue;
|
|
23
25
|
}
|
|
24
|
-
|
|
26
|
+
const isClosingFence = char === fenceChar && length >= fenceLength && suffix.trim().length === 0;
|
|
27
|
+
if (isClosingFence) {
|
|
25
28
|
inFence = false;
|
|
26
29
|
fenceChar = null;
|
|
27
30
|
fenceLength = 0;
|
package/dist/schemas.js
CHANGED
|
@@ -149,7 +149,7 @@ export const fetchUrlOutputSchema = z.strictObject({
|
|
|
149
149
|
.min(0)
|
|
150
150
|
.max(config.constants.maxHtmlBytes * 4)
|
|
151
151
|
.optional()
|
|
152
|
-
.describe('
|
|
152
|
+
.describe('Markdown fragment size before final inline truncation.'),
|
|
153
153
|
truncated: z.boolean().optional().describe('True if markdown was truncated.'),
|
|
154
154
|
});
|
|
155
155
|
export function parseCachedPayload(raw) {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"transform.d.ts","sourceRoot":"","sources":["../../src/transform/transform.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"transform.d.ts","sourceRoot":"","sources":["../../src/transform/transform.ts"],"names":[],"mappings":"AA2CA,OAAO,EACL,wBAAwB,EACxB,wBAAwB,EACxB,oBAAoB,EACrB,MAAM,uBAAuB,CAAC;AAoB/B,OAAO,KAAK,EACV,gBAAgB,EAChB,iBAAiB,EACjB,gBAAgB,EAChB,uBAAuB,EACvB,aAAa,EACb,gBAAgB,EAChB,qBAAqB,EAEtB,MAAM,YAAY,CAAC;AA+BpB,UAAU,WAAW;IACnB,aAAa,EAAE,MAAM,CAAC;IACtB,SAAS,EAAE,MAAM,CAAC;CACnB;AA4LD,wBAAgB,mBAAmB,CACjC,GAAG,EAAE,MAAM,EACX,KAAK,EAAE,MAAM,EACb,MAAM,CAAC,EAAE,WAAW,GACnB,qBAAqB,GAAG,IAAI,CAE9B;AAED,wBAAgB,iBAAiB,CAC/B,OAAO,EAAE,qBAAqB,GAAG,IAAI,EACrC,OAAO,CAAC,EAAE;IAAE,SAAS,CAAC,EAAE,OAAO,CAAA;CAAE,GAChC,MAAM,CAER;AA4aD,wBAAgB,cAAc,CAC5B,IAAI,EAAE,MAAM,EACZ,GAAG,EAAE,MAAM,EACX,OAAO,GAAE;IAAE,cAAc,CAAC,EAAE,OAAO,CAAC;IAAC,MAAM,CAAC,EAAE,WAAW,CAAA;CAExD,GACA,gBAAgB,CAGlB;AAuKD,wBAAgB,cAAc,CAC5B,IAAI,EAAE,MAAM,EACZ,QAAQ,CAAC,EAAE,aAAa,EACxB,OAAO,CAAC,EAAE;IACR,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,MAAM,CAAC,EAAE,WAAW,GAAG,SAAS,CAAC;IACjC,QAAQ,CAAC,EAAE,QAAQ,GAAG,SAAS,CAAC;IAChC,gBAAgB,CAAC,EAAE,OAAO,GAAG,SAAS,CAAC;CACxC,GACA,MAAM,CAyBR;AA2DD,wBAAgB,sBAAsB,CACpC,OAAO,EAAE,gBAAgB,GAAG,IAAI,EAChC,sBAAsB,EAAE,MAAM,GAAG,QAAQ,GACxC,OAAO,CAQT;AAKD,wBAAgB,gCAAgC,CAC9C,OAAO,EAAE,gBAAgB,GAAG,IAAI,GAC/B,OAAO,IAAI,gBAAgB,CAE7B;AAED,wBAAgB,0BAA0B,CACxC,GAAG,EAAE,MAAM,EACX,OAAO,EAAE,gBAAgB,GAAG,IAAI,EAChC,aAAa,EAAE,iBAAiB,EAChC,wBAAwB,EAAE,OAAO,EACjC,eAAe,EAAE,OAAO,GACvB,aAAa,GAAG,SAAS,CAuB3B;AA0bD,wBAAgB,gCAAgC,CAC9C,IAAI,EAAE,MAAM,EACZ,GAAG,EAAE,MAAM,EACX,OAAO,EAAE,gBAAgB,GACxB,uBAAuB,CAMzB;AAaD,UAAU,kBAAkB;IAC1B,UAAU,EAAE,MAAM,CAAC;IACnB,aAAa,EAAE,MAAM,CAAC;IACtB,QAAQ,EAAE,MAAM,CAAC;CAClB;AAED,wBAAgB,qBAAqB,IAAI,kBAAkB,GAAG,IAAI,CAEjE;AAED,wBAAsB,2BAA2B,IAAI,OAAO,CAAC,IAAI,CAAC,CAEjE;AAED,KAAK,yBAAyB,GAAG,gBAAgB,GAAG;IAAE,QAAQ,CAAC,EAAE,MAAM,CAAA;CAAE,CAAC;AAuG1E,wBAAsB,uBAAuB,CAC3C,IAAI,EAAE,MAAM,EACZ,GAAG,EAAE,MAAM,EACX,OAAO,EAAE,gBAAgB,GACxB,OAAO,CAAC,uBAAuB,CAAC,CAElC;AAED,wBAAsB,yBAAyB,CAC7C,UAAU,EAAE,UAAU,EACtB,GAAG,EAAE,MAAM,EACX,OAAO,EAAE,yBAAyB,GACjC,OAAO,CAAC,uBAAuB,CAAC,CAElC;AAED,OAAO,EACL,wBAAwB,EACxB,wBAAwB,EACxB,oBAAoB,GACrB,CAAC"}
|
|
@@ -3,7 +3,7 @@ import { isProbablyReaderable, Readability } from '@mozilla/readability';
|
|
|
3
3
|
import { parseHTML } from 'linkedom';
|
|
4
4
|
import { config } from '../lib/core.js';
|
|
5
5
|
import { getOperationId, getRequestId, logDebug, logError, logInfo, logWarn, redactUrl, } from '../lib/core.js';
|
|
6
|
-
import { evaluateArticleContent, extractNoscriptImages, getVisibleTextLength, normalizeTabContent, prepareDocumentForMarkdown, removeNoiseFromHtml, serializeDocumentForMarkdown, stripDocsControls, surfaceCodeEditorContent, } from '../lib/dom-prep.js';
|
|
6
|
+
import { evaluateArticleContent, extractNoscriptImages, getVisibleTextLength, normalizeTabContent, prepareDocumentForMarkdown, removeNoiseFromHtml, serializeDocumentForMarkdown, stripDocsControls, stripScreenReaderText, surfaceCodeEditorContent, } from '../lib/dom-prep.js';
|
|
7
7
|
import { isRawTextContentUrl } from '../lib/http.js';
|
|
8
8
|
import { composeAbortSignal, FetchError, getErrorMessage, getUtf8ByteLength, isAsciiOnly, isObject, throwIfAborted, toError, trimDanglingTagFragment, truncateToUtf8Boundary, } from '../lib/utils.js';
|
|
9
9
|
import { extractLanguageFromClassName } from './html-translators.js';
|
|
@@ -321,6 +321,7 @@ function prepareReadabilityDocument(readabilityDoc) {
|
|
|
321
321
|
normalizeTabContent(readabilityDoc);
|
|
322
322
|
surfaceCodeEditorContent(readabilityDoc);
|
|
323
323
|
stripDocsControls(readabilityDoc);
|
|
324
|
+
stripScreenReaderText(readabilityDoc);
|
|
324
325
|
for (const el of readabilityDoc.querySelectorAll('[class*="breadcrumb"],[class*="pagination"]')) {
|
|
325
326
|
if (el.tagName === 'HTML' || el.tagName === 'BODY')
|
|
326
327
|
continue;
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@j0hanz/fetch-url-mcp",
|
|
3
|
-
"version": "1.
|
|
3
|
+
"version": "1.11.0",
|
|
4
4
|
"mcpName": "io.github.j0hanz/fetch-url-mcp",
|
|
5
5
|
"description": "A web content fetcher MCP server that converts HTML to clean, AI and human readable markdown.",
|
|
6
6
|
"type": "module",
|
|
@@ -59,10 +59,9 @@
|
|
|
59
59
|
"type-check:diagnostics": "tsc --noEmit --extendedDiagnostics",
|
|
60
60
|
"type-check:trace": "node -e \"require('fs').rmSync('.ts-trace',{recursive:true,force:true})\" && tsc --noEmit --generateTrace .ts-trace",
|
|
61
61
|
"lint": "eslint .",
|
|
62
|
-
"lint:tests": "eslint
|
|
62
|
+
"lint:tests": "eslint __tests__",
|
|
63
63
|
"lint:fix": "eslint . --fix",
|
|
64
64
|
"test": "node scripts/tasks.mjs test",
|
|
65
|
-
"test:fast": "node --test --import tsx/esm src/__tests__/**/*.test.ts node-tests/**/*.test.ts",
|
|
66
65
|
"test:coverage": "node scripts/tasks.mjs test --coverage",
|
|
67
66
|
"knip": "knip",
|
|
68
67
|
"knip:fix": "knip --fix",
|