@j0hanz/superfetch 1.0.5 → 1.0.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/dist/config/index.d.ts +5 -0
- package/dist/config/index.d.ts.map +1 -1
- package/dist/config/index.js +5 -0
- package/dist/config/index.js.map +1 -1
- package/dist/index.js +33 -3
- package/dist/index.js.map +1 -1
- package/dist/middleware/error-handler.d.ts.map +1 -1
- package/dist/middleware/error-handler.js +1 -1
- package/dist/middleware/error-handler.js.map +1 -1
- package/dist/middleware/rate-limiter.d.ts.map +1 -1
- package/dist/middleware/rate-limiter.js +37 -12
- package/dist/middleware/rate-limiter.js.map +1 -1
- package/dist/resources/cached-content.d.ts +0 -1
- package/dist/resources/cached-content.d.ts.map +1 -1
- package/dist/resources/cached-content.js +3 -28
- package/dist/resources/cached-content.js.map +1 -1
- package/dist/server.d.ts.map +1 -1
- package/dist/server.js +3 -2
- package/dist/server.js.map +1 -1
- package/dist/services/cache.d.ts.map +1 -1
- package/dist/services/cache.js +13 -6
- package/dist/services/cache.js.map +1 -1
- package/dist/services/card-extractor.d.ts +0 -4
- package/dist/services/card-extractor.d.ts.map +1 -1
- package/dist/services/card-extractor.js +6 -1
- package/dist/services/card-extractor.js.map +1 -1
- package/dist/services/extractor.d.ts.map +1 -1
- package/dist/services/extractor.js +4 -4
- package/dist/services/extractor.js.map +1 -1
- package/dist/services/fetcher.d.ts.map +1 -1
- package/dist/services/fetcher.js +39 -8
- package/dist/services/fetcher.js.map +1 -1
- package/dist/services/parser.d.ts.map +1 -1
- package/dist/services/parser.js +6 -7
- package/dist/services/parser.js.map +1 -1
- package/dist/tools/handlers/fetch-links.tool.d.ts.map +1 -1
- package/dist/tools/handlers/fetch-links.tool.js +4 -0
- package/dist/tools/handlers/fetch-links.tool.js.map +1 -1
- package/dist/tools/handlers/fetch-urls.tool.d.ts.map +1 -1
- package/dist/tools/handlers/fetch-urls.tool.js +25 -11
- package/dist/tools/handlers/fetch-urls.tool.js.map +1 -1
- package/dist/tools/utils/fetch-pipeline.d.ts.map +1 -1
- package/dist/tools/utils/fetch-pipeline.js +39 -10
- package/dist/tools/utils/fetch-pipeline.js.map +1 -1
- package/dist/transformers/markdown.transformer.js +1 -1
- package/dist/transformers/markdown.transformer.js.map +1 -1
- package/dist/utils/content-cleaner.d.ts.map +1 -1
- package/dist/utils/content-cleaner.js +26 -8
- package/dist/utils/content-cleaner.js.map +1 -1
- package/dist/utils/tool-error-handler.d.ts.map +1 -1
- package/dist/utils/tool-error-handler.js +4 -2
- package/dist/utils/tool-error-handler.js.map +1 -1
- package/dist/utils/url-validator.d.ts +10 -0
- package/dist/utils/url-validator.d.ts.map +1 -1
- package/dist/utils/url-validator.js +43 -5
- package/dist/utils/url-validator.js.map +1 -1
- package/package.json +8 -6
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"content-cleaner.js","sourceRoot":"","sources":["../../src/utils/content-cleaner.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,wDAAwD;AACxD,MAAM,sBAAsB,GAAG,IAAI,MAAM,CACvC;IACE,sBAAsB;IACtB,yEAAyE;IACzE,4DAA4D;IAC5D,8HAA8H;IAC9H,6EAA6E;IAC7E,2EAA2E;IAC3E,wBAAwB;IACxB,wBAAwB;IACxB,uBAAuB;IACvB,6FAA6F;IAC7F,oDAAoD;IACpD,+EAA+E;IAC/E,eAAe;IACf,+GAA+G;IAC/G,yCAAyC;IACzC,gFAAgF;IAChF,qCAAqC;IACrC,+DAA+D;IAC/D,uEAAuE;IACvE,oBAAoB;IACpB,iDAAiD;IACjD,qDAAqD;IACrD,aAAa;IACb,gFAAgF;IAChF,iFAAiF;IACjF,mCAAmC;IACnC,iBAAiB;IACjB,6FAA6F;IAC7F,qGAAqG;IACrG,SAAS;IACT,sDAAsD;IACtD,iGAAiG;IACjG,2CAA2C;IAC3C,iBAAiB;IACjB,mCAAmC;IACnC,6EAA6E;IAC7E,aAAa;IACb,yDAAyD;IACzD,mBAAmB;IACnB,gBAAgB;IAChB,8FAA8F;IAC9F,gDAAgD;IAChD,cAAc;IACd,+EAA+E;IAC/E,0DAA0D;IAC1D,cAAc;IACd,gBAAgB;IAChB,MAAM;IACN,yDAAyD;CAC1D,CAAC,IAAI,CAAC,GAAG,CAAC,EACX,GAAG,CACJ,CAAC;AAEF,4CAA4C;AAC5C,MAAM,wBAAwB,GAAG,IAAI,MAAM,CACzC;IACE,SAAS;IACT,SAAS;IACT,QAAQ;IACR,8BAA8B;IAC9B,eAAe;IACf,cAAc;IACd,cAAc;IACd,mBAAmB;IACnB,iBAAiB;IACjB,gBAAgB;IAChB,UAAU;IACV,SAAS;IACT,aAAa;IACb,UAAU;IACV,aAAa;CACd,CAAC,IAAI,CAAC,GAAG,CAAC,EACX,GAAG,CACJ,CAAC;AAEF,+CAA+C;AAC/C,MAAM,iBAAiB,GAAG,IAAI,MAAM,CAClC;IACE,4DAA4D;IAC5D,6DAA6D;IAC7D,gDAAgD;IAChD,mEAAmE;IACnE,+DAA+D;IAC/D,gDAAgD;CACjD,CAAC,IAAI,CAAC,GAAG,CAAC,EACX,GAAG,CACJ,CAAC;AAEF,8CAA8C;AAC9C,MAAM,oBAAoB,GAAG,EAAE,CAAC;AAChC,MAAM,kBAAkB,GAAG,CAAC,CAAC;AAC7B,MAAM,oBAAoB,GAAG,CAAC,CAAC;AAC/B,MAAM,oBAAoB,GAAG,EAAE,CAAC;AAEhC
|
|
1
|
+
{"version":3,"file":"content-cleaner.js","sourceRoot":"","sources":["../../src/utils/content-cleaner.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,wDAAwD;AACxD,MAAM,sBAAsB,GAAG,IAAI,MAAM,CACvC;IACE,sBAAsB;IACtB,yEAAyE;IACzE,4DAA4D;IAC5D,8HAA8H;IAC9H,6EAA6E;IAC7E,2EAA2E;IAC3E,wBAAwB;IACxB,wBAAwB;IACxB,uBAAuB;IACvB,6FAA6F;IAC7F,oDAAoD;IACpD,+EAA+E;IAC/E,eAAe;IACf,+GAA+G;IAC/G,yCAAyC;IACzC,gFAAgF;IAChF,qCAAqC;IACrC,+DAA+D;IAC/D,uEAAuE;IACvE,oBAAoB;IACpB,iDAAiD;IACjD,qDAAqD;IACrD,aAAa;IACb,gFAAgF;IAChF,iFAAiF;IACjF,mCAAmC;IACnC,iBAAiB;IACjB,6FAA6F;IAC7F,qGAAqG;IACrG,SAAS;IACT,sDAAsD;IACtD,iGAAiG;IACjG,2CAA2C;IAC3C,iBAAiB;IACjB,mCAAmC;IACnC,6EAA6E;IAC7E,aAAa;IACb,yDAAyD;IACzD,mBAAmB;IACnB,gBAAgB;IAChB,8FAA8F;IAC9F,gDAAgD;IAChD,cAAc;IACd,+EAA+E;IAC/E,0DAA0D;IAC1D,cAAc;IACd,gBAAgB;IAChB,MAAM;IACN,yDAAyD;CAC1D,CAAC,IAAI,CAAC,GAAG,CAAC,EACX,GAAG,CACJ,CAAC;AAEF,4CAA4C;AAC5C,MAAM,wBAAwB,GAAG,IAAI,MAAM,CACzC;IACE,SAAS;IACT,SAAS;IACT,QAAQ;IACR,8BAA8B;IAC9B,eAAe;IACf,cAAc;IACd,cAAc;IACd,mBAAmB;IACnB,iBAAiB;IACjB,gBAAgB;IAChB,UAAU;IACV,SAAS;IACT,aAAa;IACb,UAAU;IACV,aAAa;CACd,CAAC,IAAI,CAAC,GAAG,CAAC,EACX,GAAG,CACJ,CAAC;AAEF,+CAA+C;AAC/C,MAAM,iBAAiB,GAAG,IAAI,MAAM,CAClC;IACE,4DAA4D;IAC5D,6DAA6D;IAC7D,gDAAgD;IAChD,mEAAmE;IACnE,+DAA+D;IAC/D,gDAAgD;CACjD,CAAC,IAAI,CAAC,GAAG,CAAC,EACX,GAAG,CACJ,CAAC;AAEF,8CAA8C;AAC9C,MAAM,oBAAoB,GAAG,EAAE,CAAC;AAChC,MAAM,kBAAkB,GAAG,CAAC,CAAC;AAC7B,MAAM,oBAAoB,GAAG,CAAC,CAAC;AAC/B,MAAM,oBAAoB,GAAG,EAAE,CAAC;AAEhC,wEAAwE;AACxE,MAAM,sBAAsB,GAAG,GAAG,CAAC;AAEnC;;;GAGG;AACH,SAAS,WAAW,CAAC,IAAY;IAC/B,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC;IAE5B,2BAA2B;IAC3B,IAAI,CAAC,OAAO,EAAE,CAAC;QACb,OAAO,IAAI,CAAC;IACd,CAAC;IAED,qDAAqD;IACrD,IAAI,OAAO,CAAC,MAAM,GAAG,sBAAsB,EAAE,CAAC;QAC5C,OAAO,KAAK,CAAC;IACf,CAAC;IAED,mDAAmD;IACnD,IAAI,sBAAsB,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;QACzC,OAAO,IAAI,CAAC;IACd,CAAC;IAED,8CAA8C;IAC9C,IAAI,OAAO,CAAC,MAAM,GAAG,oBAAoB,EAAE,CAAC;QAC1C,IAAI,wBAAwB,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;YAC3C,OAAO,IAAI,CAAC;QACd,CAAC;QAED,+CAA+C;QAC/C,IAAI,iBAAiB,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;YACpC,OAAO,IAAI,CAAC;QACd,CAAC;IACH,CAAC;IAED,OAAO,KAAK,CAAC;AACf,CAAC;AAED,8DAA8D;AAC9D,MAAM,mBAAmB,GACvB,2KAA2K,CAAC;AAO9K,MAAM,iBAAiB,GAAG,IAAI,GAAG,EAAsB,CAAC;AACxD,MAAM,0BAA0B,GAAG,IAAI,CAAC;AACxC,MAAM,wBAAwB,GAAG,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,CAAC,YAAY;AAE5D;;;GAGG;AACH,SAAS,oBAAoB,CAAC,IAAY;IACxC,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;IAC1C,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAEvB,oBAAoB;IACpB,MAAM,MAAM,GAAG,iBAAiB,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;IAC9C,IAAI,MAAM,KAAK,SAAS,EAAE,CAAC;QACzB,gCAAgC;QAChC,IAAI,GAAG,GAAG,MAAM,CAAC,SAAS,GAAG,wBAAwB,EAAE,CAAC;YACtD,OAAO,MAAM,CAAC,KAAK,CAAC;QACtB,CAAC;QACD,2BAA2B;QAC3B,iBAAiB,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;IACpC,CAAC;IAED,kDAAkD;IAClD,MAAM,MAAM,GAAG,mBAAmB,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IAEjD,+CAA+C;IAC/C,IAAI,iBAAiB,CAAC,IAAI,IAAI,0BAA0B,EAAE,CAAC;QACzD,6CAA6C;QAC7C,MAAM,YAAY,GAAG,IAAI,CAAC,IAAI,CAAC,0BAA0B,GAAG,GAAG,CAAC,CAAC;QACjE,MAAM,QAAQ,GAAG,iBAAiB,CAAC,IAAI,EAAE,CAAC;QAC1C,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,YAAY,EAAE,CAAC,EAAE,EAAE,CAAC;YACtC,MAAM,GAAG,GAAG,QAAQ,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC;YAClC,IAAI,GAAG,KAAK,SAAS,EAAE,CAAC;gBACtB,iBAAiB,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YAChC,CAAC;QACH,CAAC;IACH,CAAC;IACD,iBAAiB,CAAC,GAAG,CAAC,OAAO,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE,SAAS,EAAE,GAAG,EAAE,CAAC,CAAC;IAElE,OAAO,MAAM,CAAC;AAChB,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,cAAc,CAAC,IAAY;IACzC,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC;IAE5B,6BAA6B;IAC7B,IAAI,OAAO,CAAC,MAAM,GAAG,oBAAoB,EAAE,CAAC;QAC1C,iFAAiF;QACjF,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;YAC5B,OAAO,IAAI,CAAC;QACd,CAAC;IACH,CAAC;IAED,mBAAmB;IACnB,IAAI,WAAW,CAAC,OAAO,CAAC,EAAE,CAAC;QACzB,OAAO,IAAI,CAAC;IACd,CAAC;IAED,0DAA0D;IAC1D,IAAI,oBAAoB,CAAC,OAAO,CAAC,EAAE,CAAC;QAClC,OAAO,IAAI,CAAC;IACd,CAAC;IAED,OAAO,OAAO,CAAC;AACjB,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,YAAY,CAAC,IAAY;IACvC,IAAI,OAAO,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC;IAE1B,YAAY;IACZ,IAAI,OAAO,CAAC,MAAM,GAAG,kBAAkB,EAAE,CAAC;QACxC,OAAO,IAAI,CAAC;IACd,CAAC;IAED,uDAAuD;IACvD,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,wBAAwB,EAAE,IAAI,CAAC,CAAC;IAE1D,4DAA4D;IAC5D,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,yCAAyC,EAAE,EAAE,CAAC,CAAC;IAEzE,2DAA2D;IAC3D,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,WAAW,EAAE,EAAE,CAAC,CAAC;IAE3C,mBAAmB;IACnB,IAAI,WAAW,CAAC,OAAO,CAAC,EAAE,CAAC;QACzB,OAAO,IAAI,CAAC;IACd,CAAC;IAED,OAAO,OAAO,CAAC,IAAI,EAAE,CAAC;AACxB,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,cAAc,CAAC,KAAe;IAC5C,OAAO,KAAK;SACT,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC;SAC1B,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE;QACf,IAAI,IAAI,CAAC,MAAM,GAAG,oBAAoB;YAAE,OAAO,KAAK,CAAC;QACrD,IAAI,WAAW,CAAC,IAAI,CAAC;YAAE,OAAO,KAAK,CAAC;QACpC,OAAO,IAAI,CAAC;IACd,CAAC,CAAC,CAAC;AACP,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,cAAc,CAAC,IAAY;IACzC,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC;IAE5B,mBAAmB;IACnB,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACzB,OAAO,IAAI,CAAC;IACd,CAAC;IAED,qDAAqD;IACrD,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;QAC1D,OAAO,IAAI,CAAC;IACd,CAAC;IAED,OAAO,OAAO,CAAC;AACjB,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,kBAAkB,CAAC,IAAY;IAC7C,OAAO,IAAI,CAAC,OAAO,CAAC,wBAAwB,EAAE,IAAI,CAAC,CAAC;AACtD,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,sBAAsB,CAAC,IAAY;IACjD,OAAO,CACL,IAAI;QACF,yCAAyC;SACxC,OAAO,CACN,0EAA0E,EAC1E,EAAE,CACH;QACD,kCAAkC;SACjC,OAAO,CACN,6HAA6H,EAC7H,EAAE,CACH;QACD,0BAA0B;SACzB,OAAO,CACN,6EAA6E,EAC7E,EAAE,CACH;QACD,4BAA4B;SAC3B,OAAO,CAAC,SAAS,EAAE,GAAG,CAAC;SACvB,IAAI,EAAE,CACV,CAAC;AACJ,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"tool-error-handler.d.ts","sourceRoot":"","sources":["../../src/utils/tool-error-handler.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,oBAAoB,CAAC;
|
|
1
|
+
{"version":3,"file":"tool-error-handler.d.ts","sourceRoot":"","sources":["../../src/utils/tool-error-handler.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,oBAAoB,CAAC;AAgB5D,wBAAgB,uBAAuB,CACrC,OAAO,EAAE,MAAM,EACf,GAAG,EAAE,MAAM,EACX,IAAI,EAAE,MAAM,GACX,iBAAiB,CAOnB;AAED,wBAAgB,eAAe,CAC7B,KAAK,EAAE,OAAO,EACd,GAAG,EAAE,MAAM,EACX,eAAe,SAAqB,GACnC,iBAAiB,CA8CnB"}
|
|
@@ -1,5 +1,7 @@
|
|
|
1
|
-
import { AbortError, AppError, FetchError, RateLimitError, TimeoutError, UrlValidationError, } from '../errors/
|
|
2
|
-
|
|
1
|
+
import { AbortError, AppError, FetchError, RateLimitError, TimeoutError, UrlValidationError, } from '../errors/app-error.js';
|
|
2
|
+
// Stack traces only exposed when explicitly enabled in development
|
|
3
|
+
const isDevelopment = process.env.NODE_ENV === 'development' &&
|
|
4
|
+
process.env.EXPOSE_STACK_TRACES === 'true';
|
|
3
5
|
export function createToolErrorResponse(message, url, code) {
|
|
4
6
|
const structuredContent = { error: message, url, errorCode: code };
|
|
5
7
|
return {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"tool-error-handler.js","sourceRoot":"","sources":["../../src/utils/tool-error-handler.ts"],"names":[],"mappings":"AAEA,OAAO,EACL,UAAU,EACV,QAAQ,EACR,UAAU,EACV,cAAc,EACd,YAAY,EACZ,kBAAkB,GACnB,MAAM,
|
|
1
|
+
{"version":3,"file":"tool-error-handler.js","sourceRoot":"","sources":["../../src/utils/tool-error-handler.ts"],"names":[],"mappings":"AAEA,OAAO,EACL,UAAU,EACV,QAAQ,EACR,UAAU,EACV,cAAc,EACd,YAAY,EACZ,kBAAkB,GACnB,MAAM,wBAAwB,CAAC;AAEhC,mEAAmE;AACnE,MAAM,aAAa,GACjB,OAAO,CAAC,GAAG,CAAC,QAAQ,KAAK,aAAa;IACtC,OAAO,CAAC,GAAG,CAAC,mBAAmB,KAAK,MAAM,CAAC;AAE7C,MAAM,UAAU,uBAAuB,CACrC,OAAe,EACf,GAAW,EACX,IAAY;IAEZ,MAAM,iBAAiB,GAAG,EAAE,KAAK,EAAE,OAAO,EAAE,GAAG,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC;IACnE,OAAO;QACL,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,iBAAiB,CAAC,EAAE,CAAC;QACpE,iBAAiB;QACjB,OAAO,EAAE,IAAI;KACd,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,eAAe,CAC7B,KAAc,EACd,GAAW,EACX,eAAe,GAAG,kBAAkB;IAEpC,IAAI,KAAK,YAAY,kBAAkB,EAAE,CAAC;QACxC,MAAM,OAAO,GAAG,aAAa;YAC3B,CAAC,CAAC,GAAG,KAAK,CAAC,OAAO,UAAU,KAAK,CAAC,GAAG,YAAY,KAAK,CAAC,KAAK,IAAI,EAAE,EAAE;YACpE,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC;QAClB,OAAO,uBAAuB,CAAC,OAAO,EAAE,GAAG,EAAE,aAAa,CAAC,CAAC;IAC9D,CAAC;IACD,IAAI,KAAK,YAAY,UAAU,EAAE,CAAC;QAChC,MAAM,OAAO,GAAG,aAAa;YAC3B,CAAC,CAAC,kBAAkB,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,KAAK,CAAC,KAAK,IAAI,EAAE,EAAE;YACnF,CAAC,CAAC,kBAAkB,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC;QAChE,OAAO,uBAAuB,CAAC,OAAO,EAAE,GAAG,EAAE,SAAS,CAAC,CAAC;IAC1D,CAAC;IACD,IAAI,KAAK,YAAY,YAAY,EAAE,CAAC;QAClC,MAAM,OAAO,GAAG,aAAa;YAC3B,CAAC,CAAC,2BAA2B,KAAK,CAAC,SAAS,OAAO,KAAK,CAAC,KAAK,IAAI,EAAE,EAAE;YACtE,CAAC,CAAC,2BAA2B,KAAK,CAAC,SAAS,IAAI,CAAC;QACnD,OAAO,uBAAuB,CAAC,OAAO,EAAE,GAAG,EAAE,SAAS,CAAC,CAAC;IAC1D,CAAC;IACD,IAAI,KAAK,YAAY,cAAc,EAAE,CAAC;QACpC,MAAM,OAAO,GAAG,aAAa;YAC3B,CAAC,CAAC,6BAA6B,KAAK,CAAC,UAAU,MAAM,KAAK,CAAC,KAAK,IAAI,EAAE,EAAE;YACxE,CAAC,CAAC,6BAA6B,KAAK,CAAC,UAAU,GAAG,CAAC;QACrD,OAAO,uBAAuB,CAAC,OAAO,EAAE,GAAG,EAAE,cAAc,CAAC,CAAC;IAC/D,CAAC;IACD,IAAI,KAAK,YAAY,UAAU,EAAE,CAAC;QAChC,MAAM,IAAI,GAAG,KAAK,CAAC,UAAU,CAAC,CAAC,CAAC,QAAQ,KAAK,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC,aAAa,CAAC;QAC3E,MAAM,OAAO,GAAG,aAAa;YAC3B,CAAC,CAAC,GAAG,KAAK,CAAC,OAAO,KAAK,KAAK,CAAC,KAAK,IAAI,EAAE,EAAE;YAC1C,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC;QAClB,OAAO,uBAAuB,CAAC,OAAO,EAAE,GAAG,EAAE,IAAI,CAAC,CAAC;IACrD,CAAC;IACD,IAAI,KAAK,YAAY,QAAQ,EAAE,CAAC;QAC9B,MAAM,OAAO,GAAG,aAAa;YAC3B,CAAC,CAAC,GAAG,KAAK,CAAC,OAAO,KAAK,KAAK,CAAC,KAAK,IAAI,EAAE,EAAE;YAC1C,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC;QAClB,OAAO,uBAAuB,CAAC,OAAO,EAAE,GAAG,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC;IAC3D,CAAC;IAED,MAAM,OAAO,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe,CAAC;IACzE,MAAM,WAAW,GACf,aAAa,IAAI,KAAK,YAAY,KAAK;QACrC,CAAC,CAAC,GAAG,eAAe,KAAK,OAAO,KAAK,KAAK,CAAC,KAAK,IAAI,EAAE,EAAE;QACxD,CAAC,CAAC,GAAG,eAAe,KAAK,OAAO,EAAE,CAAC;IAEvC,OAAO,uBAAuB,CAAC,WAAW,EAAE,GAAG,EAAE,eAAe,CAAC,CAAC;AACpE,CAAC"}
|
|
@@ -1,3 +1,13 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Check if an IP address is in a blocked private range
|
|
3
|
+
*/
|
|
4
|
+
export declare function isBlockedIp(ip: string): boolean;
|
|
5
|
+
/**
|
|
6
|
+
* Validate resolved IP addresses to prevent DNS rebinding attacks.
|
|
7
|
+
* This should be called after DNS resolution to ensure the resolved
|
|
8
|
+
* IPs are not in blocked private ranges.
|
|
9
|
+
*/
|
|
10
|
+
export declare function validateResolvedIps(hostname: string): Promise<void>;
|
|
1
11
|
export declare function validateAndNormalizeUrl(urlString: string): string;
|
|
2
12
|
export declare function isInternalUrl(url: string, baseUrl: string): boolean;
|
|
3
13
|
//# sourceMappingURL=url-validator.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"url-validator.d.ts","sourceRoot":"","sources":["../../src/utils/url-validator.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"url-validator.d.ts","sourceRoot":"","sources":["../../src/utils/url-validator.ts"],"names":[],"mappings":"AAiCA;;GAEG;AACH,wBAAgB,WAAW,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAE/C;AAED;;;;GAIG;AACH,wBAAsB,mBAAmB,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAmCzE;AAED,wBAAgB,uBAAuB,CAAC,SAAS,EAAE,MAAM,GAAG,MAAM,CA2EjE;AAED,wBAAgB,aAAa,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,GAAG,OAAO,CAQnE"}
|
|
@@ -1,5 +1,6 @@
|
|
|
1
|
+
import dns from 'dns/promises';
|
|
2
|
+
import { config } from '../config/index.js';
|
|
1
3
|
import { UrlValidationError, ValidationError } from '../errors/app-error.js';
|
|
2
|
-
const MAX_URL_LENGTH = 2048;
|
|
3
4
|
const BLOCKED_HOSTS = new Set([
|
|
4
5
|
'localhost',
|
|
5
6
|
'127.0.0.1',
|
|
@@ -25,8 +26,45 @@ const BLOCKED_IP_PATTERNS = [
|
|
|
25
26
|
/^::ffff:172\.(1[6-9]|2\d|3[01])\./,
|
|
26
27
|
/^::ffff:192\.168\./,
|
|
27
28
|
];
|
|
28
|
-
|
|
29
|
-
|
|
29
|
+
/**
|
|
30
|
+
* Check if an IP address is in a blocked private range
|
|
31
|
+
*/
|
|
32
|
+
export function isBlockedIp(ip) {
|
|
33
|
+
return BLOCKED_IP_PATTERNS.some((pattern) => pattern.test(ip));
|
|
34
|
+
}
|
|
35
|
+
/**
|
|
36
|
+
* Validate resolved IP addresses to prevent DNS rebinding attacks.
|
|
37
|
+
* This should be called after DNS resolution to ensure the resolved
|
|
38
|
+
* IPs are not in blocked private ranges.
|
|
39
|
+
*/
|
|
40
|
+
export async function validateResolvedIps(hostname) {
|
|
41
|
+
// Skip validation for direct IP addresses (already validated in validateAndNormalizeUrl)
|
|
42
|
+
if (/^[\d.]+$/.test(hostname) || hostname.includes(':')) {
|
|
43
|
+
return;
|
|
44
|
+
}
|
|
45
|
+
try {
|
|
46
|
+
// Resolve IPv4 addresses
|
|
47
|
+
const ipv4Addresses = await dns.resolve4(hostname).catch(() => []);
|
|
48
|
+
for (const ip of ipv4Addresses) {
|
|
49
|
+
if (isBlockedIp(ip) || BLOCKED_HOSTS.has(ip)) {
|
|
50
|
+
throw new UrlValidationError(`DNS rebinding detected: ${hostname} resolves to blocked IP ${ip}`, hostname);
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
// Resolve IPv6 addresses
|
|
54
|
+
const ipv6Addresses = await dns.resolve6(hostname).catch(() => []);
|
|
55
|
+
for (const ip of ipv6Addresses) {
|
|
56
|
+
if (isBlockedIp(ip) || BLOCKED_HOSTS.has(ip)) {
|
|
57
|
+
throw new UrlValidationError(`DNS rebinding detected: ${hostname} resolves to blocked IP ${ip}`, hostname);
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
}
|
|
61
|
+
catch (error) {
|
|
62
|
+
// Re-throw UrlValidationError, ignore DNS resolution errors
|
|
63
|
+
if (error instanceof UrlValidationError) {
|
|
64
|
+
throw error;
|
|
65
|
+
}
|
|
66
|
+
// DNS resolution failed - let the actual request handle the error
|
|
67
|
+
}
|
|
30
68
|
}
|
|
31
69
|
export function validateAndNormalizeUrl(urlString) {
|
|
32
70
|
// Check for empty or whitespace-only input
|
|
@@ -38,8 +76,8 @@ export function validateAndNormalizeUrl(urlString) {
|
|
|
38
76
|
throw new ValidationError('URL cannot be empty');
|
|
39
77
|
}
|
|
40
78
|
// Check URL length to prevent DoS
|
|
41
|
-
if (trimmedUrl.length >
|
|
42
|
-
throw new ValidationError(`URL exceeds maximum length of ${
|
|
79
|
+
if (trimmedUrl.length > config.constants.maxUrlLength) {
|
|
80
|
+
throw new ValidationError(`URL exceeds maximum length of ${config.constants.maxUrlLength} characters`, { length: trimmedUrl.length, maxLength: config.constants.maxUrlLength });
|
|
43
81
|
}
|
|
44
82
|
let url;
|
|
45
83
|
try {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"url-validator.js","sourceRoot":"","sources":["../../src/utils/url-validator.ts"],"names":[],"mappings":"AAAA,OAAO,
|
|
1
|
+
{"version":3,"file":"url-validator.js","sourceRoot":"","sources":["../../src/utils/url-validator.ts"],"names":[],"mappings":"AAAA,OAAO,GAAG,MAAM,cAAc,CAAC;AAE/B,OAAO,EAAE,MAAM,EAAE,MAAM,oBAAoB,CAAC;AAE5C,OAAO,EAAE,kBAAkB,EAAE,eAAe,EAAE,MAAM,wBAAwB,CAAC;AAE7E,MAAM,aAAa,GAAG,IAAI,GAAG,CAAC;IAC5B,WAAW;IACX,WAAW;IACX,SAAS;IACT,KAAK;IACL,iBAAiB;IACjB,0BAA0B;IAC1B,oBAAoB;IACpB,iBAAiB;IACjB,eAAe;CAChB,CAAC,CAAC;AAEH,MAAM,mBAAmB,GAAsB;IAC7C,OAAO;IACP,4BAA4B;IAC5B,aAAa;IACb,QAAQ;IACR,MAAM;IACN,aAAa;IACb,SAAS;IACT,SAAS;IACT,eAAe;IACf,cAAc;IACd,mCAAmC;IACnC,oBAAoB;CACrB,CAAC;AAEF;;GAEG;AACH,MAAM,UAAU,WAAW,CAAC,EAAU;IACpC,OAAO,mBAAmB,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC;AACjE,CAAC;AAED;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,mBAAmB,CAAC,QAAgB;IACxD,yFAAyF;IACzF,IAAI,UAAU,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,QAAQ,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;QACxD,OAAO;IACT,CAAC;IAED,IAAI,CAAC;QACH,yBAAyB;QACzB,MAAM,aAAa,GAAG,MAAM,GAAG,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,EAAE,CAAC,CAAC;QACnE,KAAK,MAAM,EAAE,IAAI,aAAa,EAAE,CAAC;YAC/B,IAAI,WAAW,CAAC,EAAE,CAAC,IAAI,aAAa,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,CAAC;gBAC7C,MAAM,IAAI,kBAAkB,CAC1B,2BAA2B,QAAQ,2BAA2B,EAAE,EAAE,EAClE,QAAQ,CACT,CAAC;YACJ,CAAC;QACH,CAAC;QAED,yBAAyB;QACzB,MAAM,aAAa,GAAG,MAAM,GAAG,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,EAAE,CAAC,CAAC;QACnE,KAAK,MAAM,EAAE,IAAI,aAAa,EAAE,CAAC;YAC/B,IAAI,WAAW,CAAC,EAAE,CAAC,IAAI,aAAa,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,CAAC;gBAC7C,MAAM,IAAI,kBAAkB,CAC1B,2BAA2B,QAAQ,2BAA2B,EAAE,EAAE,EAClE,QAAQ,CACT,CAAC;YACJ,CAAC;QACH,CAAC;IACH,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,4DAA4D;QAC5D,IAAI,KAAK,YAAY,kBAAkB,EAAE,CAAC;YACxC,MAAM,KAAK,CAAC;QACd,CAAC;QACD,kEAAkE;IACpE,CAAC;AACH,CAAC;AAED,MAAM,UAAU,uBAAuB,CAAC,SAAiB;IACvD,2CAA2C;IAC3C,IAAI,CAAC,SAAS,IAAI,OAAO,SAAS,KAAK,QAAQ,EAAE,CAAC;QAChD,MAAM,IAAI,eAAe,CAAC,iBAAiB,CAAC,CAAC;IAC/C,CAAC;IAED,MAAM,UAAU,GAAG,SAAS,CAAC,IAAI,EAAE,CAAC;IACpC,IAAI,CAAC,UAAU,EAAE,CAAC;QAChB,MAAM,IAAI,eAAe,CAAC,qBAAqB,CAAC,CAAC;IACnD,CAAC;IAED,kCAAkC;IAClC,IAAI,UAAU,CAAC,MAAM,GAAG,MAAM,CAAC,SAAS,CAAC,YAAY,EAAE,CAAC;QACtD,MAAM,IAAI,eAAe,CACvB,iCAAiC,MAAM,CAAC,SAAS,CAAC,YAAY,aAAa,EAC3E,EAAE,MAAM,EAAE,UAAU,CAAC,MAAM,EAAE,SAAS,EAAE,MAAM,CAAC,SAAS,CAAC,YAAY,EAAE,CACxE,CAAC;IACJ,CAAC;IAED,IAAI,GAAQ,CAAC;IAEb,IAAI,CAAC;QACH,GAAG,GAAG,IAAI,GAAG,CAAC,UAAU,CAAC,CAAC;IAC5B,CAAC;IAAC,MAAM,CAAC;QACP,MAAM,IAAI,kBAAkB,CAAC,oBAAoB,EAAE,UAAU,CAAC,CAAC;IACjE,CAAC;IAED,+BAA+B;IAC/B,IAAI,GAAG,CAAC,QAAQ,KAAK,OAAO,IAAI,GAAG,CAAC,QAAQ,KAAK,QAAQ,EAAE,CAAC;QAC1D,MAAM,IAAI,kBAAkB,CAC1B,qBAAqB,GAAG,CAAC,QAAQ,qCAAqC,EACtE,UAAU,CACX,CAAC;IACJ,CAAC;IAED,+CAA+C;IAC/C,IAAI,GAAG,CAAC,QAAQ,IAAI,GAAG,CAAC,QAAQ,EAAE,CAAC;QACjC,MAAM,IAAI,kBAAkB,CAC1B,gDAAgD,EAChD,UAAU,CACX,CAAC;IACJ,CAAC;IAED,MAAM,QAAQ,GAAG,GAAG,CAAC,QAAQ,CAAC,WAAW,EAAE,CAAC;IAE5C,uBAAuB;IACvB,IAAI,CAAC,QAAQ,EAAE,CAAC;QACd,MAAM,IAAI,kBAAkB,CAAC,gCAAgC,EAAE,UAAU,CAAC,CAAC;IAC7E,CAAC;IAED,sCAAsC;IACtC,IAAI,aAAa,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC;QAChC,MAAM,IAAI,kBAAkB,CAC1B,iBAAiB,QAAQ,kCAAkC,EAC3D,UAAU,CACX,CAAC;IACJ,CAAC;IAED,0BAA0B;IAC1B,IAAI,WAAW,CAAC,QAAQ,CAAC,EAAE,CAAC;QAC1B,MAAM,IAAI,kBAAkB,CAC1B,qBAAqB,QAAQ,+BAA+B,EAC5D,UAAU,CACX,CAAC;IACJ,CAAC;IAED,0EAA0E;IAC1E,IAAI,QAAQ,CAAC,QAAQ,CAAC,QAAQ,CAAC,IAAI,QAAQ,CAAC,QAAQ,CAAC,WAAW,CAAC,EAAE,CAAC;QAClE,MAAM,IAAI,kBAAkB,CAC1B,6BAA6B,QAAQ,4CAA4C,EACjF,UAAU,CACX,CAAC;IACJ,CAAC;IAED,OAAO,GAAG,CAAC,IAAI,CAAC;AAClB,CAAC;AAED,MAAM,UAAU,aAAa,CAAC,GAAW,EAAE,OAAe;IACxD,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;QACrC,MAAM,UAAU,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,CAAC;QACpC,OAAO,MAAM,CAAC,QAAQ,KAAK,UAAU,CAAC,QAAQ,CAAC;IACjD,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,KAAK,CAAC;IACf,CAAC;AACH,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@j0hanz/superfetch",
|
|
3
|
-
"version": "1.0.
|
|
3
|
+
"version": "1.0.6",
|
|
4
4
|
"mcpName": "io.github.j0hanz/superfetch",
|
|
5
5
|
"description": "Intelligent web content fetcher MCP server that converts HTML to clean, AI-readable JSONL format",
|
|
6
6
|
"type": "module",
|
|
@@ -14,7 +14,7 @@
|
|
|
14
14
|
],
|
|
15
15
|
"repository": {
|
|
16
16
|
"type": "git",
|
|
17
|
-
"url": "
|
|
17
|
+
"url": "https://github.com/j0hanz/super-fetch-mcp-server.git"
|
|
18
18
|
},
|
|
19
19
|
"homepage": "https://github.com/j0hanz/super-fetch-mcp-server#readme",
|
|
20
20
|
"bugs": {
|
|
@@ -29,7 +29,9 @@
|
|
|
29
29
|
"content-extraction",
|
|
30
30
|
"readability",
|
|
31
31
|
"jsonl",
|
|
32
|
-
"ai-tools"
|
|
32
|
+
"ai-tools",
|
|
33
|
+
"model-context-protocol",
|
|
34
|
+
"superfetch"
|
|
33
35
|
],
|
|
34
36
|
"scripts": {
|
|
35
37
|
"dev": "tsx watch src/index.ts",
|
|
@@ -46,7 +48,7 @@
|
|
|
46
48
|
"knip:fix": "knip --fix"
|
|
47
49
|
},
|
|
48
50
|
"dependencies": {
|
|
49
|
-
"@modelcontextprotocol/sdk": "^1.
|
|
51
|
+
"@modelcontextprotocol/sdk": "^1.25.1",
|
|
50
52
|
"@mozilla/readability": "^0.6.0",
|
|
51
53
|
"axios": "^1.13.2",
|
|
52
54
|
"cheerio": "^1.1.2",
|
|
@@ -68,12 +70,12 @@
|
|
|
68
70
|
"eslint": "^9.23.2",
|
|
69
71
|
"eslint-config-prettier": "^10.1.8",
|
|
70
72
|
"eslint-plugin-unused-imports": "^4.3.0",
|
|
71
|
-
"knip": "^5.
|
|
73
|
+
"knip": "^5.74.0",
|
|
72
74
|
"prettier": "^3.7.4",
|
|
73
75
|
"shx": "^0.4.0",
|
|
74
76
|
"tsx": "^4.21.0",
|
|
75
77
|
"typescript": "^5.9.3",
|
|
76
|
-
"typescript-eslint": "^8.
|
|
78
|
+
"typescript-eslint": "^8.50.0"
|
|
77
79
|
},
|
|
78
80
|
"engines": {
|
|
79
81
|
"node": ">=20.0.0"
|