@j0hanz/superfetch 1.1.0 → 1.1.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +40 -9
- package/dist/config/index.d.ts +11 -1
- package/dist/config/index.d.ts.map +1 -1
- package/dist/config/index.js +44 -2
- package/dist/config/index.js.map +1 -1
- package/dist/config/types.d.ts +1 -0
- package/dist/config/types.d.ts.map +1 -1
- package/dist/index.js +122 -4
- package/dist/index.js.map +1 -1
- package/dist/middleware/error-handler.js +1 -1
- package/dist/middleware/error-handler.js.map +1 -1
- package/dist/services/cache.d.ts.map +1 -1
- package/dist/services/cache.js +7 -3
- package/dist/services/cache.js.map +1 -1
- package/dist/services/context.d.ts +9 -0
- package/dist/services/context.d.ts.map +1 -0
- package/dist/services/context.js +9 -0
- package/dist/services/context.js.map +1 -0
- package/dist/services/fetcher.d.ts.map +1 -1
- package/dist/services/fetcher.js +250 -175
- package/dist/services/fetcher.js.map +1 -1
- package/dist/services/logger.d.ts.map +1 -1
- package/dist/services/logger.js +10 -1
- package/dist/services/logger.js.map +1 -1
- package/dist/tools/handlers/fetch-url.tool.d.ts.map +1 -1
- package/dist/tools/handlers/fetch-url.tool.js +1 -1
- package/dist/tools/handlers/fetch-url.tool.js.map +1 -1
- package/dist/tools/handlers/fetch-urls.tool.d.ts.map +1 -1
- package/dist/tools/handlers/fetch-urls.tool.js +6 -1
- package/dist/tools/handlers/fetch-urls.tool.js.map +1 -1
- package/dist/utils/url-validator.d.ts +4 -1
- package/dist/utils/url-validator.d.ts.map +1 -1
- package/dist/utils/url-validator.js +3 -53
- package/dist/utils/url-validator.js.map +1 -1
- package/package.json +4 -4
|
@@ -89,7 +89,7 @@ async function processSingleUrl(url, options) {
|
|
|
89
89
|
const cachedResult = attemptCacheRetrievalForUrl(normalizedUrl, options.format);
|
|
90
90
|
if (cachedResult)
|
|
91
91
|
return cachedResult;
|
|
92
|
-
const html = await fetchUrlWithRetry(normalizedUrl);
|
|
92
|
+
const html = await fetchUrlWithRetry(normalizedUrl, options.requestOptions, options.maxRetries);
|
|
93
93
|
const { sourceHtml, title, metadata } = processContentExtraction(html, normalizedUrl, options);
|
|
94
94
|
const { content, contentBlocks } = transformContentForFormat(sourceHtml, normalizedUrl, metadata, options.format);
|
|
95
95
|
const { content: finalContent } = enforceContentLengthLimit(content, options.maxContentLength);
|
|
@@ -173,6 +173,11 @@ export async function fetchUrlsToolHandler(input) {
|
|
|
173
173
|
includeMetadata: input.includeMetadata ?? true,
|
|
174
174
|
maxContentLength: input.maxContentLength,
|
|
175
175
|
format,
|
|
176
|
+
requestOptions: {
|
|
177
|
+
customHeaders: input.customHeaders,
|
|
178
|
+
timeout: input.timeout,
|
|
179
|
+
},
|
|
180
|
+
maxRetries: input.retries,
|
|
176
181
|
};
|
|
177
182
|
// Process URLs in batches using native Promise.allSettled
|
|
178
183
|
const results = [];
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"fetch-urls.tool.js","sourceRoot":"","sources":["../../../src/tools/handlers/fetch-urls.tool.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"fetch-urls.tool.js","sourceRoot":"","sources":["../../../src/tools/handlers/fetch-urls.tool.ts"],"names":[],"mappings":"AAWA,OAAO,KAAK,KAAK,MAAM,yBAAyB,CAAC;AACjD,OAAO,EAAE,cAAc,EAAE,MAAM,6BAA6B,CAAC;AAC7D,OAAO,EAAE,iBAAiB,EAAE,MAAM,2BAA2B,CAAC;AAC9D,OAAO,EAAE,QAAQ,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,0BAA0B,CAAC;AACvE,OAAO,EAAE,SAAS,EAAE,MAAM,0BAA0B,CAAC;AAErD,OAAO,EAAE,uBAAuB,EAAE,MAAM,mCAAmC,CAAC;AAC5E,OAAO,EAAE,uBAAuB,EAAE,MAAM,8BAA8B,CAAC;AACvE,OAAO,EACL,0BAA0B,EAC1B,gCAAgC,EAChC,yBAAyB,GAC1B,MAAM,oBAAoB,CAAC;AAE5B,OAAO,EAAE,OAAO,EAAE,MAAM,yCAAyC,CAAC;AAClE,OAAO,EAAE,cAAc,EAAE,MAAM,4CAA4C,CAAC;AAE5E,MAAM,kBAAkB,GAAG,EAAE,CAAC;AAC9B,MAAM,mBAAmB,GAAG,CAAC,CAAC;AAC9B,MAAM,eAAe,GAAG,CAAC,CAAC;AAE1B,MAAM,CAAC,MAAM,oBAAoB,GAAG,YAAY,CAAC;AACjD,MAAM,CAAC,MAAM,2BAA2B,GACtC,mKAAmK,CAAC;AAEtK,SAAS,mBAAmB,CAC1B,OAAyB;IAEzB,MAAM,OAAO,GAAiB;QAC5B,KAAK,EAAE,OAAO,CAAC,MAAM;QACrB,UAAU,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,MAAM;QAC7D,MAAM,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,MAAM;QAC1D,MAAM,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,MAAM;QACxD,kBAAkB,EAAE,OAAO,CAAC,MAAM,CAChC,CAAC,GAAG,EAAE,MAAM,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,MAAM,CAAC,aAAa,IAAI,CAAC,CAAC,EAClD,CAAC,CACF;KACF,CAAC;IAEF,MAAM,iBAAiB,GAAyB;QAC9C,OAAO;QACP,OAAO;QACP,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;KACpC,CAAC;IAEF,OAAO;QACL,OAAO,EAAE;YACP;gBACE,IAAI,EAAE,MAAe;gBACrB,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,iBAAiB,EAAE,IAAI,EAAE,CAAC,CAAC;aACjD;SACF;QACD,iBAAiB;KAClB,CAAC;AACJ,CAAC;AAsBD,SAAS,2BAA2B,CAClC,aAAqB,EACrB,MAA4B;IAE5B,MAAM,cAAc,GAAG,MAAM,KAAK,UAAU,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,KAAK,CAAC;IAClE,MAAM,QAAQ,GAAG,KAAK,CAAC,cAAc,CAAC,cAAc,EAAE,aAAa,CAAC,CAAC;IAErE,IAAI,CAAC,QAAQ;QAAE,OAAO,IAAI,CAAC;IAE3B,MAAM,MAAM,GAAG,KAAK,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;IACnC,IAAI,CAAC,MAAM;QAAE,OAAO,IAAI,CAAC;IAEzB,QAAQ,CAAC,iBAAiB,EAAE,EAAE,GAAG,EAAE,aAAa,EAAE,CAAC,CAAC;IACpD,OAAO;QACL,GAAG,EAAE,aAAa;QAClB,OAAO,EAAE,IAAI;QACb,OAAO,EAAE,MAAM,CAAC,OAAO;QACvB,MAAM,EAAE,IAAI;KACb,CAAC;AACJ,CAAC;AAED,SAAS,yBAAyB,CAChC,IAAY,EACZ,aAAqB,EACrB,QAAmC,EACnC,MAA4B;IAE5B,IAAI,MAAM,KAAK,UAAU,EAAE,CAAC;QAC1B,OAAO;YACL,OAAO,EAAE,cAAc,CAAC,IAAI,EAAE,QAAQ,CAAC;SACxC,CAAC;IACJ,CAAC;IAED,MAAM,MAAM,GAAG,SAAS,CAAC,IAAI,CAAC,CAAC;IAC/B,OAAO;QACL,OAAO,EAAE,OAAO,CAAC,MAAM,EAAE,QAAQ,CAAC;QAClC,aAAa,EAAE,MAAM,CAAC,MAAM;KAC7B,CAAC;AACJ,CAAC;AAED,SAAS,wBAAwB,CAC/B,IAAY,EACZ,aAAqB,EACrB,OAAgC;IAMhC,IAAI,CAAC,OAAO,CAAC,kBAAkB,EAAE,CAAC;QAChC,MAAM,EAAE,QAAQ,EAAE,aAAa,EAAE,GAAG,cAAc,CAAC,IAAI,EAAE,aAAa,EAAE;YACtE,cAAc,EAAE,KAAK;SACtB,CAAC,CAAC;QACH,OAAO;YACL,UAAU,EAAE,IAAI;YAChB,KAAK,EAAE,aAAa,CAAC,KAAK;YAC1B,QAAQ,EAAE,0BAA0B,CAClC,aAAa,EACb,IAAI,EACJ,aAAa,EACb,KAAK,EACL,OAAO,CAAC,eAAe,CACxB;SACF,CAAC;IACJ,CAAC;IAED,MAAM,EAAE,OAAO,EAAE,QAAQ,EAAE,aAAa,EAAE,GAAG,cAAc,CACzD,IAAI,EACJ,aAAa,EACb,EAAE,cAAc,EAAE,IAAI,EAAE,CACzB,CAAC;IAEF,MAAM,wBAAwB,GAAG,gCAAgC,CAC/D,IAAI,EACJ,OAAO,CACR,CAAC;IAEF,OAAO;QACL,UAAU,EAAE,wBAAwB,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI;QAC7D,KAAK,EAAE,wBAAwB,CAAC,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,aAAa,CAAC,KAAK;QACrE,QAAQ,EAAE,0BAA0B,CAClC,aAAa,EACb,OAAO,EACP,aAAa,EACb,wBAAwB,EACxB,OAAO,CAAC,eAAe,CACxB;KACF,CAAC;AACJ,CAAC;AAED,KAAK,UAAU,gBAAgB,CAC7B,GAAW,EACX,OAAgC;IAEhC,IAAI,CAAC;QACH,MAAM,aAAa,GAAG,uBAAuB,CAAC,GAAG,CAAC,CAAC;QAEnD,MAAM,YAAY,GAAG,2BAA2B,CAC9C,aAAa,EACb,OAAO,CAAC,MAAM,CACf,CAAC;QACF,IAAI,YAAY;YAAE,OAAO,YAAY,CAAC;QAEtC,MAAM,IAAI,GAAG,MAAM,iBAAiB,CAClC,aAAa,EACb,OAAO,CAAC,cAAc,EACtB,OAAO,CAAC,UAAU,CACnB,CAAC;QAEF,MAAM,EAAE,UAAU,EAAE,KAAK,EAAE,QAAQ,EAAE,GAAG,wBAAwB,CAC9D,IAAI,EACJ,aAAa,EACb,OAAO,CACR,CAAC;QAEF,MAAM,EAAE,OAAO,EAAE,aAAa,EAAE,GAAG,yBAAyB,CAC1D,UAAU,EACV,aAAa,EACb,QAAQ,EACR,OAAO,CAAC,MAAM,CACf,CAAC;QAEF,MAAM,EAAE,OAAO,EAAE,YAAY,EAAE,GAAG,yBAAyB,CACzD,OAAO,EACP,OAAO,CAAC,gBAAgB,CACzB,CAAC;QAEF,MAAM,cAAc,GAAG,OAAO,CAAC,MAAM,KAAK,UAAU,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,KAAK,CAAC;QAC1E,MAAM,QAAQ,GAAG,KAAK,CAAC,cAAc,CAAC,cAAc,EAAE,aAAa,CAAC,CAAC;QACrE,IAAI,QAAQ,EAAE,CAAC;YACb,KAAK,CAAC,GAAG,CAAC,QAAQ,EAAE,YAAY,CAAC,CAAC;QACpC,CAAC;QAED,OAAO;YACL,GAAG,EAAE,aAAa;YAClB,OAAO,EAAE,IAAI;YACb,KAAK;YACL,OAAO,EAAE,YAAY;YACrB,aAAa;YACb,MAAM,EAAE,KAAK;SACd,CAAC;IACJ,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,YAAY,GAChB,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe,CAAC;QAC3D,MAAM,SAAS,GACb,KAAK,YAAY,KAAK;YACtB,MAAM,IAAI,KAAK;YACf,OAAO,KAAK,CAAC,IAAI,KAAK,QAAQ;YAC5B,CAAC,CAAC,KAAK,CAAC,IAAI;YACZ,CAAC,CAAC,aAAa,CAAC;QAEpB,OAAO,CAAC,6BAA6B,EAAE,EAAE,GAAG,EAAE,KAAK,EAAE,YAAY,EAAE,CAAC,CAAC;QAErE,OAAO;YACL,GAAG;YACH,OAAO,EAAE,KAAK;YACd,MAAM,EAAE,KAAK;YACb,KAAK,EAAE,YAAY;YACnB,SAAS;SACV,CAAC;IACJ,CAAC;AACH,CAAC;AAED,SAAS,uBAAuB,CAAC,EAAE,MAAM,EAAyB;IAChE,IAAI,MAAM,YAAY,KAAK,EAAE,CAAC;QAC5B,OAAO,MAAM,CAAC,OAAO,CAAC;IACxB,CAAC;IACD,IAAI,OAAO,MAAM,KAAK,QAAQ,EAAE,CAAC;QAC/B,OAAO,MAAM,CAAC;IAChB,CAAC;IACD,IACE,MAAM;QACN,OAAO,MAAM,KAAK,QAAQ;QAC1B,SAAS,IAAI,MAAM;QACnB,OAAQ,MAAkC,CAAC,OAAO,KAAK,QAAQ,EAC/D,CAAC;QACD,MAAM,GAAG,GAAI,MAAkC,CAAC,OAAO,CAAC;QACxD,OAAO,GAAa,CAAC;IACvB,CAAC;IACD,OAAO,eAAe,CAAC;AACzB,CAAC;AAED,SAAS,kBAAkB,CACzB,KAAqB;IAErB,IAAI,KAAK,CAAC,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC5B,OAAO,uBAAuB,CAC5B,8BAA8B,EAC9B,EAAE,EACF,kBAAkB,CACnB,CAAC;IACJ,CAAC;IAED,IAAI,KAAK,CAAC,IAAI,CAAC,MAAM,GAAG,kBAAkB,EAAE,CAAC;QAC3C,OAAO,uBAAuB,CAC5B,WAAW,kBAAkB,yBAAyB,EACtD,EAAE,EACF,kBAAkB,CACnB,CAAC;IACJ,CAAC;IAED,MAAM,SAAS,GAAG,KAAK,CAAC,IAAI,CAAC,MAAM,CACjC,CAAC,GAAG,EAAE,EAAE,CAAC,OAAO,GAAG,KAAK,QAAQ,IAAI,GAAG,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,CAC1D,CAAC;IAEF,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC3B,OAAO,uBAAuB,CAC5B,wBAAwB,EACxB,EAAE,EACF,kBAAkB,CACnB,CAAC;IACJ,CAAC;IAED,OAAO,SAAS,CAAC;AACnB,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,oBAAoB,CACxC,KAAqB;IAErB,IAAI,CAAC;QACH,MAAM,gBAAgB,GAAG,kBAAkB,CAAC,KAAK,CAAC,CAAC;QACnD,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,gBAAgB,CAAC,EAAE,CAAC;YACrC,OAAO,gBAAgB,CAAC;QAC1B,CAAC;QAED,MAAM,SAAS,GAAG,gBAAgB,CAAC;QACnC,MAAM,WAAW,GAAG,IAAI,CAAC,GAAG,CAC1B,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,CAAC,WAAW,IAAI,mBAAmB,CAAC,EACrD,eAAe,CAChB,CAAC;QACF,MAAM,eAAe,GAAG,KAAK,CAAC,eAAe,IAAI,IAAI,CAAC;QACtD,MAAM,MAAM,GAAG,KAAK,CAAC,MAAM,IAAI,OAAO,CAAC;QAEvC,QAAQ,CAAC,0BAA0B,EAAE;YACnC,QAAQ,EAAE,SAAS,CAAC,MAAM;YAC1B,WAAW;YACX,MAAM;SACP,CAAC,CAAC;QAEH,MAAM,cAAc,GAA4B;YAC9C,kBAAkB,EAAE,KAAK,CAAC,kBAAkB,IAAI,IAAI;YACpD,eAAe,EAAE,KAAK,CAAC,eAAe,IAAI,IAAI;YAC9C,gBAAgB,EAAE,KAAK,CAAC,gBAAgB;YACxC,MAAM;YACN,cAAc,EAAE;gBACd,aAAa,EAAE,KAAK,CAAC,aAAa;gBAClC,OAAO,EAAE,KAAK,CAAC,OAAO;aACvB;YACD,UAAU,EAAE,KAAK,CAAC,OAAO;SAC1B,CAAC;QAEF,0DAA0D;QAC1D,MAAM,OAAO,GAAqB,EAAE,CAAC;QACrC,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,CAAC,WAAW,EAAE,SAAS,CAAC,MAAM,CAAC,CAAC;QAE1D,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,SAAS,CAAC,MAAM,EAAE,CAAC,IAAI,SAAS,EAAE,CAAC;YACrD,MAAM,KAAK,GAAG,SAAS,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,SAAS,CAAC,CAAC;YAChD,MAAM,UAAU,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CACnC,gBAAgB,CAAC,GAAG,EAAE,cAAc,CAAC,CACtC,CAAC;YAEF,QAAQ,CAAC,kBAAkB,EAAE;gBAC3B,KAAK,EAAE,CAAC,GAAG,SAAS,GAAG,CAAC;gBACxB,IAAI,EAAE,KAAK,CAAC,MAAM;gBAClB,KAAK,EAAE,SAAS,CAAC,MAAM;aACxB,CAAC,CAAC;YAEH,MAAM,cAAc,GAAG,MAAM,OAAO,CAAC,UAAU,CAAC,UAAU,CAAC,CAAC;YAE5D,MAAM,YAAY,GAAqB,cAAc,CAAC,GAAG,CACvD,CAAC,MAAM,EAAE,KAAK,EAAE,EAAE;gBAChB,IAAI,MAAM,CAAC,MAAM,KAAK,WAAW,EAAE,CAAC;oBAClC,OAAO,MAAM,CAAC,KAAK,CAAC;gBACtB,CAAC;gBACD,OAAO;oBACL,GAAG,EAAE,KAAK,CAAC,KAAK,CAAC,IAAI,SAAS;oBAC9B,OAAO,EAAE,KAAc;oBACvB,MAAM,EAAE,KAAc;oBACtB,KAAK,EAAE,uBAAuB,CAAC,MAAM,CAAC;oBACtC,SAAS,EAAE,kBAAkB;iBAC9B,CAAC;YACJ,CAAC,CACF,CAAC;YAEF,OAAO,CAAC,IAAI,CAAC,GAAG,YAAY,CAAC,CAAC;QAChC,CAAC;QAED,IAAI,CAAC,eAAe,EAAE,CAAC;YACrB,MAAM,UAAU,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;YAC7D,IAAI,UAAU,IAAI,CAAC,UAAU,CAAC,OAAO,EAAE,CAAC;gBACtC,MAAM,QAAQ,GAAG,UAAU,CAAC,KAAK,IAAI,eAAe,CAAC;gBACrD,OAAO,uBAAuB,CAC5B,iBAAiB,QAAQ,EAAE,EAC3B,UAAU,CAAC,GAAG,EACd,UAAU,CAAC,SAAS,IAAI,aAAa,CACtC,CAAC;YACJ,CAAC;QACH,CAAC;QAED,OAAO,mBAAmB,CAAC,OAAO,CAAC,CAAC;IACtC,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,QAAQ,CACN,uBAAuB,EACvB,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS,CAC3C,CAAC;QAEF,OAAO,uBAAuB,CAC5B,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,sBAAsB,EAC/D,EAAE,EACF,aAAa,CACd,CAAC;IACJ,CAAC;AACH,CAAC"}
|
|
@@ -1,4 +1,7 @@
|
|
|
1
|
-
|
|
1
|
+
/**
|
|
2
|
+
* Check if an IP address is in a blocked private range
|
|
3
|
+
*/
|
|
4
|
+
export declare function isBlockedIp(ip: string): boolean;
|
|
2
5
|
export declare function validateAndNormalizeUrl(urlString: string): string;
|
|
3
6
|
export declare function isInternalUrl(url: string, baseUrl: string): boolean;
|
|
4
7
|
//# sourceMappingURL=url-validator.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"url-validator.d.ts","sourceRoot":"","sources":["../../src/utils/url-validator.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"url-validator.d.ts","sourceRoot":"","sources":["../../src/utils/url-validator.ts"],"names":[],"mappings":"AAEA;;GAEG;AACH,wBAAgB,WAAW,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAE/C;AAED,wBAAgB,uBAAuB,CAAC,SAAS,EAAE,MAAM,GAAG,MAAM,CA2DjE;AAED,wBAAgB,aAAa,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,GAAG,OAAO,CAQnE"}
|
|
@@ -1,59 +1,9 @@
|
|
|
1
|
-
import dns from 'dns/promises';
|
|
2
1
|
import { config } from '../config/index.js';
|
|
3
|
-
const BLOCKED_HOSTS = new Set([
|
|
4
|
-
'localhost',
|
|
5
|
-
'127.0.0.1',
|
|
6
|
-
'0.0.0.0',
|
|
7
|
-
'::1',
|
|
8
|
-
'169.254.169.254',
|
|
9
|
-
'metadata.google.internal',
|
|
10
|
-
'metadata.azure.com',
|
|
11
|
-
'100.100.100.200',
|
|
12
|
-
'instance-data',
|
|
13
|
-
]);
|
|
14
|
-
const BLOCKED_IP_PATTERNS = [
|
|
15
|
-
/^10\./,
|
|
16
|
-
/^172\.(1[6-9]|2\d|3[01])\./,
|
|
17
|
-
/^192\.168\./,
|
|
18
|
-
/^127\./,
|
|
19
|
-
/^0\./,
|
|
20
|
-
/^169\.254\./,
|
|
21
|
-
/^fc00:/i,
|
|
22
|
-
/^fe80:/i,
|
|
23
|
-
/^::ffff:127\./,
|
|
24
|
-
/^::ffff:10\./,
|
|
25
|
-
/^::ffff:172\.(1[6-9]|2\d|3[01])\./,
|
|
26
|
-
/^::ffff:192\.168\./,
|
|
27
|
-
];
|
|
28
2
|
/**
|
|
29
3
|
* Check if an IP address is in a blocked private range
|
|
30
4
|
*/
|
|
31
|
-
function isBlockedIp(ip) {
|
|
32
|
-
return
|
|
33
|
-
}
|
|
34
|
-
export async function validateResolvedIps(hostname) {
|
|
35
|
-
if (/^[\d.]+$/.test(hostname) || hostname.includes(':')) {
|
|
36
|
-
return;
|
|
37
|
-
}
|
|
38
|
-
try {
|
|
39
|
-
const ipv4Addresses = await dns.resolve4(hostname).catch(() => []);
|
|
40
|
-
for (const ip of ipv4Addresses) {
|
|
41
|
-
if (isBlockedIp(ip) || BLOCKED_HOSTS.has(ip)) {
|
|
42
|
-
throw new Error(`DNS rebinding detected: ${hostname} resolves to blocked IP ${ip}`);
|
|
43
|
-
}
|
|
44
|
-
}
|
|
45
|
-
const ipv6Addresses = await dns.resolve6(hostname).catch(() => []);
|
|
46
|
-
for (const ip of ipv6Addresses) {
|
|
47
|
-
if (isBlockedIp(ip) || BLOCKED_HOSTS.has(ip)) {
|
|
48
|
-
throw new Error(`DNS rebinding detected: ${hostname} resolves to blocked IP ${ip}`);
|
|
49
|
-
}
|
|
50
|
-
}
|
|
51
|
-
}
|
|
52
|
-
catch (error) {
|
|
53
|
-
if (error instanceof Error && error.message.includes('DNS rebinding')) {
|
|
54
|
-
throw error;
|
|
55
|
-
}
|
|
56
|
-
}
|
|
5
|
+
export function isBlockedIp(ip) {
|
|
6
|
+
return config.security.blockedIpPatterns.some((pattern) => pattern.test(ip));
|
|
57
7
|
}
|
|
58
8
|
export function validateAndNormalizeUrl(urlString) {
|
|
59
9
|
if (!urlString || typeof urlString !== 'string') {
|
|
@@ -83,7 +33,7 @@ export function validateAndNormalizeUrl(urlString) {
|
|
|
83
33
|
if (!hostname) {
|
|
84
34
|
throw new Error('URL must have a valid hostname');
|
|
85
35
|
}
|
|
86
|
-
if (
|
|
36
|
+
if (config.security.blockedHosts.has(hostname)) {
|
|
87
37
|
throw new Error(`Blocked host: ${hostname}. Internal hosts are not allowed`);
|
|
88
38
|
}
|
|
89
39
|
if (isBlockedIp(hostname)) {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"url-validator.js","sourceRoot":"","sources":["../../src/utils/url-validator.ts"],"names":[],"mappings":"AAAA,OAAO,
|
|
1
|
+
{"version":3,"file":"url-validator.js","sourceRoot":"","sources":["../../src/utils/url-validator.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,oBAAoB,CAAC;AAE5C;;GAEG;AACH,MAAM,UAAU,WAAW,CAAC,EAAU;IACpC,OAAO,MAAM,CAAC,QAAQ,CAAC,iBAAiB,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC;AAC/E,CAAC;AAED,MAAM,UAAU,uBAAuB,CAAC,SAAiB;IACvD,IAAI,CAAC,SAAS,IAAI,OAAO,SAAS,KAAK,QAAQ,EAAE,CAAC;QAChD,MAAM,IAAI,KAAK,CAAC,iBAAiB,CAAC,CAAC;IACrC,CAAC;IAED,MAAM,UAAU,GAAG,SAAS,CAAC,IAAI,EAAE,CAAC;IACpC,IAAI,CAAC,UAAU,EAAE,CAAC;QAChB,MAAM,IAAI,KAAK,CAAC,qBAAqB,CAAC,CAAC;IACzC,CAAC;IAED,IAAI,UAAU,CAAC,MAAM,GAAG,MAAM,CAAC,SAAS,CAAC,YAAY,EAAE,CAAC;QACtD,MAAM,IAAI,KAAK,CACb,iCAAiC,MAAM,CAAC,SAAS,CAAC,YAAY,aAAa,CAC5E,CAAC;IACJ,CAAC;IAED,IAAI,GAAQ,CAAC;IAEb,IAAI,CAAC;QACH,GAAG,GAAG,IAAI,GAAG,CAAC,UAAU,CAAC,CAAC;IAC5B,CAAC;IAAC,MAAM,CAAC;QACP,MAAM,IAAI,KAAK,CAAC,oBAAoB,CAAC,CAAC;IACxC,CAAC;IAED,IAAI,GAAG,CAAC,QAAQ,KAAK,OAAO,IAAI,GAAG,CAAC,QAAQ,KAAK,QAAQ,EAAE,CAAC;QAC1D,MAAM,IAAI,KAAK,CACb,qBAAqB,GAAG,CAAC,QAAQ,qCAAqC,CACvE,CAAC;IACJ,CAAC;IAED,IAAI,GAAG,CAAC,QAAQ,IAAI,GAAG,CAAC,QAAQ,EAAE,CAAC;QACjC,MAAM,IAAI,KAAK,CAAC,gDAAgD,CAAC,CAAC;IACpE,CAAC;IAED,MAAM,QAAQ,GAAG,GAAG,CAAC,QAAQ,CAAC,WAAW,EAAE,CAAC;IAE5C,IAAI,CAAC,QAAQ,EAAE,CAAC;QACd,MAAM,IAAI,KAAK,CAAC,gCAAgC,CAAC,CAAC;IACpD,CAAC;IAED,IAAI,MAAM,CAAC,QAAQ,CAAC,YAAY,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC;QAC/C,MAAM,IAAI,KAAK,CACb,iBAAiB,QAAQ,kCAAkC,CAC5D,CAAC;IACJ,CAAC;IAED,IAAI,WAAW,CAAC,QAAQ,CAAC,EAAE,CAAC;QAC1B,MAAM,IAAI,KAAK,CACb,qBAAqB,QAAQ,+BAA+B,CAC7D,CAAC;IACJ,CAAC;IAED,IAAI,QAAQ,CAAC,QAAQ,CAAC,QAAQ,CAAC,IAAI,QAAQ,CAAC,QAAQ,CAAC,WAAW,CAAC,EAAE,CAAC;QAClE,MAAM,IAAI,KAAK,CACb,6BAA6B,QAAQ,4CAA4C,CAClF,CAAC;IACJ,CAAC;IAED,OAAO,GAAG,CAAC,IAAI,CAAC;AAClB,CAAC;AAED,MAAM,UAAU,aAAa,CAAC,GAAW,EAAE,OAAe;IACxD,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;QACrC,MAAM,UAAU,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,CAAC;QACpC,OAAO,MAAM,CAAC,QAAQ,KAAK,UAAU,CAAC,QAAQ,CAAC;IACjD,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,KAAK,CAAC;IACf,CAAC;AACH,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@j0hanz/superfetch",
|
|
3
|
-
"version": "1.1.
|
|
3
|
+
"version": "1.1.2",
|
|
4
4
|
"mcpName": "io.github.j0hanz/superfetch",
|
|
5
5
|
"description": "Intelligent web content fetcher MCP server that converts HTML to clean, AI-readable JSONL format",
|
|
6
6
|
"type": "module",
|
|
@@ -50,14 +50,14 @@
|
|
|
50
50
|
"dependencies": {
|
|
51
51
|
"@modelcontextprotocol/sdk": "^1.25.1",
|
|
52
52
|
"@mozilla/readability": "^0.6.0",
|
|
53
|
-
"axios": "^1.
|
|
53
|
+
"axios": "^1.7.9",
|
|
54
54
|
"cheerio": "^1.1.2",
|
|
55
55
|
"domhandler": "^5.0.3",
|
|
56
56
|
"express": "^5.2.1",
|
|
57
57
|
"jsdom": "^27.3.0",
|
|
58
58
|
"node-cache": "^5.1.2",
|
|
59
59
|
"turndown": "^7.2.2",
|
|
60
|
-
"zod": "^3.
|
|
60
|
+
"zod": "^3.24.1"
|
|
61
61
|
},
|
|
62
62
|
"devDependencies": {
|
|
63
63
|
"@eslint/js": "^9.39.2",
|
|
@@ -69,7 +69,7 @@
|
|
|
69
69
|
"eslint": "^9.23.2",
|
|
70
70
|
"eslint-config-prettier": "^10.1.8",
|
|
71
71
|
"eslint-plugin-unused-imports": "^4.3.0",
|
|
72
|
-
"knip": "^5.
|
|
72
|
+
"knip": "^5.76.2",
|
|
73
73
|
"prettier": "^3.7.4",
|
|
74
74
|
"shx": "^0.4.0",
|
|
75
75
|
"tsx": "^4.21.0",
|