@page2ai/core 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +128 -0
- package/dist/browser/browser-adapter.d.ts +15 -0
- package/dist/browser/browser-adapter.d.ts.map +1 -0
- package/dist/browser/browser-adapter.js +74 -0
- package/dist/browser/browser-adapter.js.map +1 -0
- package/dist/browser/index.d.ts +3 -0
- package/dist/browser/index.d.ts.map +1 -0
- package/dist/browser/index.js +7 -0
- package/dist/browser/index.js.map +1 -0
- package/dist/node/basic-renderer.d.ts +9 -0
- package/dist/node/basic-renderer.d.ts.map +1 -0
- package/dist/node/basic-renderer.js +347 -0
- package/dist/node/basic-renderer.js.map +1 -0
- package/dist/node/fetch-protected.d.ts +19 -0
- package/dist/node/fetch-protected.d.ts.map +1 -0
- package/dist/node/fetch-protected.js +92 -0
- package/dist/node/fetch-protected.js.map +1 -0
- package/dist/node/index.d.ts +28 -0
- package/dist/node/index.d.ts.map +1 -0
- package/dist/node/index.js +136 -0
- package/dist/node/index.js.map +1 -0
- package/dist/node/linkedom-adapter.d.ts +17 -0
- package/dist/node/linkedom-adapter.d.ts.map +1 -0
- package/dist/node/linkedom-adapter.js +112 -0
- package/dist/node/linkedom-adapter.js.map +1 -0
- package/dist/node/ssrf-guard.d.ts +6 -0
- package/dist/node/ssrf-guard.d.ts.map +1 -0
- package/dist/node/ssrf-guard.js +73 -0
- package/dist/node/ssrf-guard.js.map +1 -0
- package/dist/shared/constants.d.ts +12 -0
- package/dist/shared/constants.d.ts.map +1 -0
- package/dist/shared/constants.js +164 -0
- package/dist/shared/constants.js.map +1 -0
- package/dist/shared/md-postprocess.d.ts +18 -0
- package/dist/shared/md-postprocess.d.ts.map +1 -0
- package/dist/shared/md-postprocess.js +538 -0
- package/dist/shared/md-postprocess.js.map +1 -0
- package/dist/shared/page-adapter.d.ts +26 -0
- package/dist/shared/page-adapter.d.ts.map +1 -0
- package/dist/shared/page-adapter.js +15 -0
- package/dist/shared/page-adapter.js.map +1 -0
- package/dist/shared/profiles.d.ts +6 -0
- package/dist/shared/profiles.d.ts.map +1 -0
- package/dist/shared/profiles.js +185 -0
- package/dist/shared/profiles.js.map +1 -0
- package/dist/shared/types.d.ts +265 -0
- package/dist/shared/types.d.ts.map +1 -0
- package/dist/shared/types.js +4 -0
- package/dist/shared/types.js.map +1 -0
- package/dist/shared/utils.d.ts +21 -0
- package/dist/shared/utils.d.ts.map +1 -0
- package/dist/shared/utils.js +184 -0
- package/dist/shared/utils.js.map +1 -0
- package/package.json +81 -0
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"basic-renderer.js","sourceRoot":"","sources":["../../src/node/basic-renderer.ts"],"names":[],"mappings":"AAAA,0EAA0E;AAC1E,gFAAgF;AAChF,0EAA0E;AAC1E,+EAA+E;AAC/E,6EAA6E;AAC7E,6CAA6C;AAC7C,EAAE;AACF,2EAA2E;AAC3E,6EAA6E;AAG7E,OAAO,EAAE,MAAM,EAAE,eAAe,EAAE,WAAW,EAAE,QAAQ,EAAE,iBAAiB,EAAE,MAAM,oBAAoB,CAAC;AAEvG,MAAM,SAAS,GAAG,IAAI,GAAG,CAAC,CAAC,QAAQ,EAAE,OAAO,EAAE,UAAU,EAAE,UAAU,EAAE,KAAK,EAAE,QAAQ,CAAC,CAAC,CAAC;AACxF,MAAM,WAAW,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,QAAQ,EAAE,GAAG,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,GAAG,EAAE,GAAG,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,GAAG,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,CAAC,CAAC,CAAC;AAC9L,MAAM,WAAW,GAAG,IAAI,GAAG,CAAC,CAAC,MAAM,EAAE,IAAI,EAAE,OAAO,EAAE,QAAQ,EAAE,IAAI,EAAE,YAAY,EAAE,IAAI,EAAE,YAAY,EAAE,IAAI,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,OAAO,EAAE,GAAG,EAAE,KAAK,EAAE,QAAQ,EAAE,KAAK,EAAE,MAAM,EAAE,UAAU,EAAE,IAAI,EAAE,SAAS,EAAE,YAAY,EAAE,MAAM,CAAC,CAAC,CAAC;AAEjT,MAAM,EAAE,GAAG,CAAC,CAAS,EAAU,EAAE,CAAC,GAAG,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,EAAE,KAAK,CAAC,GAAG,GAAG,CAAC;AAgB7E,MAAM,CAAC,MAAM,mBAAmB,GAAG,CACjC,GAAa,EACb,OAAoB,EACpB,IAAgB,EACR,EAAE;IACV,MAAM,KAAK,GAAa,EAAE,CAAC;IAE3B,IAAI,IAAI,CAAC,kBAAkB,EAAE,CAAC;QAC5B,KAAK,CAAC,IAAI,CAAC,GAAG,gBAAgB,CAAC,GAAG,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC;IACrD,CAAC;IAED,MAAM,KAAK,GAAG,WAAW,CAAC,GAAG,CAAC,aAAa,CAAC,OAAO,CAAC,EAAE,WAAW,IAAI,EAAE,CAAC,CAAC;IACzE,IAAI,KAAK;QAAE,KAAK,CAAC,IAAI,CAAC,KAAK,KAAK,EAAE,EAAE,EAAE,CAAC,CAAC;IAExC,MAAM,IAAI,GAAG,eAAe,CAAC,GAAG,CAAC,CAAC;IAClC,IAAI,IAAI,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;QAC1B,MAAM,SAAS,GAAG,iBAAiB,CAAC,IAAI,CAAC,CAAC;QAC1C,MAAM,gBAAgB,GAAG,IAAI,OAAO,EAAW,CAAC;QAChD,SAAS,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,gBAAgB,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;QAE9E,MAAM,KAAK,GAAa,EAAE,CAAC;QAC3B,KAAK,OAAO,CAAC;QACb,YAAY,CAAC,IAAI,EAAE,KAAK,EAAE,CAAC,EAAE,IAAI,EAAE,SAAS,EAAE,gBAAgB,CAAC,CAAC;QAChE,KAAK,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC,CAAC;IACvB,CAAC;SAAM,CAAC;QACN,KAAK,CAAC,IAAI,CAAC,EAAE,EAAE,mDAAmD,EAAE,EAAE,CAAC,CAAC;IAC1E,CAAC;IAED,OAAO,cAAc,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,EAAE,GAAG,IAAI,CAAC;AACxD,CAAC,CAAC;AAEF,MAAM,gBAAgB,GAAG,CAAC,GAAa,EAAE,OAAe,EAAY,EAAE;IACpE,MAAM,KAAK,GAAG,WAAW,CAAC,GAAG,CAAC,aAAa,CAAC,OAAO,CAAC,EAAE,WAAW,IAAI,EAAE,CAAC,CAAC;IACzE,MAAM,WAAW,GAAG,WAAW,CAC7B,GAAG,CAAC,aAAa,CAAC,0BAA0B,CAAC,EAAE,YAAY,CAAC,SAAS,CAAC,IAAI,EAAE,CAC7E,CAAC;IACF,MAAM,IAAI,GAAG,GAAG,CAAC,eAAe,EAAE,YAAY,CAAC,MAAM,CAAC,IAAI,SAAS,CAAC;IACpE,MAAM,SAAS,GAAG,MAAM,CACtB,GAAG,CAAC,aAAa,CAAC,uBAAuB,CAAC,EAAE,YAAY,CAAC,MAAM,CAAC,IAAI,EAAE,EACtE,OAAO,CACR,CAAC;IACF,MAAM,OAAO,GAAG,WAAW,CACzB,GAAG,CAAC,aAAa,CAAC,2BAA2B,CAAC,EAAE,YAAY,CAAC,SAAS,CAAC,IAAI,EAAE,CAC9E,CAAC;IACF,MAAM,aAAa,GAAG,WAAW,CAC/B,GAAG,CAAC,aAAa,CAAC,iCAAiC,CAAC,EAAE,YAAY,CAAC,SAAS,CAAC,IAAI,EAAE,CACpF,CAAC;IACF,MAAM,MAAM,GAAG,WAAW,CACxB,GAAG,CAAC,aAAa,CAAC,qBAAqB,CAAC,EAAE,YAAY,CAAC,SAAS,CAAC;QAC/D,GAAG,CAAC,aAAa,CAAC,iCAAiC,CAAC,EAAE,YAAY,CAAC,SAAS,CAAC;QAC7E,EAAE,CACL,CAAC;IACF,MAAM,aAAa,GAAG,WAAW,CAC/B,GAAG,CAAC,aAAa,CAAC,yCAAyC,CAAC,EAAE,YAAY,CAAC,SAAS,CAAC,IAAI,EAAE,CAC5F,CAAC;IAEF,MAAM,EAAE,GAAa,CAAC,KAAK,CAAC,CAAC;IAC7B,EAAE,CAAC,IAAI,CAAC,UAAU,EAAE,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;IAC/B,EAAE,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;IAClC,EAAE,CAAC,IAAI,CAAC,gBAAgB,EAAE,CAAC,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC,EAAE,CAAC,CAAC;IACxD,EAAE,CAAC,IAAI,CAAC,aAAa,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACjC,IAAI,WAAW;QAAE,EAAE,CAAC,IAAI,CAAC,gBAAgB,EAAE,CAAC,WAAW,CAAC,EAAE,CAAC,CAAC;IAC5D,IAAI,SAAS;QAAE,EAAE,CAAC,IAAI,CAAC,cAAc,EAAE,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC;IACtD,IAAI,OAAO,IAAI,OAAO,KAAK,KAAK;QAAE,EAAE,CAAC,IAAI,CAAC,aAAa,EAAE,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;IACtE,IAAI,aAAa,IAAI,aAAa,KAAK,WAAW;QAAE,EAAE,CAAC,IAAI,CAAC,mBAAmB,EAAE,CAAC,aAAa,CAAC,EAAE,CAAC,CAAC;IACpG,IAAI,MAAM;QAAE,EAAE,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;IAC7C,IAAI,aAAa;QAAE,EAAE,CAAC,IAAI,CAAC,cAAc,EAAE,CAAC,aAAa,CAAC,EAAE,CAAC,CAAC;IAC9D,EAAE,CAAC,IAAI,CAAC,cAAc,EAAE,CAAC,cAAc,CAAC,EAAE,CAAC,CAAC;IAC5C,EAAE,CAAC,IAAI,CAAC,sBAAsB,EAAE,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;IAC7C,EAAE,CAAC,IAAI,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;IACnB,OAAO,EAAE,CAAC;AACZ,CAAC,CAAC;AAEF,MAAM,eAAe,GAAG,CAAC,GAAa,EAAkB,EAAE;IACxD,MAAM,IAAI,GACR,GAAG,CAAC,aAAa,CAAC,cAAc,CAAC;QACjC,GAAG,CAAC,aAAa,CAAC,SAAS,CAAC;QAC5B,GAAG,CAAC,aAAa,CAAC,MAAM,CAAC;QACzB,GAAG,CAAC,aAAa,CAAC,eAAe,CAAC;QAClC,GAAG,CAAC,IAAI;QACR,GAAG,CAAC,eAAe;QACnB,IAAI,CAAC;IACP,OAAQ,IAA2B,IAAI,IAAI,CAAC;AAC9C,CAAC,CAAC;AAEF,2EAA2E;AAC3E,yEAAyE;AACzE,sEAAsE;AACtE,MAAM,iBAAiB,GAAG,CAAC,IAAa,EAAc,EAAE;IACtD,MAAM,MAAM,GAAe,EAAE,CAAC;IAC9B,MAAM,cAAc,GAAG,IAAI,OAAO,EAAW,CAAC;IAC9C,oDAAoD;IACpD,IAAI,CAAC,gBAAgB,CAAC,kBAAkB,CAAC,CAAC,OAAO,CAAC,CAAC,EAAE,EAAE,EAAE;QACvD,IAAI,cAAc,CAAC,GAAG,CAAC,EAAE,CAAC;YAAE,OAAO;QACnC,MAAM,IAAI,GAAG,CAAC,GAAG,EAAE,CAAC,gBAAgB,CAAC,sBAAsB,CAAC,CAAC,CAAC;QAC9D,MAAM,MAAM,GAA0C,EAAE,CAAC;QACzD,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;YACvB,MAAM,KAAK,GAAG,WAAW,CAAC,GAAG,CAAC,WAAW,IAAI,EAAE,CAAC,CAAC;YACjD,IAAI,CAAC,KAAK;gBAAE,SAAS;YACrB,MAAM,QAAQ,GAAG,GAAG,CAAC,YAAY,CAAC,eAAe,CAAC,CAAC;YACnD,IAAI,CAAC,QAAQ;gBAAE,SAAS;YACxB,+DAA+D;YAC/D,MAAM,GAAG,GAAG,EAAE,CAAC,aAAa,CAAC;YAC7B,IAAI,CAAC,GAAG;gBAAE,SAAS;YACnB,MAAM,KAAK,GAAG,GAAG,CAAC,cAAc,CAAC,QAAQ,CAAC,CAAC;YAC3C,IAAI,CAAC,KAAK;gBAAE,SAAS;YACrB,MAAM,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC;QACpC,CAAC;QACD,IAAI,MAAM,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;YACvB,MAAM,CAAC,IAAI,CAAC,EAAE,SAAS,EAAE,EAAE,EAAE,MAAM,EAAE,CAAC,CAAC;YACvC,cAAc,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QACzB,CAAC;IACH,CAAC,CAAC,CAAC;IACH,iEAAiE;IACjE,IAAI,CAAC,gBAAgB,CAAC,gDAAgD,CAAC,CAAC,OAAO,CAAC,CAAC,EAAE,EAAE,EAAE;QACrF,IAAI,EAAE,CAAC,YAAY,CAAC,MAAM,CAAC,KAAK,UAAU;YAAE,OAAO;QACnD,MAAM,MAAM,GAAG,EAAE,CAAC,aAAa,CAAC;QAChC,IAAI,CAAC,MAAM,IAAI,cAAc,CAAC,GAAG,CAAC,MAAM,CAAC;YAAE,OAAO;QAClD,MAAM,aAAa,GAAG,CAAC,GAAG,MAAM,CAAC,QAAQ,CAAC,CAAC,MAAM,CAC/C,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,YAAY,CAAC,MAAM,CAAC,KAAK,UAAU,CAC7C,CAAC;QACF,IAAI,aAAa,CAAC,MAAM,GAAG,CAAC;YAAE,OAAO;QACrC,MAAM,MAAM,GAAG,aAAa,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;YACvC,KAAK,EAAE,CAAC,CAAC,YAAY,CAAC,iBAAiB,CAAC;gBACjC,CAAC,CAAC,YAAY,CAAC,YAAY,CAAC;gBAC5B,CAAC,CAAC,YAAY,CAAC,YAAY,CAAC;gBAC5B,CAAC,CAAC,EAAE;gBACJ,OAAO;YACd,EAAE,EAAE,CAAC;SACN,CAAC,CAAC,CAAC;QACJ,MAAM,CAAC,IAAI,CAAC,EAAE,SAAS,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC,CAAC;QAC3C,cAAc,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;IAC7B,CAAC,CAAC,CAAC;IACH,OAAO,MAAM,CAAC;AAChB,CAAC,CAAC;AAEF,MAAM,YAAY,GAAG,CACnB,EAAW,EACX,GAAa,EACb,KAAa,EACb,IAAgB,EAChB,SAAqB,EACrB,gBAAkC,EAC5B,EAAE;IACR,KAAK,MAAM,KAAK,IAAI,EAAE,CAAC,QAAQ,EAAE,CAAC;QAChC,kBAAkB,CAAC,KAAK,EAAE,GAAG,EAAE,KAAK,EAAE,IAAI,EAAE,SAAS,EAAE,gBAAgB,CAAC,CAAC;IAC3E,CAAC;AACH,CAAC,CAAC;AAEF,MAAM,kBAAkB,GAAG,CACzB,EAAW,EACX,GAAa,EACb,KAAa,EACb,IAAgB,EAChB,SAAqB,EACrB,gBAAkC,EAC5B,EAAE;IACR,yEAAyE;IACzE,6CAA6C;IAC7C,IAAI,gBAAgB,CAAC,GAAG,CAAC,EAAE,CAAC;QAAE,OAAO;IACrC,2EAA2E;IAC3E,MAAM,KAAK,GAAG,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,KAAK,EAAE,CAAC,CAAC;IACxD,IAAI,KAAK,EAAE,CAAC;QACV,KAAK,MAAM,KAAK,IAAI,KAAK,CAAC,MAAM,EAAE,CAAC;YACjC,GAAG,CAAC,IAAI,CAAC,EAAE,EAAE,YAAY,QAAQ,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC;YACtD,YAAY,CAAC,KAAK,CAAC,EAAE,EAAE,GAAG,EAAE,KAAK,EAAE,IAAI,EAAE,SAAS,EAAE,gBAAgB,CAAC,CAAC;QACxE,CAAC;QACD,OAAO;IACT,CAAC;IACD,UAAU,CAAC,EAAE,EAAE,GAAG,EAAE,KAAK,EAAE,IAAI,EAAE,SAAS,EAAE,gBAAgB,CAAC,CAAC;AAChE,CAAC,CAAC;AAEF,MAAM,UAAU,GAAG,CACjB,EAAW,EACX,GAAa,EACb,KAAa,EACb,IAAgB,EAChB,SAAqB,EACrB,gBAAkC,EAC5B,EAAE;IACR,MAAM,GAAG,GAAG,EAAE,CAAC,OAAO,CAAC;IACvB,IAAI,SAAS,CAAC,GAAG,CAAC,GAAG,CAAC;QAAE,OAAO;IAC/B,IAAI,EAAE,CAAC,YAAY,CAAC,aAAa,CAAC,KAAK,MAAM;QAAE,OAAO;IAEtD,MAAM,GAAG,GAAG,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IAEhC,IAAI,UAAU,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;QACzB,MAAM,KAAK,GAAG,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;QACnC,MAAM,IAAI,GAAG,YAAY,CAAC,EAAE,EAAE,IAAI,CAAC,CAAC,IAAI,EAAE,CAAC;QAC3C,IAAI,IAAI;YAAE,GAAG,CAAC,IAAI,CAAC,EAAE,EAAE,GAAG,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,IAAI,EAAE,EAAE,EAAE,CAAC,CAAC;QAC3D,OAAO;IACT,CAAC;IAED,IAAI,GAAG,KAAK,KAAK,EAAE,CAAC;QAClB,eAAe,CAAC,EAAE,EAAE,GAAG,CAAC,CAAC;QACzB,OAAO;IACT,CAAC;IAED,IAAI,GAAG,KAAK,GAAG,EAAE,CAAC;QAChB,MAAM,IAAI,GAAG,YAAY,CAAC,EAAE,EAAE,IAAI,CAAC,CAAC,IAAI,EAAE,CAAC;QAC3C,IAAI,IAAI;YAAE,GAAG,CAAC,IAAI,CAAC,EAAE,EAAE,IAAI,EAAE,EAAE,CAAC,CAAC;QACjC,OAAO;IACT,CAAC;IAED,IAAI,GAAG,KAAK,GAAG,EAAE,CAAC;QAChB,MAAM,IAAI,GAAG,MAAM,CAAC,EAAE,CAAC,YAAY,CAAC,MAAM,CAAC,IAAI,EAAE,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC;QACjE,MAAM,IAAI,GAAG,YAAY,CAAC,EAAE,EAAE,IAAI,CAAC,CAAC,IAAI,EAAE,IAAI,WAAW,CAAC,EAAE,CAAC,WAAW,IAAI,EAAE,CAAC,CAAC;QAChF,IAAI,IAAI,IAAI,IAAI;YAAE,GAAG,CAAC,IAAI,CAAC,GAAG,GAAG,IAAI,QAAQ,CAAC,IAAI,CAAC,KAAK,IAAI,GAAG,CAAC,CAAC;aAC5D,IAAI,IAAI;YAAE,GAAG,CAAC,IAAI,CAAC,GAAG,GAAG,GAAG,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACnD,OAAO;IACT,CAAC;IAED,IAAI,GAAG,KAAK,YAAY,EAAE,CAAC;QACzB,MAAM,KAAK,GAAa,EAAE,CAAC;QAC3B,YAAY,CAAC,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,IAAI,EAAE,SAAS,EAAE,gBAAgB,CAAC,CAAC;QAClE,MAAM,IAAI,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,CAAC;QACrC,IAAI,IAAI,EAAE,CAAC;YACT,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YACb,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,CAAC;YACpD,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACf,CAAC;QACD,OAAO;IACT,CAAC;IAED,IAAI,GAAG,KAAK,IAAI,IAAI,GAAG,KAAK,IAAI,EAAE,CAAC;QACjC,MAAM,KAAK,GAAG,CAAC,GAAG,EAAE,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,KAAK,IAAI,CAAC,CAAC;QACjE,IAAI,CAAC,GAAG,CAAC,CAAC;QACV,KAAK,MAAM,EAAE,IAAI,KAAK,EAAE,CAAC;YACvB,MAAM,MAAM,GAAG,GAAG,KAAK,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC;YAC5C,MAAM,IAAI,GAAG,YAAY,CAAC,EAAE,EAAE,IAAI,CAAC,CAAC,IAAI,EAAE,CAAC;YAC3C,IAAI,IAAI;gBAAE,GAAG,CAAC,IAAI,CAAC,GAAG,GAAG,GAAG,MAAM,IAAI,IAAI,EAAE,CAAC,CAAC;YAC9C,KAAK,MAAM,GAAG,IAAI,EAAE,CAAC,QAAQ,EAAE,CAAC;gBAC9B,IAAI,GAAG,CAAC,OAAO,KAAK,IAAI,IAAI,GAAG,CAAC,OAAO,KAAK,IAAI,EAAE,CAAC;oBACjD,kBAAkB,CAAC,GAAG,EAAE,GAAG,EAAE,KAAK,GAAG,CAAC,EAAE,IAAI,EAAE,SAAS,EAAE,gBAAgB,CAAC,CAAC;gBAC7E,CAAC;YACH,CAAC;YACD,CAAC,EAAE,CAAC;QACN,CAAC;QACD,OAAO;IACT,CAAC;IAED,IAAI,GAAG,KAAK,OAAO,EAAE,CAAC;QACpB,WAAW,CAAC,EAAsB,EAAE,GAAG,CAAC,CAAC;QACzC,OAAO;IACT,CAAC;IAED,IAAI,GAAG,KAAK,KAAK,EAAE,CAAC;QAClB,IAAI,CAAC,IAAI,CAAC,aAAa;YAAE,OAAO;QAChC,MAAM,GAAG,GAAG,WAAW,CAAC,EAAE,CAAC,YAAY,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC;QACtD,MAAM,GAAG,GAAG,MAAM,CAAC,EAAE,CAAC,YAAY,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,YAAY,CAAC,UAAU,CAAC,IAAI,EAAE,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC;QAC9F,IAAI,GAAG;YAAE,GAAG,CAAC,IAAI,CAAC,GAAG,GAAG,KAAK,QAAQ,CAAC,GAAG,IAAI,OAAO,CAAC,KAAK,GAAG,GAAG,CAAC,CAAC;QAClE,OAAO;IACT,CAAC;IAED,IAAI,GAAG,KAAK,IAAI,EAAE,CAAC;QAAC,GAAG,CAAC,IAAI,CAAC,EAAE,EAAE,KAAK,EAAE,EAAE,CAAC,CAAC;QAAC,OAAO;IAAC,CAAC;IACtD,IAAI,GAAG,KAAK,IAAI,EAAE,CAAC;QAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QAAC,OAAO;IAAC,CAAC;IAE3C,YAAY,CAAC,EAAE,EAAE,GAAG,EAAE,KAAK,EAAE,IAAI,EAAE,SAAS,EAAE,gBAAgB,CAAC,CAAC;AAClE,CAAC,CAAC;AAEF,MAAM,YAAY,GAAG,CAAC,EAAW,EAAE,IAAgB,EAAU,EAAE;IAC7D,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,KAAK,MAAM,IAAI,IAAI,EAAE,CAAC,UAAU,EAAE,CAAC;QACjC,IAAI,IAAI,CAAC,QAAQ,KAAK,CAAC,CAAC,eAAe,EAAE,CAAC;YACxC,MAAM,CAAC,GAAG,WAAW,CAAC,IAAI,CAAC,WAAW,IAAI,EAAE,CAAC,CAAC;YAC9C,IAAI,CAAC;gBAAE,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YACrB,SAAS;QACX,CAAC;QACD,IAAI,IAAI,CAAC,QAAQ,KAAK,CAAC,CAAC,kBAAkB;YAAE,SAAS;QACrD,MAAM,CAAC,GAAG,IAAe,CAAC;QAC1B,MAAM,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC;QACtB,IAAI,SAAS,CAAC,GAAG,CAAC,GAAG,CAAC;YAAE,SAAS;QACjC,IAAI,GAAG,KAAK,IAAI,EAAE,CAAC;YAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAAC,SAAS;QAAC,CAAC;QACjD,IAAI,GAAG,KAAK,GAAG,EAAE,CAAC;YAChB,MAAM,IAAI,GAAG,MAAM,CAAC,CAAC,CAAC,YAAY,CAAC,MAAM,CAAC,IAAI,EAAE,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC;YAChE,MAAM,IAAI,GAAG,YAAY,CAAC,CAAC,EAAE,IAAI,CAAC,IAAI,WAAW,CAAC,CAAC,CAAC,WAAW,IAAI,EAAE,CAAC,CAAC;YACvE,IAAI,IAAI,IAAI,IAAI;gBAAE,KAAK,CAAC,IAAI,CAAC,IAAI,QAAQ,CAAC,IAAI,CAAC,KAAK,IAAI,GAAG,CAAC,CAAC;iBACxD,IAAI,IAAI;gBAAE,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC;YAC1C,SAAS;QACX,CAAC;QACD,IAAI,GAAG,KAAK,QAAQ,IAAI,GAAG,KAAK,GAAG,EAAE,CAAC;YACpC,MAAM,CAAC,GAAG,YAAY,CAAC,CAAC,EAAE,IAAI,CAAC,IAAI,WAAW,CAAC,CAAC,CAAC,WAAW,IAAI,EAAE,CAAC,CAAC;YACpE,IAAI,CAAC;gBAAE,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;YAC9B,SAAS;QACX,CAAC;QACD,IAAI,GAAG,KAAK,IAAI,IAAI,GAAG,KAAK,GAAG,EAAE,CAAC;YAChC,MAAM,CAAC,GAAG,YAAY,CAAC,CAAC,EAAE,IAAI,CAAC,IAAI,WAAW,CAAC,CAAC,CAAC,WAAW,IAAI,EAAE,CAAC,CAAC;YACpE,IAAI,CAAC;gBAAE,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YAC5B,SAAS;QACX,CAAC;QACD,IAAI,GAAG,KAAK,MAAM,EAAE,CAAC;YACnB,MAAM,CAAC,GAAG,WAAW,CAAC,CAAC,CAAC,WAAW,IAAI,EAAE,CAAC,CAAC;YAC3C,IAAI,CAAC;gBAAE,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;YAC9B,SAAS;QACX,CAAC;QACD,IAAI,GAAG,KAAK,KAAK,EAAE,CAAC;YAClB,MAAM,GAAG,GAAG,WAAW,CAAC,CAAC,CAAC,YAAY,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC;YACrD,MAAM,GAAG,GAAG,MAAM,CAAC,CAAC,CAAC,YAAY,CAAC,KAAK,CAAC,IAAI,EAAE,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC;YAC9D,IAAI,GAAG;gBAAE,KAAK,CAAC,IAAI,CAAC,KAAK,QAAQ,CAAC,GAAG,IAAI,OAAO,CAAC,KAAK,GAAG,GAAG,CAAC,CAAC;YAC9D,SAAS;QACX,CAAC;QACD,IAAI,WAAW,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC;YACzB,MAAM,CAAC,GAAG,YAAY,CAAC,CAAC,EAAE,IAAI,CAAC,IAAI,WAAW,CAAC,CAAC,CAAC,WAAW,IAAI,EAAE,CAAC,CAAC;YACpE,IAAI,CAAC;gBAAE,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YACrB,SAAS;QACX,CAAC;QACD,MAAM,CAAC,GAAG,WAAW,CAAC,CAAC,CAAC,WAAW,IAAI,EAAE,CAAC,CAAC;QAC3C,IAAI,CAAC;YAAE,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IACvB,CAAC;IACD,OAAO,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC,WAAW,EAAE,IAAI,CAAC,CAAC;AACpD,CAAC,CAAC;AAEF,MAAM,eAAe,GAAG,CAAC,GAAY,EAAE,GAAa,EAAQ,EAAE;IAC5D,MAAM,MAAM,GAAG,GAAG,CAAC,aAAa,CAAC,MAAM,CAAC,IAAI,GAAG,CAAC;IAChD,MAAM,IAAI,GAAG,CAAC,MAAM,CAAC,WAAW,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,YAAY,EAAE,EAAE,CAAC,CAAC;IACrF,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE;QAAE,OAAO;IAEzB,IAAI,IAAI,GAAG,EAAE,CAAC;IACd,MAAM,SAAS,GAAG,MAAM,CAAC,YAAY,CAAC,OAAO,CAAC,IAAI,GAAG,CAAC,YAAY,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC;IAClF,MAAM,SAAS,GAAG,SAAS,CAAC,KAAK,CAAC,4BAA4B,CAAC,CAAC;IAChE,IAAI,SAAS,EAAE,CAAC;QACd,MAAM,CAAC,GAAG,SAAS,CAAC,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC;QACrC,IAAI,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,uBAAuB,CAAC,IAAI,CAAC,CAAC,CAAC;YAAE,IAAI,GAAG,CAAC,CAAC;IACtE,CAAC;IACD,MAAM,QAAQ,GAAG,GAAG,CAAC,YAAY,CAAC,eAAe,CAAC,IAAI,MAAM,CAAC,YAAY,CAAC,eAAe,CAAC,IAAI,EAAE,CAAC;IACjG,IAAI,CAAC,IAAI,IAAI,QAAQ,EAAE,CAAC;QACtB,MAAM,CAAC,GAAG,QAAQ,CAAC,WAAW,EAAE,CAAC;QACjC,IAAI,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,uBAAuB,CAAC,IAAI,CAAC,CAAC,CAAC;YAAE,IAAI,GAAG,CAAC,CAAC;IACtE,CAAC;IAED,MAAM,KAAK,GAAG,eAAe,CAAC,IAAI,CAAC,CAAC;IACpC,GAAG,CAAC,IAAI,CAAC,EAAE,EAAE,GAAG,KAAK,GAAG,IAAI,EAAE,CAAC,CAAC;IAChC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;IAC7C,GAAG,CAAC,IAAI,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;AACtB,CAAC,CAAC;AAEF,MAAM,WAAW,GAAG,CAAC,GAAqB,EAAE,GAAa,EAAQ,EAAE;IACjE,MAAM,IAAI,GAAG,CAAC,GAAG,GAAG,CAAC,gBAAgB,CAAC,IAAI,CAAC,CAAC,CAAC;IAC7C,IAAI,CAAC,IAAI,CAAC,MAAM;QAAE,OAAO;IACzB,MAAM,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAC3B,CAAC,GAAG,EAAE,CAAC,gBAAgB,CAAC,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,iBAAiB,CAAC,CAAC,CAAC,WAAW,IAAI,EAAE,CAAC,CAAC,CACtF,CAAC;IACF,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC;IACrD,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,GAAG,OAAO,CAAC,CAAC,MAAM,GAAG,KAAK;QAAE,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IACnF,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACb,MAAM,MAAM,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;IACzB,GAAG,CAAC,IAAI,CAAC,KAAK,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IACtC,GAAG,CAAC,IAAI,CAAC,KAAK,MAAM,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IACvD,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC;IACjE,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;AACf,CAAC,CAAC;AAEF,MAAM,cAAc,GAAG,CAAC,CAAS,EAAU,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC"}
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
export interface FetchProtectedOptions {
|
|
2
|
+
timeoutMs?: number;
|
|
3
|
+
maxBytes?: number;
|
|
4
|
+
userAgent?: string;
|
|
5
|
+
acceptContentTypes?: RegExp;
|
|
6
|
+
}
|
|
7
|
+
export interface FetchProtectedResult {
|
|
8
|
+
text: string;
|
|
9
|
+
finalUrl: string;
|
|
10
|
+
contentType: string;
|
|
11
|
+
bytesRead: number;
|
|
12
|
+
statusCode: number;
|
|
13
|
+
}
|
|
14
|
+
export declare class FetchProtectionError extends Error {
|
|
15
|
+
readonly url: string;
|
|
16
|
+
constructor(reason: string, url: string);
|
|
17
|
+
}
|
|
18
|
+
export declare const fetchProtected: (rawUrl: string, opts?: FetchProtectedOptions) => Promise<FetchProtectedResult>;
|
|
19
|
+
//# sourceMappingURL=fetch-protected.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"fetch-protected.d.ts","sourceRoot":"","sources":["../../src/node/fetch-protected.ts"],"names":[],"mappings":"AAaA,MAAM,WAAW,qBAAqB;IACpC,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,kBAAkB,CAAC,EAAE,MAAM,CAAC;CAC7B;AAED,MAAM,WAAW,oBAAoB;IACnC,IAAI,EAAE,MAAM,CAAC;IACb,QAAQ,EAAE,MAAM,CAAC;IACjB,WAAW,EAAE,MAAM,CAAC;IACpB,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;CACpB;AAOD,qBAAa,oBAAqB,SAAQ,KAAK;aACD,GAAG,EAAE,MAAM;gBAA3C,MAAM,EAAE,MAAM,EAAkB,GAAG,EAAE,MAAM;CAIxD;AAID,eAAO,MAAM,cAAc,GACzB,QAAQ,MAAM,EACd,OAAM,qBAA0B,KAC/B,OAAO,CAAC,oBAAoB,CA+E9B,CAAC"}
|
|
@@ -0,0 +1,92 @@
|
|
|
1
|
+
// Hardened fetch for the Node MCP server:
|
|
2
|
+
// - SSRF guard (assertSafeUrl blocks private ranges, metadata endpoints)
|
|
3
|
+
// - AbortController timeout (default 15s)
|
|
4
|
+
// - Response size cap (default 10MB) — abort if body exceeds
|
|
5
|
+
// - Follows redirects and returns the FINAL URL for absUrl base resolution
|
|
6
|
+
// - Strips UTF-8 BOM from response text
|
|
7
|
+
// - Basic content-type guard: only text/html, text/markdown, text/plain accepted
|
|
8
|
+
//
|
|
9
|
+
// KNOWN LIMITATION (v0.1): no charset detection for non-UTF-8 pages (Windows-1251
|
|
10
|
+
// docs will mojibake). Planned for v0.2 via `fetch-charset-detection`.
|
|
11
|
+
import { assertSafeUrl } from './ssrf-guard.js';
|
|
12
|
+
const DEFAULT_UA = 'page2ai-core/0.1 (+https://github.com/igorsaevets/page2ai-core)';
|
|
13
|
+
const DEFAULT_TIMEOUT = 15000;
|
|
14
|
+
const DEFAULT_MAX_BYTES = 10 * 1024 * 1024; // 10 MB
|
|
15
|
+
const DEFAULT_ACCEPT = /^(text\/html|text\/markdown|text\/plain|application\/xhtml\+xml)/i;
|
|
16
|
+
export class FetchProtectionError extends Error {
|
|
17
|
+
url;
|
|
18
|
+
constructor(reason, url) {
|
|
19
|
+
super(`fetch-protected: ${reason} (url: ${url})`);
|
|
20
|
+
this.url = url;
|
|
21
|
+
this.name = 'FetchProtectionError';
|
|
22
|
+
}
|
|
23
|
+
}
|
|
24
|
+
const stripBom = (s) => s.replace(/^/, '');
|
|
25
|
+
export const fetchProtected = async (rawUrl, opts = {}) => {
|
|
26
|
+
assertSafeUrl(rawUrl);
|
|
27
|
+
const timeoutMs = opts.timeoutMs ?? DEFAULT_TIMEOUT;
|
|
28
|
+
const maxBytes = opts.maxBytes ?? DEFAULT_MAX_BYTES;
|
|
29
|
+
const ua = opts.userAgent ?? DEFAULT_UA;
|
|
30
|
+
const accept = opts.acceptContentTypes ?? DEFAULT_ACCEPT;
|
|
31
|
+
const controller = new AbortController();
|
|
32
|
+
const timer = setTimeout(() => controller.abort(), timeoutMs);
|
|
33
|
+
try {
|
|
34
|
+
const resp = await fetch(rawUrl, {
|
|
35
|
+
signal: controller.signal,
|
|
36
|
+
headers: {
|
|
37
|
+
'user-agent': ua,
|
|
38
|
+
accept: 'text/html,text/markdown,text/plain,application/xhtml+xml,*/*;q=0.5',
|
|
39
|
+
},
|
|
40
|
+
redirect: 'follow',
|
|
41
|
+
});
|
|
42
|
+
// After redirects — validate FINAL URL is still SSRF-safe (fetch may have
|
|
43
|
+
// followed a redirect INTO a private range).
|
|
44
|
+
const finalUrl = resp.url || rawUrl;
|
|
45
|
+
if (finalUrl !== rawUrl) {
|
|
46
|
+
try {
|
|
47
|
+
assertSafeUrl(finalUrl);
|
|
48
|
+
}
|
|
49
|
+
catch (e) {
|
|
50
|
+
throw new FetchProtectionError(`redirect landed on SSRF-blocked URL: ${e.message}`, rawUrl);
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
const contentType = resp.headers.get('content-type') || '';
|
|
54
|
+
if (!resp.ok) {
|
|
55
|
+
throw new FetchProtectionError(`HTTP ${resp.status} ${resp.statusText}, content-type=${contentType}`, rawUrl);
|
|
56
|
+
}
|
|
57
|
+
if (!accept.test(contentType)) {
|
|
58
|
+
throw new FetchProtectionError(`content-type "${contentType}" not in accepted set`, rawUrl);
|
|
59
|
+
}
|
|
60
|
+
// Read with size cap. reader.read() returns Uint8Array chunks; abort if
|
|
61
|
+
// cumulative bytes exceed maxBytes to prevent OOM on malicious 100MB pages.
|
|
62
|
+
const reader = resp.body?.getReader();
|
|
63
|
+
if (!reader) {
|
|
64
|
+
throw new FetchProtectionError('response body is null', rawUrl);
|
|
65
|
+
}
|
|
66
|
+
const chunks = [];
|
|
67
|
+
let bytesRead = 0;
|
|
68
|
+
while (true) {
|
|
69
|
+
const { value, done } = await reader.read();
|
|
70
|
+
if (done)
|
|
71
|
+
break;
|
|
72
|
+
if (!value)
|
|
73
|
+
continue;
|
|
74
|
+
bytesRead += value.byteLength;
|
|
75
|
+
if (bytesRead > maxBytes) {
|
|
76
|
+
try {
|
|
77
|
+
await reader.cancel();
|
|
78
|
+
}
|
|
79
|
+
catch { /* best effort */ }
|
|
80
|
+
throw new FetchProtectionError(`response body exceeds ${maxBytes} bytes cap`, rawUrl);
|
|
81
|
+
}
|
|
82
|
+
chunks.push(value);
|
|
83
|
+
}
|
|
84
|
+
const buf = Buffer.concat(chunks.map((c) => Buffer.from(c.buffer, c.byteOffset, c.byteLength)));
|
|
85
|
+
const text = stripBom(buf.toString('utf-8'));
|
|
86
|
+
return { text, finalUrl, contentType, bytesRead, statusCode: resp.status };
|
|
87
|
+
}
|
|
88
|
+
finally {
|
|
89
|
+
clearTimeout(timer);
|
|
90
|
+
}
|
|
91
|
+
};
|
|
92
|
+
//# sourceMappingURL=fetch-protected.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"fetch-protected.js","sourceRoot":"","sources":["../../src/node/fetch-protected.ts"],"names":[],"mappings":"AAAA,0CAA0C;AAC1C,yEAAyE;AACzE,0CAA0C;AAC1C,6DAA6D;AAC7D,2EAA2E;AAC3E,wCAAwC;AACxC,iFAAiF;AACjF,EAAE;AACF,kFAAkF;AAClF,uEAAuE;AAEvE,OAAO,EAAE,aAAa,EAAoB,MAAM,iBAAiB,CAAC;AAiBlE,MAAM,UAAU,GAAG,iEAAiE,CAAC;AACrF,MAAM,eAAe,GAAG,KAAK,CAAC;AAC9B,MAAM,iBAAiB,GAAG,EAAE,GAAG,IAAI,GAAG,IAAI,CAAC,CAAE,QAAQ;AACrD,MAAM,cAAc,GAAG,mEAAmE,CAAC;AAE3F,MAAM,OAAO,oBAAqB,SAAQ,KAAK;IACD;IAA5C,YAAY,MAAc,EAAkB,GAAW;QACrD,KAAK,CAAC,oBAAoB,MAAM,UAAU,GAAG,GAAG,CAAC,CAAC;QADR,QAAG,GAAH,GAAG,CAAQ;QAErD,IAAI,CAAC,IAAI,GAAG,sBAAsB,CAAC;IACrC,CAAC;CACF;AAED,MAAM,QAAQ,GAAG,CAAC,CAAS,EAAU,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;AAE5D,MAAM,CAAC,MAAM,cAAc,GAAG,KAAK,EACjC,MAAc,EACd,OAA8B,EAAE,EACD,EAAE;IACjC,aAAa,CAAC,MAAM,CAAC,CAAC;IAEtB,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,IAAI,eAAe,CAAC;IACpD,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,IAAI,iBAAiB,CAAC;IACpD,MAAM,EAAE,GAAG,IAAI,CAAC,SAAS,IAAI,UAAU,CAAC;IACxC,MAAM,MAAM,GAAG,IAAI,CAAC,kBAAkB,IAAI,cAAc,CAAC;IAEzD,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;IACzC,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,UAAU,CAAC,KAAK,EAAE,EAAE,SAAS,CAAC,CAAC;IAE9D,IAAI,CAAC;QACH,MAAM,IAAI,GAAG,MAAM,KAAK,CAAC,MAAM,EAAE;YAC/B,MAAM,EAAE,UAAU,CAAC,MAAM;YACzB,OAAO,EAAE;gBACP,YAAY,EAAE,EAAE;gBAChB,MAAM,EAAE,oEAAoE;aAC7E;YACD,QAAQ,EAAE,QAAQ;SACnB,CAAC,CAAC;QAEH,0EAA0E;QAC1E,6CAA6C;QAC7C,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,IAAI,MAAM,CAAC;QACpC,IAAI,QAAQ,KAAK,MAAM,EAAE,CAAC;YACxB,IAAI,CAAC;gBACH,aAAa,CAAC,QAAQ,CAAC,CAAC;YAC1B,CAAC;YAAC,OAAO,CAAC,EAAE,CAAC;gBACX,MAAM,IAAI,oBAAoB,CAC5B,wCAAyC,CAAsB,CAAC,OAAO,EAAE,EACzE,MAAM,CACP,CAAC;YACJ,CAAC;QACH,CAAC;QAED,MAAM,WAAW,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,IAAI,EAAE,CAAC;QAC3D,IAAI,CAAC,IAAI,CAAC,EAAE,EAAE,CAAC;YACb,MAAM,IAAI,oBAAoB,CAC5B,QAAQ,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,UAAU,kBAAkB,WAAW,EAAE,EACrE,MAAM,CACP,CAAC;QACJ,CAAC;QACD,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,EAAE,CAAC;YAC9B,MAAM,IAAI,oBAAoB,CAC5B,iBAAiB,WAAW,uBAAuB,EACnD,MAAM,CACP,CAAC;QACJ,CAAC;QAED,wEAAwE;QACxE,4EAA4E;QAC5E,MAAM,MAAM,GAAG,IAAI,CAAC,IAAI,EAAE,SAAS,EAAE,CAAC;QACtC,IAAI,CAAC,MAAM,EAAE,CAAC;YACZ,MAAM,IAAI,oBAAoB,CAAC,uBAAuB,EAAE,MAAM,CAAC,CAAC;QAClE,CAAC;QACD,MAAM,MAAM,GAAiB,EAAE,CAAC;QAChC,IAAI,SAAS,GAAG,CAAC,CAAC;QAClB,OAAO,IAAI,EAAE,CAAC;YACZ,MAAM,EAAE,KAAK,EAAE,IAAI,EAAE,GAAG,MAAM,MAAM,CAAC,IAAI,EAAE,CAAC;YAC5C,IAAI,IAAI;gBAAE,MAAM;YAChB,IAAI,CAAC,KAAK;gBAAE,SAAS;YACrB,SAAS,IAAI,KAAK,CAAC,UAAU,CAAC;YAC9B,IAAI,SAAS,GAAG,QAAQ,EAAE,CAAC;gBACzB,IAAI,CAAC;oBAAC,MAAM,MAAM,CAAC,MAAM,EAAE,CAAC;gBAAC,CAAC;gBAAC,MAAM,CAAC,CAAC,iBAAiB,CAAC,CAAC;gBAC1D,MAAM,IAAI,oBAAoB,CAC5B,yBAAyB,QAAQ,YAAY,EAC7C,MAAM,CACP,CAAC;YACJ,CAAC;YACD,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACrB,CAAC;QAED,MAAM,GAAG,GAAG,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,UAAU,EAAE,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC;QAChG,MAAM,IAAI,GAAG,QAAQ,CAAC,GAAG,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC;QAE7C,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,WAAW,EAAE,SAAS,EAAE,UAAU,EAAE,IAAI,CAAC,MAAM,EAAE,CAAC;IAC7E,CAAC;YAAS,CAAC;QACT,YAAY,CAAC,KAAK,CAAC,CAAC;IACtB,CAAC;AACH,CAAC,CAAC"}
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
import type { ProfileName } from '../shared/types.js';
|
|
2
|
+
export interface HtmlToMarkdownOptions {
|
|
3
|
+
baseUrl?: string;
|
|
4
|
+
profile?: ProfileName | 'auto';
|
|
5
|
+
includeFrontmatter?: boolean;
|
|
6
|
+
includeImages?: boolean;
|
|
7
|
+
}
|
|
8
|
+
export interface FetchAndConvertOptions extends HtmlToMarkdownOptions {
|
|
9
|
+
timeoutMs?: number;
|
|
10
|
+
maxBytes?: number;
|
|
11
|
+
userAgent?: string;
|
|
12
|
+
tryMdSuffixFirst?: boolean;
|
|
13
|
+
}
|
|
14
|
+
export interface ConversionResult {
|
|
15
|
+
markdown: string;
|
|
16
|
+
title: string;
|
|
17
|
+
baseUrl: string;
|
|
18
|
+
charCount: number;
|
|
19
|
+
extractedAt: string;
|
|
20
|
+
source: 'html-parse' | 'md-suffix' | 'llms-txt';
|
|
21
|
+
}
|
|
22
|
+
export declare const htmlToMarkdown: (html: string, opts?: HtmlToMarkdownOptions) => ConversionResult;
|
|
23
|
+
export declare const fetchAndConvert: (url: string, opts?: FetchAndConvertOptions) => Promise<ConversionResult>;
|
|
24
|
+
export type { ProfileName } from '../shared/types.js';
|
|
25
|
+
export { LinkedomAdapter } from './linkedom-adapter.js';
|
|
26
|
+
export { SsrfBlockedError, assertSafeUrl } from './ssrf-guard.js';
|
|
27
|
+
export { FetchProtectionError, fetchProtected } from './fetch-protected.js';
|
|
28
|
+
//# sourceMappingURL=index.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/node/index.ts"],"names":[],"mappings":"AAmBA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAC;AAEtD,MAAM,WAAW,qBAAqB;IACpC,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,OAAO,CAAC,EAAE,WAAW,GAAG,MAAM,CAAC;IAC/B,kBAAkB,CAAC,EAAE,OAAO,CAAC;IAC7B,aAAa,CAAC,EAAE,OAAO,CAAC;CACzB;AAED,MAAM,WAAW,sBAAuB,SAAQ,qBAAqB;IACnE,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,gBAAgB,CAAC,EAAE,OAAO,CAAC;CAC5B;AAED,MAAM,WAAW,gBAAgB;IAC/B,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,MAAM,CAAC;IAChB,SAAS,EAAE,MAAM,CAAC;IAClB,WAAW,EAAE,MAAM,CAAC;IACpB,MAAM,EAAE,YAAY,GAAG,WAAW,GAAG,UAAU,CAAC;CACjD;AAED,eAAO,MAAM,cAAc,GACzB,MAAM,MAAM,EACZ,OAAM,qBAA0B,KAC/B,gBA+CF,CAAC;AAqDF,eAAO,MAAM,eAAe,GAC1B,KAAK,MAAM,EACX,OAAM,sBAA2B,KAChC,OAAO,CAAC,gBAAgB,CAa1B,CAAC;AAGF,YAAY,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAC;AACtD,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AACxD,OAAO,EAAE,gBAAgB,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAC;AAClE,OAAO,EAAE,oBAAoB,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAC"}
|
|
@@ -0,0 +1,136 @@
|
|
|
1
|
+
// @page2ai/core — Node.js entry point.
|
|
2
|
+
//
|
|
3
|
+
// Public API:
|
|
4
|
+
// htmlToMarkdown(html, opts?) — convert an HTML string to Markdown
|
|
5
|
+
// fetchAndConvert(url, opts?) — SSRF-guarded fetch + convert
|
|
6
|
+
//
|
|
7
|
+
// v0.1 SCOPE: basic renderer only (headings, paragraphs, lists, links, images,
|
|
8
|
+
// code blocks, tables, frontmatter, blockquotes, inline formatting) + STATIC
|
|
9
|
+
// tab discovery (aria-controls / role=tablist) that emits '## Tab: {label}'
|
|
10
|
+
// sections instead of concatenating panels. No interactive tab-click, no
|
|
11
|
+
// lazy-load, no <details> auto-expand. For the full pipeline see the browser build.
|
|
12
|
+
//
|
|
13
|
+
// v0.2 will port the full lib/core/html-to-md.ts renderer via PageAdapter.
|
|
14
|
+
import { parseHTML } from 'linkedom';
|
|
15
|
+
import { LinkedomAdapter } from './linkedom-adapter.js';
|
|
16
|
+
import { renderBasicMarkdown } from './basic-renderer.js';
|
|
17
|
+
import { fetchProtected } from './fetch-protected.js';
|
|
18
|
+
import { assertSafeUrl } from './ssrf-guard.js';
|
|
19
|
+
export const htmlToMarkdown = (html, opts = {}) => {
|
|
20
|
+
const injectedBase = opts.baseUrl ?? 'about:blank';
|
|
21
|
+
try {
|
|
22
|
+
const { document } = parseHTML(html);
|
|
23
|
+
const adapter = new LinkedomAdapter(document, injectedBase);
|
|
24
|
+
const title = (document.querySelector('title')?.textContent || '').trim();
|
|
25
|
+
const markdown = renderBasicMarkdown(document, adapter, {
|
|
26
|
+
baseUrl: adapter.baseUrl(),
|
|
27
|
+
includeFrontmatter: opts.includeFrontmatter ?? true,
|
|
28
|
+
includeImages: opts.includeImages ?? true,
|
|
29
|
+
});
|
|
30
|
+
return {
|
|
31
|
+
markdown,
|
|
32
|
+
title,
|
|
33
|
+
baseUrl: adapter.baseUrl(),
|
|
34
|
+
charCount: markdown.length,
|
|
35
|
+
extractedAt: new Date().toISOString(),
|
|
36
|
+
source: 'html-parse',
|
|
37
|
+
};
|
|
38
|
+
}
|
|
39
|
+
catch (e) {
|
|
40
|
+
const errMsg = e instanceof Error ? e.message : String(e);
|
|
41
|
+
const fallback = [
|
|
42
|
+
'---',
|
|
43
|
+
`title: "Page2AI extraction failed"`,
|
|
44
|
+
`source: "${injectedBase}"`,
|
|
45
|
+
`captured_at: "${new Date().toISOString()}"`,
|
|
46
|
+
`extractor: "page2ai-core"`,
|
|
47
|
+
`extractor_version: "0.1.0"`,
|
|
48
|
+
`error: "${errMsg.replace(/"/g, '\\"')}"`,
|
|
49
|
+
'---',
|
|
50
|
+
'',
|
|
51
|
+
'# Page2AI extraction failed',
|
|
52
|
+
'',
|
|
53
|
+
`Could not parse HTML from ${injectedBase}.`,
|
|
54
|
+
'',
|
|
55
|
+
`Error: ${errMsg}`,
|
|
56
|
+
'',
|
|
57
|
+
].join('\n');
|
|
58
|
+
return {
|
|
59
|
+
markdown: fallback,
|
|
60
|
+
title: 'Page2AI extraction failed',
|
|
61
|
+
baseUrl: injectedBase,
|
|
62
|
+
charCount: fallback.length,
|
|
63
|
+
extractedAt: new Date().toISOString(),
|
|
64
|
+
source: 'html-parse',
|
|
65
|
+
};
|
|
66
|
+
}
|
|
67
|
+
};
|
|
68
|
+
// Wrap a raw markdown response (from .md suffix) in the frontmatter block so
|
|
69
|
+
// downstream consumers see the same shape as html-to-md output.
|
|
70
|
+
const wrapMdWithFrontmatter = (raw, url, title) => {
|
|
71
|
+
const yq = (s) => '"' + String(s).replace(/"/g, '\\"') + '"';
|
|
72
|
+
const fm = [
|
|
73
|
+
'---',
|
|
74
|
+
`title: ${yq(title)}`,
|
|
75
|
+
`source: ${yq(url)}`,
|
|
76
|
+
`captured_at: ${yq(new Date().toISOString())}`,
|
|
77
|
+
`extractor: ${yq('page2ai-core')}`,
|
|
78
|
+
`extractor_version: ${yq('0.1.0')}`,
|
|
79
|
+
`extractor_source: ${yq('md-suffix')}`,
|
|
80
|
+
'---',
|
|
81
|
+
'',
|
|
82
|
+
];
|
|
83
|
+
return fm.join('\n') + raw.replace(/^/, '').trim() + '\n';
|
|
84
|
+
};
|
|
85
|
+
// Attempt the URL.md convention (Mintlify, Anthropic docs, some others).
|
|
86
|
+
// Returns null if not applicable or on any failure — callers fall back to HTML.
|
|
87
|
+
const tryMdSuffix = async (originalUrl, opts) => {
|
|
88
|
+
const url = originalUrl.replace(/[#?].*$/, '');
|
|
89
|
+
if (url.endsWith('.md') || url.endsWith('.txt'))
|
|
90
|
+
return null;
|
|
91
|
+
const candidate = url.endsWith('/') ? `${url}index.md` : `${url}.md`;
|
|
92
|
+
try {
|
|
93
|
+
const resp = await fetchProtected(candidate, {
|
|
94
|
+
timeoutMs: opts.timeoutMs,
|
|
95
|
+
maxBytes: opts.maxBytes ?? 500 * 1024, // markdown pages should be small
|
|
96
|
+
userAgent: opts.userAgent,
|
|
97
|
+
acceptContentTypes: /^(text\/markdown|text\/plain|application\/octet-stream)/i,
|
|
98
|
+
});
|
|
99
|
+
if (!resp.text || resp.text.length < 32)
|
|
100
|
+
return null;
|
|
101
|
+
// Cheap sanity: markdown response should not look like HTML
|
|
102
|
+
if (/^\s*<(!doctype|html)/i.test(resp.text))
|
|
103
|
+
return null;
|
|
104
|
+
const title = (resp.text.match(/^#\s+(.+)$/m)?.[1] || '').trim();
|
|
105
|
+
return {
|
|
106
|
+
markdown: wrapMdWithFrontmatter(resp.text, resp.finalUrl, title),
|
|
107
|
+
title,
|
|
108
|
+
baseUrl: resp.finalUrl,
|
|
109
|
+
charCount: resp.text.length,
|
|
110
|
+
extractedAt: new Date().toISOString(),
|
|
111
|
+
source: 'md-suffix',
|
|
112
|
+
};
|
|
113
|
+
}
|
|
114
|
+
catch {
|
|
115
|
+
return null;
|
|
116
|
+
}
|
|
117
|
+
};
|
|
118
|
+
export const fetchAndConvert = async (url, opts = {}) => {
|
|
119
|
+
assertSafeUrl(url);
|
|
120
|
+
const tryMd = opts.tryMdSuffixFirst ?? true;
|
|
121
|
+
if (tryMd) {
|
|
122
|
+
const mdResult = await tryMdSuffix(url, opts);
|
|
123
|
+
if (mdResult)
|
|
124
|
+
return mdResult;
|
|
125
|
+
}
|
|
126
|
+
const resp = await fetchProtected(url, {
|
|
127
|
+
timeoutMs: opts.timeoutMs,
|
|
128
|
+
maxBytes: opts.maxBytes,
|
|
129
|
+
userAgent: opts.userAgent,
|
|
130
|
+
});
|
|
131
|
+
return htmlToMarkdown(resp.text, { ...opts, baseUrl: opts.baseUrl ?? resp.finalUrl });
|
|
132
|
+
};
|
|
133
|
+
export { LinkedomAdapter } from './linkedom-adapter.js';
|
|
134
|
+
export { SsrfBlockedError, assertSafeUrl } from './ssrf-guard.js';
|
|
135
|
+
export { FetchProtectionError, fetchProtected } from './fetch-protected.js';
|
|
136
|
+
//# sourceMappingURL=index.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/node/index.ts"],"names":[],"mappings":"AAAA,uCAAuC;AACvC,EAAE;AACF,cAAc;AACd,sEAAsE;AACtE,gEAAgE;AAChE,EAAE;AACF,+EAA+E;AAC/E,6EAA6E;AAC7E,4EAA4E;AAC5E,yEAAyE;AACzE,oFAAoF;AACpF,EAAE;AACF,2EAA2E;AAE3E,OAAO,EAAE,SAAS,EAAE,MAAM,UAAU,CAAC;AACrC,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AACxD,OAAO,EAAE,mBAAmB,EAAE,MAAM,qBAAqB,CAAC;AAC1D,OAAO,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAC;AACtD,OAAO,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAC;AA0BhD,MAAM,CAAC,MAAM,cAAc,GAAG,CAC5B,IAAY,EACZ,OAA8B,EAAE,EACd,EAAE;IACpB,MAAM,YAAY,GAAG,IAAI,CAAC,OAAO,IAAI,aAAa,CAAC;IACnD,IAAI,CAAC;QACH,MAAM,EAAE,QAAQ,EAAE,GAAG,SAAS,CAAC,IAAI,CAAC,CAAC;QACrC,MAAM,OAAO,GAAG,IAAI,eAAe,CAAC,QAA+B,EAAE,YAAY,CAAC,CAAC;QACnF,MAAM,KAAK,GAAG,CAAC,QAAQ,CAAC,aAAa,CAAC,OAAO,CAAC,EAAE,WAAW,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;QAC1E,MAAM,QAAQ,GAAG,mBAAmB,CAAC,QAA+B,EAAE,OAAO,EAAE;YAC7E,OAAO,EAAE,OAAO,CAAC,OAAO,EAAE;YAC1B,kBAAkB,EAAE,IAAI,CAAC,kBAAkB,IAAI,IAAI;YACnD,aAAa,EAAE,IAAI,CAAC,aAAa,IAAI,IAAI;SAC1C,CAAC,CAAC;QACH,OAAO;YACL,QAAQ;YACR,KAAK;YACL,OAAO,EAAE,OAAO,CAAC,OAAO,EAAE;YAC1B,SAAS,EAAE,QAAQ,CAAC,MAAM;YAC1B,WAAW,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;YACrC,MAAM,EAAE,YAAY;SACrB,CAAC;IACJ,CAAC;IAAC,OAAO,CAAU,EAAE,CAAC;QACpB,MAAM,MAAM,GAAG,CAAC,YAAY,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;QAC1D,MAAM,QAAQ,GAAG;YACf,KAAK;YACL,oCAAoC;YACpC,YAAY,YAAY,GAAG;YAC3B,iBAAiB,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,GAAG;YAC5C,2BAA2B;YAC3B,4BAA4B;YAC5B,WAAW,MAAM,CAAC,OAAO,CAAC,IAAI,EAAE,KAAK,CAAC,GAAG;YACzC,KAAK;YACL,EAAE;YACF,6BAA6B;YAC7B,EAAE;YACF,6BAA6B,YAAY,GAAG;YAC5C,EAAE;YACF,UAAU,MAAM,EAAE;YAClB,EAAE;SACH,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACb,OAAO;YACL,QAAQ,EAAE,QAAQ;YAClB,KAAK,EAAE,2BAA2B;YAClC,OAAO,EAAE,YAAY;YACrB,SAAS,EAAE,QAAQ,CAAC,MAAM;YAC1B,WAAW,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;YACrC,MAAM,EAAE,YAAY;SACrB,CAAC;IACJ,CAAC;AACH,CAAC,CAAC;AAEF,6EAA6E;AAC7E,gEAAgE;AAChE,MAAM,qBAAqB,GAAG,CAAC,GAAW,EAAE,GAAW,EAAE,KAAa,EAAU,EAAE;IAChF,MAAM,EAAE,GAAG,CAAC,CAAS,EAAU,EAAE,CAAC,GAAG,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,EAAE,KAAK,CAAC,GAAG,GAAG,CAAC;IAC7E,MAAM,EAAE,GAAG;QACT,KAAK;QACL,UAAU,EAAE,CAAC,KAAK,CAAC,EAAE;QACrB,WAAW,EAAE,CAAC,GAAG,CAAC,EAAE;QACpB,gBAAgB,EAAE,CAAC,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC,EAAE;QAC9C,cAAc,EAAE,CAAC,cAAc,CAAC,EAAE;QAClC,sBAAsB,EAAE,CAAC,OAAO,CAAC,EAAE;QACnC,qBAAqB,EAAE,CAAC,WAAW,CAAC,EAAE;QACtC,KAAK;QACL,EAAE;KACH,CAAC;IACF,OAAO,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,GAAG,CAAC,OAAO,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE,GAAG,IAAI,CAAC;AAC7D,CAAC,CAAC;AAEF,yEAAyE;AACzE,gFAAgF;AAChF,MAAM,WAAW,GAAG,KAAK,EACvB,WAAmB,EACnB,IAA4B,EACM,EAAE;IACpC,MAAM,GAAG,GAAG,WAAW,CAAC,OAAO,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC;IAC/C,IAAI,GAAG,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,GAAG,CAAC,QAAQ,CAAC,MAAM,CAAC;QAAE,OAAO,IAAI,CAAC;IAC7D,MAAM,SAAS,GAAG,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,GAAG,UAAU,CAAC,CAAC,CAAC,GAAG,GAAG,KAAK,CAAC;IACrE,IAAI,CAAC;QACH,MAAM,IAAI,GAAG,MAAM,cAAc,CAAC,SAAS,EAAE;YAC3C,SAAS,EAAE,IAAI,CAAC,SAAS;YACzB,QAAQ,EAAE,IAAI,CAAC,QAAQ,IAAI,GAAG,GAAG,IAAI,EAAI,iCAAiC;YAC1E,SAAS,EAAE,IAAI,CAAC,SAAS;YACzB,kBAAkB,EAAE,0DAA0D;SAC/E,CAAC,CAAC;QACH,IAAI,CAAC,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,IAAI,CAAC,MAAM,GAAG,EAAE;YAAE,OAAO,IAAI,CAAC;QACrD,4DAA4D;QAC5D,IAAI,uBAAuB,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC;YAAE,OAAO,IAAI,CAAC;QACzD,MAAM,KAAK,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,aAAa,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;QACjE,OAAO;YACL,QAAQ,EAAE,qBAAqB,CAAC,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,QAAQ,EAAE,KAAK,CAAC;YAChE,KAAK;YACL,OAAO,EAAE,IAAI,CAAC,QAAQ;YACtB,SAAS,EAAE,IAAI,CAAC,IAAI,CAAC,MAAM;YAC3B,WAAW,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;YACrC,MAAM,EAAE,WAAW;SACpB,CAAC;IACJ,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,eAAe,GAAG,KAAK,EAClC,GAAW,EACX,OAA+B,EAAE,EACN,EAAE;IAC7B,aAAa,CAAC,GAAG,CAAC,CAAC;IACnB,MAAM,KAAK,GAAG,IAAI,CAAC,gBAAgB,IAAI,IAAI,CAAC;IAC5C,IAAI,KAAK,EAAE,CAAC;QACV,MAAM,QAAQ,GAAG,MAAM,WAAW,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC;QAC9C,IAAI,QAAQ;YAAE,OAAO,QAAQ,CAAC;IAChC,CAAC;IACD,MAAM,IAAI,GAAG,MAAM,cAAc,CAAC,GAAG,EAAE;QACrC,SAAS,EAAE,IAAI,CAAC,SAAS;QACzB,QAAQ,EAAE,IAAI,CAAC,QAAQ;QACvB,SAAS,EAAE,IAAI,CAAC,SAAS;KAC1B,CAAC,CAAC;IACH,OAAO,cAAc,CAAC,IAAI,CAAC,IAAI,EAAE,EAAE,GAAG,IAAI,EAAE,OAAO,EAAE,IAAI,CAAC,OAAO,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC,CAAC;AACxF,CAAC,CAAC;AAIF,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AACxD,OAAO,EAAE,gBAAgB,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAC;AAClE,OAAO,EAAE,oBAAoB,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAC"}
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
import type { AdapterCapabilities, ComputedStyleLike, PageAdapter } from '../shared/page-adapter.js';
|
|
2
|
+
export declare class LinkedomAdapter implements PageAdapter {
|
|
3
|
+
readonly capabilities: Readonly<AdapterCapabilities>;
|
|
4
|
+
private readonly doc;
|
|
5
|
+
private readonly base;
|
|
6
|
+
constructor(doc: Document, baseUrl: string);
|
|
7
|
+
static resolveBase(doc: Document, injected: string): string;
|
|
8
|
+
getComputedStyle(_el: Element): ComputedStyleLike;
|
|
9
|
+
getPseudoContent(_el: Element, _pseudo: '::before' | '::after'): string;
|
|
10
|
+
innerText(el: Element): string;
|
|
11
|
+
shadowRoot(_el: Element): null;
|
|
12
|
+
currentSrc(_img: Element): string;
|
|
13
|
+
baseUrl(): string;
|
|
14
|
+
cssEscape(s: string): string;
|
|
15
|
+
document(): Document;
|
|
16
|
+
}
|
|
17
|
+
//# sourceMappingURL=linkedom-adapter.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"linkedom-adapter.d.ts","sourceRoot":"","sources":["../../src/node/linkedom-adapter.ts"],"names":[],"mappings":"AAMA,OAAO,KAAK,EACV,mBAAmB,EACnB,iBAAiB,EACjB,WAAW,EACZ,MAAM,2BAA2B,CAAC;AAoBnC,qBAAa,eAAgB,YAAW,WAAW;IACjD,QAAQ,CAAC,YAAY,EAAE,QAAQ,CAAC,mBAAmB,CAAC,CAMjD;IAEH,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAW;IAC/B,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAS;gBAElB,GAAG,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM;IAK1C,MAAM,CAAC,WAAW,CAAC,GAAG,EAAE,QAAQ,EAAE,QAAQ,EAAE,MAAM,GAAG,MAAM;IAU3D,gBAAgB,CAAC,GAAG,EAAE,OAAO,GAAG,iBAAiB;IAIjD,gBAAgB,CAAC,GAAG,EAAE,OAAO,EAAE,OAAO,EAAE,UAAU,GAAG,SAAS,GAAG,MAAM;IAavE,SAAS,CAAC,EAAE,EAAE,OAAO,GAAG,MAAM;IAyB9B,UAAU,CAAC,GAAG,EAAE,OAAO,GAAG,IAAI;IAI9B,UAAU,CAAC,IAAI,EAAE,OAAO,GAAG,MAAM;IAIjC,OAAO,IAAI,MAAM;IAIjB,SAAS,CAAC,CAAC,EAAE,MAAM,GAAG,MAAM;IAI5B,QAAQ,IAAI,QAAQ;CAGrB"}
|
|
@@ -0,0 +1,112 @@
|
|
|
1
|
+
// LinkedomAdapter: PageAdapter implementation for Node.js.
|
|
2
|
+
// Uses linkedom for HTML parsing. Silently degrades browser-only features
|
|
3
|
+
// AND exposes those degradations via capabilities.can* flags so the renderer
|
|
4
|
+
// can branch on real fidelity rather than trusting stub values.
|
|
5
|
+
import cssEscapePolyfill from 'css.escape';
|
|
6
|
+
const DEFAULT_STYLE = Object.freeze({
|
|
7
|
+
display: '',
|
|
8
|
+
visibility: '',
|
|
9
|
+
opacity: '1',
|
|
10
|
+
fontSize: '16px',
|
|
11
|
+
fontWeight: '400',
|
|
12
|
+
});
|
|
13
|
+
const BLOCK_TAGS = new Set([
|
|
14
|
+
'ADDRESS', 'ARTICLE', 'ASIDE', 'BLOCKQUOTE', 'DETAILS', 'DIALOG',
|
|
15
|
+
'DD', 'DIV', 'DL', 'DT', 'FIELDSET', 'FIGCAPTION', 'FIGURE',
|
|
16
|
+
'FOOTER', 'FORM', 'HEADER', 'HR', 'LI', 'MAIN', 'NAV', 'OL',
|
|
17
|
+
'P', 'PRE', 'SECTION', 'TABLE', 'TBODY', 'TD', 'TFOOT', 'TH',
|
|
18
|
+
'THEAD', 'TR', 'UL', 'H1', 'H2', 'H3', 'H4', 'H5', 'H6',
|
|
19
|
+
]);
|
|
20
|
+
const HIDDEN_TAGS = new Set(['SCRIPT', 'STYLE', 'NOSCRIPT', 'TEMPLATE']);
|
|
21
|
+
export class LinkedomAdapter {
|
|
22
|
+
capabilities = Object.freeze({
|
|
23
|
+
canComputeStyle: false,
|
|
24
|
+
canPseudoContent: false,
|
|
25
|
+
canShadowDom: false,
|
|
26
|
+
canInnerText: true, // via polyfill
|
|
27
|
+
canCurrentSrc: false,
|
|
28
|
+
});
|
|
29
|
+
doc;
|
|
30
|
+
base;
|
|
31
|
+
constructor(doc, baseUrl) {
|
|
32
|
+
this.doc = doc;
|
|
33
|
+
this.base = LinkedomAdapter.resolveBase(doc, baseUrl);
|
|
34
|
+
}
|
|
35
|
+
static resolveBase(doc, injected) {
|
|
36
|
+
const baseHref = doc.querySelector('base[href]')?.getAttribute('href');
|
|
37
|
+
if (!baseHref)
|
|
38
|
+
return injected;
|
|
39
|
+
try {
|
|
40
|
+
return new URL(baseHref, injected).href;
|
|
41
|
+
}
|
|
42
|
+
catch {
|
|
43
|
+
return injected;
|
|
44
|
+
}
|
|
45
|
+
}
|
|
46
|
+
getComputedStyle(_el) {
|
|
47
|
+
return DEFAULT_STYLE;
|
|
48
|
+
}
|
|
49
|
+
getPseudoContent(_el, _pseudo) {
|
|
50
|
+
return '';
|
|
51
|
+
}
|
|
52
|
+
// Best-effort innerText polyfill. linkedom textContent includes <script>,
|
|
53
|
+
// hidden aria-hidden text, and ignores <br> line breaks, which corrupts
|
|
54
|
+
// tables and forms. This walk:
|
|
55
|
+
// - skips <script>, <style>, <noscript>, <template>
|
|
56
|
+
// - skips aria-hidden="true"
|
|
57
|
+
// - best-effort skips inline style display:none / visibility:hidden
|
|
58
|
+
// - emits '\n' for BR
|
|
59
|
+
// - wraps BLOCK tags with '\n' padding
|
|
60
|
+
// - preserves TEXT_NODE content
|
|
61
|
+
innerText(el) {
|
|
62
|
+
const parts = [];
|
|
63
|
+
const walk = (node) => {
|
|
64
|
+
if (node.nodeType === 3 /* TEXT_NODE */) {
|
|
65
|
+
parts.push(node.textContent || '');
|
|
66
|
+
return;
|
|
67
|
+
}
|
|
68
|
+
if (node.nodeType !== 1 /* ELEMENT_NODE */)
|
|
69
|
+
return;
|
|
70
|
+
const e = node;
|
|
71
|
+
const tag = e.tagName;
|
|
72
|
+
if (HIDDEN_TAGS.has(tag))
|
|
73
|
+
return;
|
|
74
|
+
if (e.getAttribute('aria-hidden') === 'true')
|
|
75
|
+
return;
|
|
76
|
+
const inlineStyle = e.getAttribute('style') || '';
|
|
77
|
+
if (/display\s*:\s*none/i.test(inlineStyle))
|
|
78
|
+
return;
|
|
79
|
+
if (/visibility\s*:\s*hidden/i.test(inlineStyle))
|
|
80
|
+
return;
|
|
81
|
+
if (tag === 'BR') {
|
|
82
|
+
parts.push('\n');
|
|
83
|
+
return;
|
|
84
|
+
}
|
|
85
|
+
const isBlock = BLOCK_TAGS.has(tag);
|
|
86
|
+
if (isBlock)
|
|
87
|
+
parts.push('\n');
|
|
88
|
+
for (const c of e.childNodes)
|
|
89
|
+
walk(c);
|
|
90
|
+
if (isBlock)
|
|
91
|
+
parts.push('\n');
|
|
92
|
+
};
|
|
93
|
+
walk(el);
|
|
94
|
+
return parts.join('').replace(/[ \t]+/g, ' ').replace(/\n{3,}/g, '\n\n').trim();
|
|
95
|
+
}
|
|
96
|
+
shadowRoot(_el) {
|
|
97
|
+
return null;
|
|
98
|
+
}
|
|
99
|
+
currentSrc(_img) {
|
|
100
|
+
return '';
|
|
101
|
+
}
|
|
102
|
+
baseUrl() {
|
|
103
|
+
return this.base;
|
|
104
|
+
}
|
|
105
|
+
cssEscape(s) {
|
|
106
|
+
return cssEscapePolyfill(s);
|
|
107
|
+
}
|
|
108
|
+
document() {
|
|
109
|
+
return this.doc;
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
//# sourceMappingURL=linkedom-adapter.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"linkedom-adapter.js","sourceRoot":"","sources":["../../src/node/linkedom-adapter.ts"],"names":[],"mappings":"AAAA,2DAA2D;AAC3D,0EAA0E;AAC1E,6EAA6E;AAC7E,gEAAgE;AAEhE,OAAO,iBAAiB,MAAM,YAAY,CAAC;AAO3C,MAAM,aAAa,GAAsB,MAAM,CAAC,MAAM,CAAC;IACrD,OAAO,EAAE,EAAE;IACX,UAAU,EAAE,EAAE;IACd,OAAO,EAAE,GAAG;IACZ,QAAQ,EAAE,MAAM;IAChB,UAAU,EAAE,KAAK;CAClB,CAAC,CAAC;AAEH,MAAM,UAAU,GAAG,IAAI,GAAG,CAAC;IACzB,SAAS,EAAE,SAAS,EAAE,OAAO,EAAE,YAAY,EAAE,SAAS,EAAE,QAAQ;IAChE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,IAAI,EAAE,UAAU,EAAE,YAAY,EAAE,QAAQ;IAC3D,QAAQ,EAAE,MAAM,EAAE,QAAQ,EAAE,IAAI,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,IAAI;IAC3D,GAAG,EAAE,KAAK,EAAE,SAAS,EAAE,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,IAAI;IAC5D,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI;CACxD,CAAC,CAAC;AAEH,MAAM,WAAW,GAAG,IAAI,GAAG,CAAC,CAAC,QAAQ,EAAE,OAAO,EAAE,UAAU,EAAE,UAAU,CAAC,CAAC,CAAC;AAEzE,MAAM,OAAO,eAAe;IACjB,YAAY,GAAkC,MAAM,CAAC,MAAM,CAAC;QACnE,eAAe,EAAE,KAAK;QACtB,gBAAgB,EAAE,KAAK;QACvB,YAAY,EAAE,KAAK;QACnB,YAAY,EAAE,IAAI,EAAM,eAAe;QACvC,aAAa,EAAE,KAAK;KACrB,CAAC,CAAC;IAEc,GAAG,CAAW;IACd,IAAI,CAAS;IAE9B,YAAY,GAAa,EAAE,OAAe;QACxC,IAAI,CAAC,GAAG,GAAG,GAAG,CAAC;QACf,IAAI,CAAC,IAAI,GAAG,eAAe,CAAC,WAAW,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;IACxD,CAAC;IAED,MAAM,CAAC,WAAW,CAAC,GAAa,EAAE,QAAgB;QAChD,MAAM,QAAQ,GAAG,GAAG,CAAC,aAAa,CAAC,YAAY,CAAC,EAAE,YAAY,CAAC,MAAM,CAAC,CAAC;QACvE,IAAI,CAAC,QAAQ;YAAE,OAAO,QAAQ,CAAC;QAC/B,IAAI,CAAC;YACH,OAAO,IAAI,GAAG,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC,IAAI,CAAC;QAC1C,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,QAAQ,CAAC;QAClB,CAAC;IACH,CAAC;IAED,gBAAgB,CAAC,GAAY;QAC3B,OAAO,aAAa,CAAC;IACvB,CAAC;IAED,gBAAgB,CAAC,GAAY,EAAE,OAA+B;QAC5D,OAAO,EAAE,CAAC;IACZ,CAAC;IAED,0EAA0E;IAC1E,wEAAwE;IACxE,+BAA+B;IAC/B,qDAAqD;IACrD,8BAA8B;IAC9B,qEAAqE;IACrE,uBAAuB;IACvB,wCAAwC;IACxC,iCAAiC;IACjC,SAAS,CAAC,EAAW;QACnB,MAAM,KAAK,GAAa,EAAE,CAAC;QAC3B,MAAM,IAAI,GAAG,CAAC,IAAU,EAAQ,EAAE;YAChC,IAAI,IAAI,CAAC,QAAQ,KAAK,CAAC,CAAC,eAAe,EAAE,CAAC;gBACxC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,WAAW,IAAI,EAAE,CAAC,CAAC;gBACnC,OAAO;YACT,CAAC;YACD,IAAI,IAAI,CAAC,QAAQ,KAAK,CAAC,CAAC,kBAAkB;gBAAE,OAAO;YACnD,MAAM,CAAC,GAAG,IAAe,CAAC;YAC1B,MAAM,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC;YACtB,IAAI,WAAW,CAAC,GAAG,CAAC,GAAG,CAAC;gBAAE,OAAO;YACjC,IAAI,CAAC,CAAC,YAAY,CAAC,aAAa,CAAC,KAAK,MAAM;gBAAE,OAAO;YACrD,MAAM,WAAW,GAAG,CAAC,CAAC,YAAY,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC;YAClD,IAAI,qBAAqB,CAAC,IAAI,CAAC,WAAW,CAAC;gBAAE,OAAO;YACpD,IAAI,0BAA0B,CAAC,IAAI,CAAC,WAAW,CAAC;gBAAE,OAAO;YACzD,IAAI,GAAG,KAAK,IAAI,EAAE,CAAC;gBAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;gBAAC,OAAO;YAAC,CAAC;YAC/C,MAAM,OAAO,GAAG,UAAU,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;YACpC,IAAI,OAAO;gBAAE,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAC9B,KAAK,MAAM,CAAC,IAAI,CAAC,CAAC,UAAU;gBAAE,IAAI,CAAC,CAAC,CAAC,CAAC;YACtC,IAAI,OAAO;gBAAE,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAChC,CAAC,CAAC;QACF,IAAI,CAAC,EAAE,CAAC,CAAC;QACT,OAAO,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,OAAO,CAAC,SAAS,EAAE,GAAG,CAAC,CAAC,OAAO,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC,IAAI,EAAE,CAAC;IAClF,CAAC;IAED,UAAU,CAAC,GAAY;QACrB,OAAO,IAAI,CAAC;IACd,CAAC;IAED,UAAU,CAAC,IAAa;QACtB,OAAO,EAAE,CAAC;IACZ,CAAC;IAED,OAAO;QACL,OAAO,IAAI,CAAC,IAAI,CAAC;IACnB,CAAC;IAED,SAAS,CAAC,CAAS;QACjB,OAAO,iBAAiB,CAAC,CAAC,CAAC,CAAC;IAC9B,CAAC;IAED,QAAQ;QACN,OAAO,IAAI,CAAC,GAAG,CAAC;IAClB,CAAC;CACF"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"ssrf-guard.d.ts","sourceRoot":"","sources":["../../src/node/ssrf-guard.ts"],"names":[],"mappings":"AAoCA,qBAAa,gBAAiB,SAAQ,KAAK;aACG,GAAG,EAAE,MAAM;gBAA3C,MAAM,EAAE,MAAM,EAAkB,GAAG,EAAE,MAAM;CAIxD;AAQD,eAAO,MAAM,aAAa,GAAI,QAAQ,MAAM,KAAG,GA4B9C,CAAC"}
|
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
// SSRF guard for the Node MCP server: block outbound requests to loopback,
|
|
2
|
+
// private ranges, cloud metadata endpoints, and dangerous schemes. Callers
|
|
3
|
+
// invoke assertSafeUrl(url) before fetching; it throws on any block hit.
|
|
4
|
+
//
|
|
5
|
+
// KNOWN LIMITATION (v0.1): only IP-literal hosts are checked. DNS-rebinding
|
|
6
|
+
// attacks (hostname resolves to 127.0.0.1) require a resolve+revalidate step;
|
|
7
|
+
// planned for v0.2 via `dns.lookup`. Document in README.
|
|
8
|
+
const PRIVATE_IPV4_PATTERNS = [
|
|
9
|
+
/^127\./, // 127.0.0.0/8 loopback
|
|
10
|
+
/^10\./, // 10.0.0.0/8 private
|
|
11
|
+
/^169\.254\./, // 169.254.0.0/16 link-local (AWS/GCP metadata)
|
|
12
|
+
/^172\.(1[6-9]|2[0-9]|3[0-1])\./, // 172.16.0.0/12 private
|
|
13
|
+
/^192\.168\./, // 192.168.0.0/16 private
|
|
14
|
+
/^0\./, // 0.0.0.0/8 current network
|
|
15
|
+
/^100\.(6[4-9]|[7-9][0-9]|1[01][0-9]|12[0-7])\./, // 100.64/10 CGNAT
|
|
16
|
+
];
|
|
17
|
+
const BLOCKED_IPV6_PREFIXES = [
|
|
18
|
+
'::1', // loopback
|
|
19
|
+
'::ffff:', // IPv4-mapped
|
|
20
|
+
'fe80:', // link-local
|
|
21
|
+
'fc', // unique local (fc00::/7)
|
|
22
|
+
'fd', // unique local
|
|
23
|
+
];
|
|
24
|
+
const BLOCKED_HOSTNAMES = new Set([
|
|
25
|
+
'localhost',
|
|
26
|
+
'localhost.',
|
|
27
|
+
'0.0.0.0',
|
|
28
|
+
'metadata.google.internal',
|
|
29
|
+
'metadata',
|
|
30
|
+
]);
|
|
31
|
+
const ALLOWED_SCHEMES = new Set(['http:', 'https:']);
|
|
32
|
+
export class SsrfBlockedError extends Error {
|
|
33
|
+
url;
|
|
34
|
+
constructor(reason, url) {
|
|
35
|
+
super(`SSRF guard blocked ${url}: ${reason}`);
|
|
36
|
+
this.url = url;
|
|
37
|
+
this.name = 'SsrfBlockedError';
|
|
38
|
+
}
|
|
39
|
+
}
|
|
40
|
+
const isIpv4Literal = (host) => /^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$/.test(host);
|
|
41
|
+
const isIpv6Literal = (host) => /^\[?[0-9a-fA-F:]+\]?$/.test(host) && host.includes(':');
|
|
42
|
+
export const assertSafeUrl = (rawUrl) => {
|
|
43
|
+
let parsed;
|
|
44
|
+
try {
|
|
45
|
+
parsed = new URL(rawUrl);
|
|
46
|
+
}
|
|
47
|
+
catch {
|
|
48
|
+
throw new SsrfBlockedError('malformed URL', rawUrl);
|
|
49
|
+
}
|
|
50
|
+
if (!ALLOWED_SCHEMES.has(parsed.protocol)) {
|
|
51
|
+
throw new SsrfBlockedError(`scheme "${parsed.protocol}" not in {http:, https:}`, rawUrl);
|
|
52
|
+
}
|
|
53
|
+
const host = parsed.hostname.toLowerCase().replace(/^\[|\]$/g, '');
|
|
54
|
+
if (BLOCKED_HOSTNAMES.has(host)) {
|
|
55
|
+
throw new SsrfBlockedError(`hostname "${host}" blocked`, rawUrl);
|
|
56
|
+
}
|
|
57
|
+
if (isIpv4Literal(host)) {
|
|
58
|
+
for (const p of PRIVATE_IPV4_PATTERNS) {
|
|
59
|
+
if (p.test(host)) {
|
|
60
|
+
throw new SsrfBlockedError(`IPv4 "${host}" in private range`, rawUrl);
|
|
61
|
+
}
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
else if (isIpv6Literal(host)) {
|
|
65
|
+
for (const prefix of BLOCKED_IPV6_PREFIXES) {
|
|
66
|
+
if (host.startsWith(prefix)) {
|
|
67
|
+
throw new SsrfBlockedError(`IPv6 "${host}" in blocked range`, rawUrl);
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
}
|
|
71
|
+
return parsed;
|
|
72
|
+
};
|
|
73
|
+
//# sourceMappingURL=ssrf-guard.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"ssrf-guard.js","sourceRoot":"","sources":["../../src/node/ssrf-guard.ts"],"names":[],"mappings":"AAAA,2EAA2E;AAC3E,2EAA2E;AAC3E,yEAAyE;AACzE,EAAE;AACF,4EAA4E;AAC5E,8EAA8E;AAC9E,yDAAyD;AAEzD,MAAM,qBAAqB,GAAa;IACtC,QAAQ,EAA8B,uBAAuB;IAC7D,OAAO,EAA+B,qBAAqB;IAC3D,aAAa,EAAyB,+CAA+C;IACrF,gCAAgC,EAAM,wBAAwB;IAC9D,aAAa,EAAyB,yBAAyB;IAC/D,MAAM,EAAgC,4BAA4B;IAClE,gDAAgD,EAAG,kBAAkB;CACtE,CAAC;AAEF,MAAM,qBAAqB,GAAa;IACtC,KAAK,EAAO,WAAW;IACvB,SAAS,EAAG,cAAc;IAC1B,OAAO,EAAK,aAAa;IACzB,IAAI,EAAQ,0BAA0B;IACtC,IAAI,EAAQ,eAAe;CAC5B,CAAC;AAEF,MAAM,iBAAiB,GAAG,IAAI,GAAG,CAAC;IAChC,WAAW;IACX,YAAY;IACZ,SAAS;IACT,0BAA0B;IAC1B,UAAU;CACX,CAAC,CAAC;AAEH,MAAM,eAAe,GAAG,IAAI,GAAG,CAAC,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC,CAAC;AAErD,MAAM,OAAO,gBAAiB,SAAQ,KAAK;IACG;IAA5C,YAAY,MAAc,EAAkB,GAAW;QACrD,KAAK,CAAC,sBAAsB,GAAG,KAAK,MAAM,EAAE,CAAC,CAAC;QADJ,QAAG,GAAH,GAAG,CAAQ;QAErD,IAAI,CAAC,IAAI,GAAG,kBAAkB,CAAC;IACjC,CAAC;CACF;AAED,MAAM,aAAa,GAAG,CAAC,IAAY,EAAW,EAAE,CAC9C,sCAAsC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAEpD,MAAM,aAAa,GAAG,CAAC,IAAY,EAAW,EAAE,CAC9C,uBAAuB,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC;AAE3D,MAAM,CAAC,MAAM,aAAa,GAAG,CAAC,MAAc,EAAO,EAAE;IACnD,IAAI,MAAW,CAAC;IAChB,IAAI,CAAC;QACH,MAAM,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,CAAC;IAC3B,CAAC;IAAC,MAAM,CAAC;QACP,MAAM,IAAI,gBAAgB,CAAC,eAAe,EAAE,MAAM,CAAC,CAAC;IACtD,CAAC;IACD,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE,CAAC;QAC1C,MAAM,IAAI,gBAAgB,CAAC,WAAW,MAAM,CAAC,QAAQ,0BAA0B,EAAE,MAAM,CAAC,CAAC;IAC3F,CAAC;IACD,MAAM,IAAI,GAAG,MAAM,CAAC,QAAQ,CAAC,WAAW,EAAE,CAAC,OAAO,CAAC,UAAU,EAAE,EAAE,CAAC,CAAC;IACnE,IAAI,iBAAiB,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;QAChC,MAAM,IAAI,gBAAgB,CAAC,aAAa,IAAI,WAAW,EAAE,MAAM,CAAC,CAAC;IACnE,CAAC;IACD,IAAI,aAAa,CAAC,IAAI,CAAC,EAAE,CAAC;QACxB,KAAK,MAAM,CAAC,IAAI,qBAAqB,EAAE,CAAC;YACtC,IAAI,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;gBACjB,MAAM,IAAI,gBAAgB,CAAC,SAAS,IAAI,oBAAoB,EAAE,MAAM,CAAC,CAAC;YACxE,CAAC;QACH,CAAC;IACH,CAAC;SAAM,IAAI,aAAa,CAAC,IAAI,CAAC,EAAE,CAAC;QAC/B,KAAK,MAAM,MAAM,IAAI,qBAAqB,EAAE,CAAC;YAC3C,IAAI,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,EAAE,CAAC;gBAC5B,MAAM,IAAI,gBAAgB,CAAC,SAAS,IAAI,oBAAoB,EAAE,MAAM,CAAC,CAAC;YACxE,CAAC;QACH,CAAC;IACH,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC,CAAC"}
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
import type { ExtractorConfig } from './types.js';
|
|
2
|
+
export declare const INDENT = " ";
|
|
3
|
+
export declare const DEFAULTS: ExtractorConfig;
|
|
4
|
+
export declare const BLOCK_TAGS: Set<string>;
|
|
5
|
+
export declare const INLINE_FORMATTING_TAGS: Set<string>;
|
|
6
|
+
export declare const SOCIAL_DOMAIN_RE: RegExp;
|
|
7
|
+
export declare const PII_KEY_PATTERNS: RegExp[];
|
|
8
|
+
export declare const PII_VALUE_PATTERNS: Array<{
|
|
9
|
+
name: string;
|
|
10
|
+
re: RegExp;
|
|
11
|
+
}>;
|
|
12
|
+
//# sourceMappingURL=constants.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"constants.d.ts","sourceRoot":"","sources":["../../src/shared/constants.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,YAAY,CAAC;AAGlD,eAAO,MAAM,MAAM,QAAQ,CAAC;AAE5B,eAAO,MAAM,QAAQ,EAAE,eAiItB,CAAC;AAEF,eAAO,MAAM,UAAU,aAMrB,CAAC;AAEH,eAAO,MAAM,sBAAsB,aAIjC,CAAC;AAEH,eAAO,MAAM,gBAAgB,QACgT,CAAC;AAE9U,eAAO,MAAM,gBAAgB,UAM5B,CAAC;AAEF,eAAO,MAAM,kBAAkB,EAAE,KAAK,CAAC;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,EAAE,EAAE,MAAM,CAAA;CAAE,CAQlE,CAAC"}
|