@rivi-software/agentic-seo-check 0.1.0 → 0.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +1 -1
- package/README.md +65 -15
- package/dist/brand.d.ts +11 -0
- package/dist/brand.js +22 -0
- package/dist/brand.js.map +1 -0
- package/dist/cli.js +125 -80
- package/dist/cli.js.map +1 -1
- package/dist/core/audit.d.ts +16 -16
- package/dist/core/audit.js +363 -248
- package/dist/core/audit.js.map +1 -1
- package/dist/core/concurrency.d.ts +4 -4
- package/dist/core/concurrency.js +33 -23
- package/dist/core/concurrency.js.map +1 -1
- package/dist/core/crawl.d.ts +5 -5
- package/dist/core/crawl.js +250 -121
- package/dist/core/crawl.js.map +1 -1
- package/dist/core/fetch.d.ts +5 -5
- package/dist/core/fetch.js +151 -100
- package/dist/core/fetch.js.map +1 -1
- package/dist/core/html/dom.d.ts +13 -0
- package/dist/core/html/dom.js +17 -0
- package/dist/core/html/dom.js.map +1 -0
- package/dist/core/html/extract.d.ts +2 -2
- package/dist/core/html/extract.js +168 -111
- package/dist/core/html/extract.js.map +1 -1
- package/dist/core/html/structured-data.d.ts +16 -6
- package/dist/core/html/structured-data.js +180 -145
- package/dist/core/html/structured-data.js.map +1 -1
- package/dist/core/html/text.d.ts +14 -14
- package/dist/core/html/text.js +110 -107
- package/dist/core/html/text.js.map +1 -1
- package/dist/core/lighthouse.d.ts +26 -25
- package/dist/core/lighthouse.js +181 -105
- package/dist/core/lighthouse.js.map +1 -1
- package/dist/core/performance-rules.d.ts +2 -2
- package/dist/core/performance-rules.js +105 -33
- package/dist/core/performance-rules.js.map +1 -1
- package/dist/core/render-rules.d.ts +5 -4
- package/dist/core/render-rules.js +129 -62
- package/dist/core/render-rules.js.map +1 -1
- package/dist/core/render.d.ts +2 -2
- package/dist/core/render.js +29 -23
- package/dist/core/render.js.map +1 -1
- package/dist/core/report/markdown.d.ts +2 -2
- package/dist/core/report/markdown.js +110 -131
- package/dist/core/report/markdown.js.map +1 -1
- package/dist/core/robots.d.ts +5 -5
- package/dist/core/robots.js +31 -16
- package/dist/core/robots.js.map +1 -1
- package/dist/core/rules/canonical.d.ts +1 -1
- package/dist/core/rules/canonical.js +93 -57
- package/dist/core/rules/canonical.js.map +1 -1
- package/dist/core/rules/content.js +40 -26
- package/dist/core/rules/content.js.map +1 -1
- package/dist/core/rules/document.js +64 -41
- package/dist/core/rules/document.js.map +1 -1
- package/dist/core/rules/headings.js +58 -38
- package/dist/core/rules/headings.js.map +1 -1
- package/dist/core/rules/hreflang.js +70 -38
- package/dist/core/rules/hreflang.js.map +1 -1
- package/dist/core/rules/images.js +44 -26
- package/dist/core/rules/images.js.map +1 -1
- package/dist/core/rules/index.d.ts +6 -6
- package/dist/core/rules/index.js +36 -35
- package/dist/core/rules/index.js.map +1 -1
- package/dist/core/rules/indexing.d.ts +2 -2
- package/dist/core/rules/indexing.js +48 -26
- package/dist/core/rules/indexing.js.map +1 -1
- package/dist/core/rules/keyword.js +84 -47
- package/dist/core/rules/keyword.js.map +1 -1
- package/dist/core/rules/links.js +72 -37
- package/dist/core/rules/links.js.map +1 -1
- package/dist/core/rules/meta.js +55 -34
- package/dist/core/rules/meta.js.map +1 -1
- package/dist/core/rules/response.js +75 -41
- package/dist/core/rules/response.js.map +1 -1
- package/dist/core/rules/rule.d.ts +6 -6
- package/dist/core/rules/rule.js +36 -36
- package/dist/core/rules/rule.js.map +1 -1
- package/dist/core/rules/social.js +86 -45
- package/dist/core/rules/social.js.map +1 -1
- package/dist/core/rules/structured-data.d.ts +6 -4
- package/dist/core/rules/structured-data.js +314 -102
- package/dist/core/rules/structured-data.js.map +1 -1
- package/dist/core/rules/title.js +48 -33
- package/dist/core/rules/title.js.map +1 -1
- package/dist/core/score.d.ts +3 -3
- package/dist/core/score.js +11 -9
- package/dist/core/score.js.map +1 -1
- package/dist/core/site-rules.d.ts +3 -3
- package/dist/core/site-rules.js +409 -198
- package/dist/core/site-rules.js.map +1 -1
- package/dist/core/sitemap.d.ts +5 -5
- package/dist/core/sitemap.js +109 -64
- package/dist/core/sitemap.js.map +1 -1
- package/dist/core/static.d.ts +2 -2
- package/dist/core/static.js +209 -112
- package/dist/core/static.js.map +1 -1
- package/dist/core/summary.d.ts +2 -2
- package/dist/core/summary.js +11 -6
- package/dist/core/summary.js.map +1 -1
- package/dist/core/types.d.ts +14 -14
- package/dist/core/types.js +15 -8
- package/dist/core/types.js.map +1 -1
- package/dist/core/url.d.ts +11 -11
- package/dist/core/url.js +122 -70
- package/dist/core/url.js.map +1 -1
- package/dist/index.d.ts +11 -11
- package/dist/index.js +9 -9
- package/dist/index.js.map +1 -1
- package/dist/mcp-bin.js +7 -4
- package/dist/mcp-bin.js.map +1 -1
- package/dist/mcp.d.ts +3 -3
- package/dist/mcp.js +312 -122
- package/dist/mcp.js.map +1 -1
- package/dist/version.d.ts +0 -1
- package/dist/version.js +4 -2
- package/dist/version.js.map +1 -1
- package/package.json +27 -9
package/dist/core/crawl.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"crawl.js","sourceRoot":"","sources":["../../src/core/crawl.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAChD,OAAO,EAAE,kBAAkB,EAAE,SAAS,EAAE,cAAc,EAAE,QAAQ,EAAE,cAAc,EAAqB,MAAM,YAAY,CAAC;AACxH,OAAO,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAC5C,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAC5C,OAAO,EAAE,WAAW,EAAE,MAAM,sBAAsB,CAAC;AAGnD,OAAO,EAAE,MAAM,EAAE,UAAU,EAAE,aAAa,EAAE,YAAY,EAAE,QAAQ,EAAE,MAAM,UAAU,CAAC;AACrF,OAAO,EAAE,aAAa,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAC;AA8CxD,MAAM,CAAC,KAAK,UAAU,SAAS,CAAC,UAAkB,EAAE,OAAqB;IACvE,MAAM,OAAO,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC;IAClC,MAAM,QAAQ,GAAG,OAAO,CAAC,UAAU,IAAI,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;IACzD,MAAM,QAAQ,GAAG,YAAY,CAAC,UAAU,CAAC,IAAI,UAAU,CAAC;IACxD,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC;IACxC,MAAM,SAAS,GAAG,MAAM,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC;IACzC,MAAM,SAAS,GAAG,OAAO,CAAC,KAAK,CAAC,SAAS,IAAI,kBAAkB,CAAC;IAEhE,QAAQ,CAAC,EAAE,KAAK,EAAE,QAAQ,EAAE,OAAO,EAAE,YAAY,MAAM,aAAa,EAAE,CAAC,CAAC;IACxE,MAAM,MAAM,GAAG,MAAM,aAAa,CAAC,MAAM,EAAE,SAAS,EAAE,OAAO,CAAC,KAAK,CAAC,CAAC;IACrE,MAAM,MAAM,GAAG,OAAO,CAAC,aAAa,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC;IAElE,QAAQ,CAAC,EAAE,KAAK,EAAE,SAAS,EAAE,OAAO,EAAE,mBAAmB,EAAE,CAAC,CAAC;IAC7D,MAAM,iBAAiB,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,MAAM,CAAC,IAAI,CAAC,QAAQ,EAAE,IAAI,GAAG,CAAC,cAAc,EAAE,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IACxG,MAAM,QAAQ,GAAG,MAAM,YAAY,CAAC,iBAAiB,EAAE,OAAO,CAAC,KAAK,CAAC,CAAC;IAEtE,MAAM,OAAO,GAAG,IAAI,GAAG,CAAS,CAAC,QAAQ,CAAC,CAAC,CAAC;IAC5C,MAAM,UAAU,GAAG,IAAI,GAAG,CAAS,CAAC,QAAQ,CAAC,CAAC,CAAC;IAC/C,MAAM,UAAU,GAAG,IAAI,GAAG,EAAqB,CAAC;IAChD,MAAM,WAAW,GAAG,IAAI,GAAG,EAAuB,CAAC;IACnD,MAAM,eAAe,GAAG,IAAI,GAAG,EAAuB,CAAC;IACvD,MAAM,OAAO,GAAiB,EAAE,CAAC;IACjC,MAAM,OAAO,GAAa,EAAE,CAAC;IAC7B,IAAI,UAAU,GAAG,CAAC,CAAC;IAEnB,MAAM,OAAO,GAAG,CAAC,GAAW,EAAW,EAAE;QACvC,IAAI,OAAO,CAAC,eAAe,CAAC,MAAM,IAAI,CAAC,OAAO,CAAC,eAAe,CAAC,IAAI,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YAAE,OAAO,KAAK,CAAC;QACxG,IAAI,OAAO,CAAC,eAAe,CAAC,IAAI,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YAAE,OAAO,KAAK,CAAC;QACrE,OAAO,IAAI,CAAC;IACd,CAAC,CAAC;IAEF,MAAM,KAAK,GAAgB,CAAC,EAAE,GAAG,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC,CAAC;IAEzD,MAAM,WAAW,GAAG,KAAK,EAAE,IAAe,EAAiB,EAAE;QAC3D,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;YACtB,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACvB,OAAO;QACT,CAAC;QACD,MAAM,GAAG,GAAG,MAAM,SAAS,CAAC,IAAI,CAAC,GAAG,EAAE,OAAO,CAAC,KAAK,CAAC,CAAC;QACrD,MAAM,YAAY,GAAG,IAAI,CAAC,GAAG,CAAC;QAC9B,MAAM,QAAQ,GAAG,YAAY,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,GAAG,CAAC,QAAQ,CAAC;QAC5D,UAAU,CAAC,GAAG,CAAC,YAAY,EAAE,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,EAAE,MAAM,EAAE,GAAG,CAAC,MAAM,EAAE,QAAQ,EAAE,GAAG,CAAC,QAAQ,EAAE,aAAa,EAAE,GAAG,CAAC,aAAa,EAAE,KAAK,EAAE,GAAG,CAAC,KAAK,EAAE,CAAC,CAAC;QAChJ,IAAI,QAAQ,KAAK,YAAY,EAAE,CAAC;YAC9B,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,QAAQ,CAAC;gBAAE,UAAU,CAAC,GAAG,CAAC,QAAQ,EAAE,EAAE,GAAG,EAAE,GAAG,CAAC,QAAQ,EAAE,MAAM,EAAE,GAAG,CAAC,MAAM,EAAE,QAAQ,EAAE,GAAG,CAAC,QAAQ,EAAE,aAAa,EAAE,EAAE,EAAE,KAAK,EAAE,GAAG,CAAC,KAAK,EAAE,CAAC,CAAC;YAChK,IAAI,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,KAAK,QAAQ,CAAC;gBAAE,OAAO;YAC7E,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;YACtB,IAAI,CAAC,UAAU,CAAC,QAAQ,EAAE,SAAS,EAAE,OAAO,CAAC,iBAAiB,CAAC;gBAAE,OAAO,CAAC,sBAAsB;QACjG,CAAC;QACD,QAAQ,CAAC,EAAE,KAAK,EAAE,OAAO,EAAE,OAAO,EAAE,GAAG,GAAG,CAAC,MAAM,IAAI,KAAK,IAAI,GAAG,CAAC,QAAQ,EAAE,EAAE,OAAO,EAAE,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,UAAU,EAAE,UAAU,CAAC,IAAI,EAAE,GAAG,EAAE,GAAG,CAAC,QAAQ,EAAE,CAAC,CAAC;QAC7J,IAAI,GAAG,CAAC,KAAK,IAAI,OAAO,GAAG,CAAC,IAAI,KAAK,QAAQ,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,EAAE,CAAC;YACtE,IAAI,GAAG,CAAC,MAAM,KAAK,IAAI,IAAI,GAAG,CAAC,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,IAAI,OAAO,GAAG,CAAC,IAAI,KAAK,QAAQ,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC;gBAAE,OAAO,CAAC,iBAAiB;YAC5I,8FAA8F;YAC9F,IAAI,IAAI,CAAC,KAAK,KAAK,CAAC,IAAI,GAAG,CAAC,MAAM,KAAK,IAAI,EAAE,CAAC;gBAC5C,MAAM,IAAI,GAAG,WAAW,CAAC,GAAG,CAAC,IAAI,IAAI,EAAE,EAAE,EAAE,GAAG,EAAE,GAAG,CAAC,QAAQ,EAAE,QAAQ,EAAE,SAAS,EAAE,iBAAiB,EAAE,OAAO,CAAC,iBAAiB,EAAE,CAAC,CAAC;gBACnI,OAAO,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,QAAQ,EAAE,YAAY,EAAE,IAAI,CAAC,GAAG,EAAE,IAAI,EAAE,QAAQ,EAAE,cAAc,CAAC,GAAG,CAAC,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE,CAAC,CAAC;YAClH,CAAC;YACD,OAAO;QACT,CAAC;QACD,MAAM,IAAI,GAAG,WAAW,CAAC,GAAG,CAAC,IAAI,EAAE,EAAE,GAAG,EAAE,GAAG,CAAC,QAAQ,EAAE,QAAQ,EAAE,SAAS,EAAE,iBAAiB,EAAE,OAAO,CAAC,iBAAiB,EAAE,CAAC,CAAC;QAC7H,OAAO,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,QAAQ,EAAE,YAAY,EAAE,IAAI,CAAC,GAAG,EAAE,IAAI,EAAE,QAAQ,EAAE,cAAc,CAAC,GAAG,CAAC,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE,CAAC,CAAC;QAEhH,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;YAC9B,IAAI,CAAC,IAAI,CAAC,GAAG;gBAAE,SAAS;YACxB,MAAM,MAAM,GAAG,YAAY,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACtC,IAAI,CAAC,MAAM;gBAAE,SAAS;YACtB,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,CAAC;gBACnB,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,MAAM,CAAC;oBAAE,eAAe,CAAC,GAAG,CAAC,MAAM,EAAE,IAAI,GAAG,EAAE,CAAC,CAAC;gBACzE,eAAe,CAAC,GAAG,CAAC,MAAM,CAAE,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;gBAC3C,SAAS;YACX,CAAC;YACD,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,MAAM,CAAC;gBAAE,WAAW,CAAC,GAAG,CAAC,MAAM,EAAE,IAAI,GAAG,EAAE,CAAC,CAAC;YACjE,WAAW,CAAC,GAAG,CAAC,MAAM,CAAE,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;YACvC,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC;gBAAE,SAAS;YACzD,IAAI,UAAU,CAAC,GAAG,CAAC,MAAM,CAAC;gBAAE,SAAS;YACrC,UAAU,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;YACvB,IAAI,IAAI,CAAC,KAAK,GAAG,CAAC,GAAG,OAAO,CAAC,QAAQ;gBAAE,SAAS;YAChD,IAAI,OAAO,CAAC,IAAI,IAAI,OAAO,CAAC,QAAQ,EAAE,CAAC;gBACrC,UAAU,EAAE,CAAC;gBACb,SAAS;YACX,CAAC;YACD,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;YACpB,KAAK,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,GAAG,CAAC,EAAE,CAAC,CAAC;QACrD,CAAC;IACH,CAAC,CAAC;IAEF,MAAM,QAAQ,CAAC,KAAK,EAAE,OAAO,CAAC,WAAW,EAAE,OAAO,CAAC,OAAO,EAAE,WAAW,CAAC,CAAC;IAEzE,iGAAiG;IACjG,QAAQ,CAAC,EAAE,KAAK,EAAE,OAAO,EAAE,OAAO,EAAE,gCAAgC,EAAE,CAAC,CAAC;IACxE,MAAM,OAAO,GAAG,CAAC,GAAG,WAAW,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,OAAO,CAAC,aAAa,CAAC,CAAC;IACvH,MAAM,YAAY,GAAG,QAAQ,CAAC,IAAI;SAC/B,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,YAAY,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;SAChC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,UAAU,CAAC,CAAC,EAAE,SAAS,EAAE,OAAO,CAAC,iBAAiB,CAAC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,MAAM,CAAC,CAAC,CAAC,CAAC;SAC7H,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,OAAO,CAAC,aAAa,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC;IACjE,KAAK,MAAM,CAAC,IAAI,CAAC,GAAG,WAAW,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;QAAE,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC;YAAE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAC7G,MAAM,aAAa,CAAC,CAAC,GAAG,OAAO,EAAE,GAAG,YAAY,CAAC,EAAE,OAAO,CAAC,WAAW,EAAE,KAAK,EAAE,CAAC,EAAE,EAAE;QAClF,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,CAAC,EAAE,OAAO,CAAC,KAAK,CAAC,CAAC;QAC7C,UAAU,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,GAAG,EAAE,CAAC,EAAE,MAAM,EAAE,GAAG,CAAC,MAAM,EAAE,QAAQ,EAAE,GAAG,CAAC,QAAQ,EAAE,aAAa,EAAE,GAAG,CAAC,aAAa,EAAE,KAAK,EAAE,GAAG,CAAC,KAAK,EAAE,CAAC,CAAC;QAC9H,IAAI,OAAO,CAAC,OAAO;YAAE,MAAM,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;IACpD,CAAC,CAAC,CAAC;IAEH,IAAI,cAAkD,CAAC;IACvD,IAAI,OAAO,CAAC,kBAAkB,EAAE,CAAC;QAC/B,QAAQ,CAAC,EAAE,KAAK,EAAE,OAAO,EAAE,OAAO,EAAE,YAAY,IAAI,CAAC,GAAG,CAAC,eAAe,CAAC,IAAI,EAAE,OAAO,CAAC,aAAa,CAAC,iBAAiB,EAAE,CAAC,CAAC;QAC1H,cAAc,GAAG,IAAI,GAAG,EAAE,CAAC;QAC3B,MAAM,OAAO,GAAG,CAAC,GAAG,eAAe,CAAC,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,OAAO,CAAC,aAAa,CAAC,CAAC;QAC5E,MAAM,aAAa,CAAC,OAAO,EAAE,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,WAAW,EAAE,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,EAAE;YACzE,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,CAAC,EAAE,EAAE,GAAG,OAAO,CAAC,KAAK,EAAE,SAAS,EAAE,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,KAAK,CAAC,SAAS,IAAI,MAAM,EAAE,MAAM,CAAC,EAAE,CAAC,CAAC;YACpH,cAAe,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,GAAG,EAAE,CAAC,EAAE,MAAM,EAAE,GAAG,CAAC,MAAM,EAAE,QAAQ,EAAE,GAAG,CAAC,QAAQ,EAAE,aAAa,EAAE,GAAG,CAAC,aAAa,EAAE,KAAK,EAAE,GAAG,CAAC,KAAK,EAAE,CAAC,CAAC;QACrI,CAAC,CAAC,CAAC;IACL,CAAC;IAED,QAAQ,CAAC,EAAE,KAAK,EAAE,QAAQ,EAAE,OAAO,EAAE,qCAAqC,EAAE,CAAC,CAAC;IAC9E,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,QAAQ,EAAE,OAAO,CAAC,KAAK,CAAC,CAAC;IAExD,MAAM,KAAK,GAAe;QACxB,YAAY,EAAE,OAAO,CAAC,MAAM;QAC5B,eAAe,EAAE,UAAU,CAAC,IAAI;QAChC,YAAY,EAAE,UAAU,CAAC,IAAI,GAAG,CAAC,cAAc,EAAE,IAAI,IAAI,CAAC,CAAC;QAC3D,QAAQ,EAAE,OAAO,CAAC,QAAQ;QAC1B,QAAQ,EAAE,OAAO,CAAC,QAAQ;QAC1B,YAAY,EAAE,UAAU,GAAG,CAAC;QAC5B,UAAU,EAAE,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,GAAG,EAAE,GAAG,OAAO,CAAC;QACnD,eAAe,EAAE,OAAO,CAAC,aAAa;KACvC,CAAC;IAEF,OAAO;QACL,QAAQ;QACR,OAAO;QACP,UAAU;QACV,WAAW;QACX,cAAc;QACd,eAAe,EAAE,OAAO,CAAC,kBAAkB,CAAC,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,SAAS;QACzE,MAAM,EAAE,MAAM,CAAC,IAAI;QACnB,YAAY,EAAE,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI;QACtD,eAAe,EAAE,OAAO;QACxB,QAAQ;QACR,MAAM;QACN,KAAK;KACN,CAAC;AACJ,CAAC;AAED,KAAK,UAAU,QAAQ,CAAC,KAAkB,EAAE,WAAmB,EAAE,OAAe,EAAE,IAAwC;IACxH,IAAI,MAAM,GAAG,CAAC,CAAC;IACf,MAAM,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,EAAE;QAClC,MAAM,IAAI,GAAG,GAAS,EAAE;YACtB,OAAO,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,WAAW,CAAC,IAAI,KAAK,CAAC,MAAM,EAAE,CAAC;gBACzD,MAAM,IAAI,GAAG,KAAK,CAAC,KAAK,EAAG,CAAC;gBAC5B,MAAM,EAAE,CAAC;gBACT,IAAI,CAAC,IAAI,CAAC;qBACP,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC;qBACtB,OAAO,CAAC,GAAG,EAAE;oBACZ,MAAM,EAAE,CAAC;oBACT,IAAI,OAAO,GAAG,CAAC;wBAAE,UAAU,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;;wBACtC,IAAI,EAAE,CAAC;gBACd,CAAC,CAAC,CAAC;YACP,CAAC;YACD,IAAI,MAAM,KAAK,CAAC,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;gBAAE,OAAO,EAAE,CAAC;QACpD,CAAC,CAAC;QACF,IAAI,EAAE,CAAC;IACT,CAAC,CAAC,CAAC;AACL,CAAC;AAED,KAAK,UAAU,SAAS,CAAC,QAAgB,EAAE,SAAuB;IAChE,MAAM,MAAM,GAAiB,EAAE,CAAC;IAChC,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,QAAQ,CAAC,CAAC;IAC9B,MAAM,UAAU,GAAG,IAAI,GAAG,CAAC,sBAAsB,IAAI,CAAC,GAAG,EAAE,CAAC,QAAQ,CAAC,EAAE,CAAC,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC;IACvI,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,UAAU,EAAE,SAAS,CAAC,CAAC;IACnD,MAAM,CAAC,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,UAAU,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,CAAC;IAE9E,IAAI,WAAW,CAAC,QAAQ,CAAC;QAAE,OAAO,MAAM,CAAC;IAEzC,IAAI,GAAG,CAAC,QAAQ,KAAK,QAAQ,EAAE,CAAC;QAC9B,MAAM,OAAO,GAAG,UAAU,GAAG,CAAC,IAAI,GAAG,CAAC;QACtC,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,OAAO,EAAE,EAAE,GAAG,SAAS,EAAE,SAAS,EAAE,MAAM,EAAE,CAAC,CAAC;QACzE,MAAM,CAAC,YAAY,GAAG,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,OAAO,EAAE,MAAM,EAAE,GAAG,CAAC,MAAM,EAAE,QAAQ,EAAE,GAAG,CAAC,QAAQ,EAAE,gBAAgB,EAAE,GAAG,CAAC,QAAQ,CAAC,UAAU,CAAC,UAAU,CAAC,EAAE,CAAC;IAC/J,CAAC;IAED,MAAM,IAAI,GAAG,GAAG,CAAC,QAAQ,CAAC;IAC1B,MAAM,WAAW,GAAG,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,OAAO,IAAI,EAAE,CAAC;IAC7E,MAAM,UAAU,GAAG,GAAG,GAAG,CAAC,QAAQ,KAAK,WAAW,GAAG,CAAC;IACtD,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,UAAU,EAAE,EAAE,GAAG,SAAS,EAAE,SAAS,EAAE,MAAM,EAAE,CAAC,CAAC;IAC5E,MAAM,CAAC,WAAW,GAAG,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,UAAU,EAAE,MAAM,EAAE,GAAG,CAAC,MAAM,EAAE,QAAQ,EAAE,GAAG,CAAC,QAAQ,EAAE,eAAe,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC,KAAK,IAAI,EAAE,CAAC;IAChK,OAAO,MAAM,CAAC;AAChB,CAAC"}
|
|
1
|
+
{"version":3,"file":"crawl.js","sourceRoot":"","sources":["../../src/core/crawl.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAC;AACxD,OAAO,EACL,kBAAkB,EAClB,SAAS,EACT,cAAc,EACd,QAAQ,EACR,cAAc,GACf,MAAM,YAAY,CAAC;AAEpB,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAChD,OAAO,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAC5C,OAAO,EAAE,WAAW,EAAE,MAAM,sBAAsB,CAAC;AAEnD,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAO5C,OAAO,EACL,MAAM,EACN,UAAU,EACV,aAAa,EACb,YAAY,EACZ,QAAQ,GACT,MAAM,UAAU,CAAC;AAsDlB;;;;GAIG;AACH,MAAM,QAAQ,GAAG,KAAK,EACpB,KAAkB,EAClB,WAAmB,EACnB,OAAe,EACf,IAAwC,EACzB,EAAE;IACjB,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,WAAW,CAAC,CAAC;IACvC,MAAM,OAAO,GAAG,IAAI,GAAG,EAAiB,CAAC;IAEzC,MAAM,MAAM,GAAG,CAAC,IAAe,EAAQ,EAAE;QACvC,MAAM,IAAI,GAAG,CAAC,KAAK,IAAI,EAAE;YACvB,IAAI,CAAC;gBACH,MAAM,IAAI,CAAC,IAAI,CAAC,CAAC;YACnB,CAAC;YAAC,MAAM,CAAC;gBACP,oEAAoE;YACtE,CAAC;YAED,IAAI,OAAO,GAAG,CAAC,EAAE,CAAC;gBAChB,MAAM,KAAK,CAAC,OAAO,CAAC,CAAC;YACvB,CAAC;QACH,CAAC,CAAC,EAAE,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC;QAEzC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;IACpB,CAAC,CAAC;IAEF,SAAS,CAAC;QACR,OAAO,OAAO,CAAC,IAAI,GAAG,KAAK,EAAE,CAAC;YAC5B,MAAM,IAAI,GAAG,KAAK,CAAC,KAAK,EAAE,CAAC;YAE3B,IAAI,CAAC,IAAI,EAAE,CAAC;gBACV,MAAM;YACR,CAAC;YAED,MAAM,CAAC,IAAI,CAAC,CAAC;QACf,CAAC;QAED,IAAI,OAAO,CAAC,IAAI,KAAK,CAAC,EAAE,CAAC;YACvB,OAAO;QACT,CAAC;QAED,6IAA6I;QAC7I,MAAM,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IAC9B,CAAC;AACH,CAAC,CAAC;AAEF,+DAA+D;AAC/D,MAAM,SAAS,GAAG,CAChB,OAAiC,EACjC,MAAc,EACd,MAAc,EACR,EAAE;IACR,IAAI,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;IAE9B,IAAI,CAAC,GAAG,EAAE,CAAC;QACT,GAAG,GAAG,IAAI,GAAG,EAAE,CAAC;QAChB,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;IAC3B,CAAC;IAED,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;AAClB,CAAC,CAAC;AAEF,MAAM,SAAS,GAAG,KAAK,EACrB,QAAgB,EAChB,SAAuB,EACA,EAAE;IACzB,MAAM,MAAM,GAAiB,EAAE,CAAC;IAChC,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,QAAQ,CAAC,CAAC;IAE9B,MAAM,UAAU,GAAG,IAAI,GAAG,CACxB,sBAAsB,IAAI,CAAC,GAAG,EAAE,CAAC,QAAQ,CAAC,EAAE,CAAC,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EACzF,GAAG,CAAC,MAAM,CACX,CAAC,IAAI,CAAC;IAEP,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,UAAU,EAAE,SAAS,CAAC,CAAC;IACnD,MAAM,CAAC,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,UAAU,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,CAAC;IAE9E,IAAI,WAAW,CAAC,QAAQ,CAAC,EAAE,CAAC;QAC1B,OAAO,MAAM,CAAC;IAChB,CAAC;IAED,IAAI,GAAG,CAAC,QAAQ,KAAK,QAAQ,EAAE,CAAC;QAC9B,MAAM,OAAO,GAAG,UAAU,GAAG,CAAC,IAAI,GAAG,CAAC;QACtC,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,OAAO,EAAE,EAAE,GAAG,SAAS,EAAE,SAAS,EAAE,MAAM,EAAE,CAAC,CAAC;QACzE,MAAM,CAAC,YAAY,GAAG,GAAG,CAAC,KAAK;YAC7B,CAAC,CAAC,IAAI;YACN,CAAC,CAAC;gBACE,GAAG,EAAE,OAAO;gBACZ,MAAM,EAAE,GAAG,CAAC,MAAM;gBAClB,QAAQ,EAAE,GAAG,CAAC,QAAQ;gBACtB,gBAAgB,EAAE,GAAG,CAAC,QAAQ,CAAC,UAAU,CAAC,UAAU,CAAC;aACtD,CAAC;IACR,CAAC;IAED,MAAM,IAAI,GAAG,GAAG,CAAC,QAAQ,CAAC;IAC1B,MAAM,WAAW,GAAG,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,OAAO,IAAI,EAAE,CAAC;IAC7E,MAAM,UAAU,GAAG,GAAG,GAAG,CAAC,QAAQ,KAAK,WAAW,GAAG,CAAC;IACtD,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,UAAU,EAAE,EAAE,GAAG,SAAS,EAAE,SAAS,EAAE,MAAM,EAAE,CAAC,CAAC;IAC5E,MAAM,CAAC,WAAW,GAAG,GAAG,CAAC,KAAK;QAC5B,CAAC,CAAC,IAAI;QACN,CAAC,CAAC;YACE,GAAG,EAAE,UAAU;YACf,MAAM,EAAE,GAAG,CAAC,MAAM;YAClB,QAAQ,EAAE,GAAG,CAAC,QAAQ;YACtB,eAAe,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC,KAAK,IAAI;SACvD,CAAC;IAEN,OAAO,MAAM,CAAC;AAChB,CAAC,CAAC;AAEF,2EAA2E;AAC3E,MAAM,cAAc,GAAG,GAAS,EAAE;IAChC,kBAAkB;AACpB,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,SAAS,GAAG,KAAK,EAC5B,UAAkB,EAClB,OAAqB,EACC,EAAE;IACxB,MAAM,OAAO,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC;IAClC,MAAM,QAAQ,GAAG,OAAO,CAAC,UAAU,IAAI,cAAc,CAAC;IACtD,MAAM,QAAQ,GAAG,YAAY,CAAC,UAAU,CAAC,IAAI,UAAU,CAAC;IACxD,MAAM,EAAE,MAAM,EAAE,GAAG,IAAI,GAAG,CAAC,QAAQ,CAAC,CAAC;IACrC,MAAM,SAAS,GAAG,MAAM,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC;IACzC,MAAM,SAAS,GAAG,OAAO,CAAC,KAAK,CAAC,SAAS,IAAI,kBAAkB,CAAC;IAEhE,QAAQ,CAAC,EAAE,KAAK,EAAE,QAAQ,EAAE,OAAO,EAAE,YAAY,MAAM,aAAa,EAAE,CAAC,CAAC;IACxE,MAAM,MAAM,GAAG,MAAM,aAAa,CAAC,MAAM,EAAE,SAAS,EAAE,OAAO,CAAC,KAAK,CAAC,CAAC;IACrE,MAAM,MAAM,GAAG,OAAO,CAAC,aAAa,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC;IAElE,QAAQ,CAAC,EAAE,KAAK,EAAE,SAAS,EAAE,OAAO,EAAE,mBAAmB,EAAE,CAAC,CAAC;IAE7D,MAAM,iBAAiB,GAAG;QACxB,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,MAAM,CAAC,IAAI,CAAC,QAAQ,EAAE,IAAI,GAAG,CAAC,cAAc,EAAE,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC;KAC5E,CAAC;IAEF,MAAM,QAAQ,GAAG,MAAM,YAAY,CAAC,iBAAiB,EAAE,OAAO,CAAC,KAAK,CAAC,CAAC;IAEtE,MAAM,OAAO,GAAG,IAAI,GAAG,CAAS,CAAC,QAAQ,CAAC,CAAC,CAAC;IAC5C,MAAM,UAAU,GAAG,IAAI,GAAG,CAAS,CAAC,QAAQ,CAAC,CAAC,CAAC;IAC/C,MAAM,UAAU,GAAG,IAAI,GAAG,EAAqB,CAAC;IAChD,MAAM,WAAW,GAAG,IAAI,GAAG,EAAuB,CAAC;IACnD,MAAM,eAAe,GAAG,IAAI,GAAG,EAAuB,CAAC;IACvD,MAAM,OAAO,GAAiB,EAAE,CAAC;IACjC,MAAM,OAAO,GAAa,EAAE,CAAC;IAC7B,IAAI,UAAU,GAAG,CAAC,CAAC;IAEnB,MAAM,OAAO,GAAG,CAAC,GAAW,EAAW,EAAE;QACvC,IACE,OAAO,CAAC,eAAe,CAAC,MAAM;YAC9B,CAAC,OAAO,CAAC,eAAe,CAAC,IAAI,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EACnD,CAAC;YACD,OAAO,KAAK,CAAC;QACf,CAAC;QAED,IAAI,OAAO,CAAC,eAAe,CAAC,IAAI,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC;YACvD,OAAO,KAAK,CAAC;QACf,CAAC;QAED,OAAO,IAAI,CAAC;IACd,CAAC,CAAC;IAEF,MAAM,KAAK,GAAgB,CAAC,EAAE,GAAG,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC,CAAC;IAEzD,wFAAwF;IACxF,MAAM,WAAW,GAAG,CAClB,KAAiB,EACjB,IAAY,EACZ,KAAa,EACP,EAAE;QACR,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,YAAY,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;YAExD,IAAI,CAAC,MAAM,EAAE,CAAC;gBACZ,SAAS;YACX,CAAC;YAED,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,CAAC;gBACnB,SAAS,CAAC,eAAe,EAAE,MAAM,EAAE,IAAI,CAAC,CAAC;gBACzC,SAAS;YACX,CAAC;YAED,SAAS,CAAC,WAAW,EAAE,MAAM,EAAE,IAAI,CAAC,CAAC;YAErC,IACE,CAAC,aAAa,CAAC,MAAM,CAAC;gBACtB,CAAC,OAAO,CAAC,MAAM,CAAC;gBAChB,UAAU,CAAC,GAAG,CAAC,MAAM,CAAC,EACtB,CAAC;gBACD,SAAS;YACX,CAAC;YAED,UAAU,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;YAEvB,IAAI,KAAK,GAAG,CAAC,GAAG,OAAO,CAAC,QAAQ,EAAE,CAAC;gBACjC,SAAS;YACX,CAAC;YAED,IAAI,OAAO,CAAC,IAAI,IAAI,OAAO,CAAC,QAAQ,EAAE,CAAC;gBACrC,UAAU,IAAI,CAAC,CAAC;gBAChB,SAAS;YACX,CAAC;YAED,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;YACpB,KAAK,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,KAAK,GAAG,CAAC,EAAE,CAAC,CAAC;QAChD,CAAC;IACH,CAAC,CAAC;IAEF;;;OAGG;IACH,MAAM,mBAAmB,GAAG,CAAC,GAAgB,EAAE,QAAgB,EAAW,EAAE;QAC1E,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC;YAC9B,UAAU,CAAC,GAAG,CAAC,QAAQ,EAAE;gBACvB,GAAG,EAAE,GAAG,CAAC,QAAQ;gBACjB,MAAM,EAAE,GAAG,CAAC,MAAM;gBAClB,QAAQ,EAAE,GAAG,CAAC,QAAQ;gBACtB,aAAa,EAAE,EAAE;gBACjB,KAAK,EAAE,GAAG,CAAC,KAAK;aACjB,CAAC,CAAC;QACL,CAAC;QAED,IAAI,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,KAAK,QAAQ,CAAC,EAAE,CAAC;YACrE,OAAO,KAAK,CAAC;QACf,CAAC;QAED,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;QAEtB,OAAO,UAAU,CAAC,QAAQ,EAAE,SAAS,EAAE,OAAO,CAAC,iBAAiB,CAAC,CAAC;IACpE,CAAC,CAAC;IAEF,iGAAiG;IACjG,MAAM,gBAAgB,GAAG,CACvB,IAAe,EACf,GAAgB,EAChB,QAAgB,EACV,EAAE;QACR,MAAM,OAAO,GACX,GAAG,CAAC,MAAM,KAAK,IAAI;YACnB,GAAG,CAAC,MAAM,GAAG,GAAG;YAChB,CAAC,GAAG,CAAC,KAAK;YACV,GAAG,CAAC,IAAI,KAAK,IAAI;YACjB,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC;QAEvB,8FAA8F;QAC9F,IAAI,OAAO,IAAI,CAAC,IAAI,CAAC,KAAK,KAAK,CAAC,IAAI,GAAG,CAAC,MAAM,KAAK,IAAI,CAAC,EAAE,CAAC;YACzD,OAAO;QACT,CAAC;QAED,MAAM,IAAI,GAAG,WAAW,CAAC,GAAG,CAAC,IAAI,IAAI,EAAE,EAAE;YACvC,GAAG,EAAE,GAAG,CAAC,QAAQ;YACjB,QAAQ,EAAE,SAAS;YACnB,iBAAiB,EAAE,OAAO,CAAC,iBAAiB;SAC7C,CAAC,CAAC;QAEH,OAAO,CAAC,IAAI,CAAC;YACX,GAAG,EAAE,QAAQ;YACb,YAAY,EAAE,IAAI,CAAC,GAAG;YACtB,IAAI;YACJ,QAAQ,EAAE,cAAc,CAAC,GAAG,CAAC;YAC7B,KAAK,EAAE,IAAI,CAAC,KAAK;SAClB,CAAC,CAAC;IACL,CAAC,CAAC;IAEF,MAAM,WAAW,GAAG,KAAK,EAAE,IAAe,EAAiB,EAAE;QAC3D,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;YACtB,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YAEvB,OAAO;QACT,CAAC;QAED,MAAM,GAAG,GAAG,MAAM,SAAS,CAAC,IAAI,CAAC,GAAG,EAAE,OAAO,CAAC,KAAK,CAAC,CAAC;QACrD,MAAM,YAAY,GAAG,IAAI,CAAC,GAAG,CAAC;QAC9B,MAAM,QAAQ,GAAG,YAAY,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,GAAG,CAAC,QAAQ,CAAC;QAE5D,UAAU,CAAC,GAAG,CAAC,YAAY,EAAE;YAC3B,GAAG,EAAE,IAAI,CAAC,GAAG;YACb,MAAM,EAAE,GAAG,CAAC,MAAM;YAClB,QAAQ,EAAE,GAAG,CAAC,QAAQ;YACtB,aAAa,EAAE,GAAG,CAAC,aAAa;YAChC,KAAK,EAAE,GAAG,CAAC,KAAK;SACjB,CAAC,CAAC;QAEH,IAAI,QAAQ,KAAK,YAAY,IAAI,CAAC,mBAAmB,CAAC,GAAG,EAAE,QAAQ,CAAC,EAAE,CAAC;YACrE,OAAO;QACT,CAAC;QAED,QAAQ,CAAC;YACP,KAAK,EAAE,OAAO;YACd,OAAO,EAAE,GAAG,GAAG,CAAC,MAAM,IAAI,KAAK,IAAI,GAAG,CAAC,QAAQ,EAAE;YACjD,OAAO,EAAE,OAAO,CAAC,MAAM,GAAG,CAAC;YAC3B,UAAU,EAAE,UAAU,CAAC,IAAI;YAC3B,GAAG,EAAE,GAAG,CAAC,QAAQ;SAClB,CAAC,CAAC;QAEH,IAAI,GAAG,CAAC,KAAK,IAAI,GAAG,CAAC,IAAI,KAAK,IAAI,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,EAAE,CAAC;YAC3D,gBAAgB,CAAC,IAAI,EAAE,GAAG,EAAE,QAAQ,CAAC,CAAC;YAEtC,OAAO;QACT,CAAC;QAED,MAAM,IAAI,GAAG,WAAW,CAAC,GAAG,CAAC,IAAI,EAAE;YACjC,GAAG,EAAE,GAAG,CAAC,QAAQ;YACjB,QAAQ,EAAE,SAAS;YACnB,iBAAiB,EAAE,OAAO,CAAC,iBAAiB;SAC7C,CAAC,CAAC;QAEH,OAAO,CAAC,IAAI,CAAC;YACX,GAAG,EAAE,QAAQ;YACb,YAAY,EAAE,IAAI,CAAC,GAAG;YACtB,IAAI;YACJ,QAAQ,EAAE,cAAc,CAAC,GAAG,CAAC;YAC7B,KAAK,EAAE,IAAI,CAAC,KAAK;SAClB,CAAC,CAAC;QAEH,WAAW,CAAC,IAAI,CAAC,KAAK,EAAE,QAAQ,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC;IAChD,CAAC,CAAC;IAEF,MAAM,QAAQ,CAAC,KAAK,EAAE,OAAO,CAAC,WAAW,EAAE,OAAO,CAAC,OAAO,EAAE,WAAW,CAAC,CAAC;IAEzE,iGAAiG;IACjG,QAAQ,CAAC,EAAE,KAAK,EAAE,OAAO,EAAE,OAAO,EAAE,gCAAgC,EAAE,CAAC,CAAC;IAExE,MAAM,OAAO,GAAG,CAAC,GAAG,WAAW,CAAC,IAAI,EAAE,CAAC;SACpC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,MAAM,CAAC,CAAC,CAAC,CAAC;SAC9C,KAAK,CAAC,CAAC,EAAE,OAAO,CAAC,aAAa,CAAC,CAAC;IAEnC,MAAM,YAAY,GAAG,QAAQ,CAAC,IAAI;SAC/B,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,YAAY,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;SAChC,MAAM,CACL,CAAC,CAAC,EAAE,EAAE,CACJ,UAAU,CAAC,CAAC,EAAE,SAAS,EAAE,OAAO,CAAC,iBAAiB,CAAC;QACnD,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC;QAClB,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC;QACpB,MAAM,CAAC,CAAC,CAAC,CACZ;SACA,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,OAAO,CAAC,aAAa,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC;IAEjE,KAAK,MAAM,MAAM,IAAI,CAAC,GAAG,WAAW,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;QACvE,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;YAC9B,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QACvB,CAAC;IACH,CAAC;IAED,MAAM,aAAa,CACjB,CAAC,GAAG,OAAO,EAAE,GAAG,YAAY,CAAC,EAC7B,OAAO,CAAC,WAAW,EACnB,KAAK,EAAE,CAAC,EAAE,EAAE;QACV,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,CAAC,EAAE,OAAO,CAAC,KAAK,CAAC,CAAC;QAC7C,UAAU,CAAC,GAAG,CAAC,CAAC,EAAE;YAChB,GAAG,EAAE,CAAC;YACN,MAAM,EAAE,GAAG,CAAC,MAAM;YAClB,QAAQ,EAAE,GAAG,CAAC,QAAQ;YACtB,aAAa,EAAE,GAAG,CAAC,aAAa;YAChC,KAAK,EAAE,GAAG,CAAC,KAAK;SACjB,CAAC,CAAC;QAEH,IAAI,OAAO,CAAC,OAAO,EAAE,CAAC;YACpB,MAAM,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;QAC/B,CAAC;IACH,CAAC,CACF,CAAC;IAEF,IAAI,cAAkD,CAAC;IAEvD,IAAI,OAAO,CAAC,kBAAkB,EAAE,CAAC;QAC/B,QAAQ,CAAC;YACP,KAAK,EAAE,OAAO;YACd,OAAO,EAAE,YAAY,IAAI,CAAC,GAAG,CAAC,eAAe,CAAC,IAAI,EAAE,OAAO,CAAC,aAAa,CAAC,iBAAiB;SAC5F,CAAC,CAAC;QACH,MAAM,MAAM,GAAG,IAAI,GAAG,EAAqB,CAAC;QAC5C,cAAc,GAAG,MAAM,CAAC;QACxB,MAAM,OAAO,GAAG,CAAC,GAAG,eAAe,CAAC,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,OAAO,CAAC,aAAa,CAAC,CAAC;QAC5E,MAAM,aAAa,CACjB,OAAO,EACP,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,WAAW,EAAE,CAAC,CAAC,EAChC,KAAK,EAAE,CAAC,EAAE,EAAE;YACV,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,CAAC,EAAE;gBAC5B,GAAG,OAAO,CAAC,KAAK;gBAChB,SAAS,EAAE,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,KAAK,CAAC,SAAS,IAAI,MAAM,EAAE,MAAM,CAAC;aAC/D,CAAC,CAAC;YAEH,MAAM,CAAC,GAAG,CAAC,CAAC,EAAE;gBACZ,GAAG,EAAE,CAAC;gBACN,MAAM,EAAE,GAAG,CAAC,MAAM;gBAClB,QAAQ,EAAE,GAAG,CAAC,QAAQ;gBACtB,aAAa,EAAE,GAAG,CAAC,aAAa;gBAChC,KAAK,EAAE,GAAG,CAAC,KAAK;aACjB,CAAC,CAAC;QACL,CAAC,CACF,CAAC;IACJ,CAAC;IAED,QAAQ,CAAC,EAAE,KAAK,EAAE,QAAQ,EAAE,OAAO,EAAE,qCAAqC,EAAE,CAAC,CAAC;IAC9E,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,QAAQ,EAAE,OAAO,CAAC,KAAK,CAAC,CAAC;IAExD,MAAM,KAAK,GAAe;QACxB,YAAY,EAAE,OAAO,CAAC,MAAM;QAC5B,eAAe,EAAE,UAAU,CAAC,IAAI;QAChC,YAAY,EAAE,UAAU,CAAC,IAAI,GAAG,CAAC,cAAc,EAAE,IAAI,IAAI,CAAC,CAAC;QAC3D,QAAQ,EAAE,OAAO,CAAC,QAAQ;QAC1B,QAAQ,EAAE,OAAO,CAAC,QAAQ;QAC1B,YAAY,EAAE,UAAU,GAAG,CAAC;QAC5B,UAAU,EAAE,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,GAAG,EAAE,GAAG,OAAO,CAAC;QACnD,eAAe,EAAE,OAAO,CAAC,aAAa;KACvC,CAAC;IAEF,OAAO;QACL,QAAQ;QACR,OAAO;QACP,UAAU;QACV,WAAW;QACX,cAAc;QACd,eAAe,EAAE,OAAO,CAAC,kBAAkB,CAAC,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,SAAS;QACzE,MAAM,EAAE,MAAM,CAAC,IAAI;QACnB,YAAY,EAAE,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI;QACtD,eAAe,EAAE,OAAO;QACxB,QAAQ;QACR,MAAM;QACN,KAAK;KACN,CAAC;AACJ,CAAC,CAAC"}
|
package/dist/core/fetch.d.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import type { RedirectHop, ResponseInfo } from
|
|
1
|
+
import type { RedirectHop, ResponseInfo } from "./types.js";
|
|
2
2
|
export declare const DEFAULT_USER_AGENT: string;
|
|
3
3
|
export interface FetchOptions {
|
|
4
4
|
timeoutMs?: number;
|
|
@@ -20,8 +20,8 @@ export interface FetchResult {
|
|
|
20
20
|
error?: string;
|
|
21
21
|
}
|
|
22
22
|
/** GET a URL, following redirects manually so the chain is recorded. */
|
|
23
|
-
export declare
|
|
23
|
+
export declare const fetchPage: (url: string, options?: FetchOptions, readBody?: boolean) => Promise<FetchResult>;
|
|
24
24
|
/** Status/redirect check without downloading the body. */
|
|
25
|
-
export declare
|
|
26
|
-
export declare
|
|
27
|
-
export declare
|
|
25
|
+
export declare const probeUrl: (url: string, options?: FetchOptions) => Promise<FetchResult>;
|
|
26
|
+
export declare const toResponseInfo: (r: FetchResult) => ResponseInfo;
|
|
27
|
+
export declare const isHtmlResponse: (r: FetchResult) => boolean;
|
package/dist/core/fetch.js
CHANGED
|
@@ -1,119 +1,165 @@
|
|
|
1
|
-
import { VERSION } from
|
|
1
|
+
import { VERSION } from "../version.js";
|
|
2
2
|
export const DEFAULT_USER_AGENT = `Mozilla/5.0 (compatible; agentic-seo-check/${VERSION}; +https://github.com/rivi-software/agentic-seo-check)`;
|
|
3
|
-
const KEPT_HEADERS = [
|
|
3
|
+
const KEPT_HEADERS = [
|
|
4
|
+
"content-type",
|
|
5
|
+
"x-robots-tag",
|
|
6
|
+
"link",
|
|
7
|
+
"content-language",
|
|
8
|
+
"cache-control",
|
|
9
|
+
"last-modified",
|
|
10
|
+
"location",
|
|
11
|
+
"content-length",
|
|
12
|
+
"vary",
|
|
13
|
+
];
|
|
4
14
|
const REDIRECT_STATUSES = new Set([301, 302, 303, 307, 308]);
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
const
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
let current = url;
|
|
12
|
-
for (let hop = 0; hop <= maxRedirects; hop++) {
|
|
13
|
-
const controller = new AbortController();
|
|
14
|
-
const timer = setTimeout(() => controller.abort(), timeoutMs);
|
|
15
|
-
try {
|
|
16
|
-
const res = await fetch(current, {
|
|
17
|
-
method: 'GET',
|
|
18
|
-
redirect: 'manual',
|
|
19
|
-
signal: controller.signal,
|
|
20
|
-
headers: {
|
|
21
|
-
'user-agent': options.userAgent ?? DEFAULT_USER_AGENT,
|
|
22
|
-
accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
|
|
23
|
-
'accept-language': 'en,*;q=0.5',
|
|
24
|
-
...(options.headers ?? {}),
|
|
25
|
-
},
|
|
26
|
-
});
|
|
27
|
-
const headers = pickHeaders(res.headers);
|
|
28
|
-
if (REDIRECT_STATUSES.has(res.status) && headers['location']) {
|
|
29
|
-
chain.push({ url: current, status: res.status });
|
|
30
|
-
let nextUrl;
|
|
31
|
-
try {
|
|
32
|
-
nextUrl = new URL(headers['location'], current).href;
|
|
33
|
-
}
|
|
34
|
-
catch {
|
|
35
|
-
return result(url, current, res.status, headers, null, chain, started, `Invalid Location header: ${headers['location']}`);
|
|
36
|
-
}
|
|
37
|
-
await res.body?.cancel().catch(() => undefined);
|
|
38
|
-
if (chain.some((h) => h.url === nextUrl) || nextUrl === current) {
|
|
39
|
-
return result(url, nextUrl, res.status, headers, null, chain, started, 'Redirect loop');
|
|
40
|
-
}
|
|
41
|
-
current = nextUrl;
|
|
42
|
-
continue;
|
|
43
|
-
}
|
|
44
|
-
let body = null;
|
|
45
|
-
if (readBody)
|
|
46
|
-
body = await readLimited(res, options.maxBodyBytes ?? 5_000_000);
|
|
47
|
-
else
|
|
48
|
-
await res.body?.cancel().catch(() => undefined);
|
|
49
|
-
return result(url, current, res.status, headers, body, chain, started);
|
|
50
|
-
}
|
|
51
|
-
catch (err) {
|
|
52
|
-
const message = err instanceof Error ? (err.name === 'AbortError' ? `Timed out after ${timeoutMs} ms` : err.message) : String(err);
|
|
53
|
-
return result(url, current, null, {}, null, chain, started, message);
|
|
54
|
-
}
|
|
55
|
-
finally {
|
|
56
|
-
clearTimeout(timer);
|
|
15
|
+
const pickHeaders = (headers) => {
|
|
16
|
+
const out = {};
|
|
17
|
+
for (const name of KEPT_HEADERS) {
|
|
18
|
+
const value = headers.get(name);
|
|
19
|
+
if (value !== null) {
|
|
20
|
+
out[name] = value;
|
|
57
21
|
}
|
|
58
22
|
}
|
|
59
|
-
return
|
|
60
|
-
}
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
23
|
+
return out;
|
|
24
|
+
};
|
|
25
|
+
const result = (requestedUrl, finalUrl, status, headers, body, chain, started, error) => {
|
|
26
|
+
const r = {
|
|
27
|
+
requestedUrl,
|
|
28
|
+
finalUrl,
|
|
29
|
+
status,
|
|
30
|
+
headers,
|
|
31
|
+
body,
|
|
32
|
+
redirectChain: chain,
|
|
33
|
+
timeMs: Math.round(performance.now() - started),
|
|
34
|
+
contentType: headers["content-type"] ?? null,
|
|
35
|
+
};
|
|
36
|
+
if (error) {
|
|
37
|
+
r.error = error;
|
|
38
|
+
}
|
|
39
|
+
return r;
|
|
40
|
+
};
|
|
41
|
+
const readLimited = async (res, maxBytes) => {
|
|
42
|
+
if (!res.body) {
|
|
43
|
+
return "";
|
|
44
|
+
}
|
|
68
45
|
const reader = res.body.getReader();
|
|
69
46
|
const chunks = [];
|
|
70
47
|
let total = 0;
|
|
48
|
+
let truncated = false;
|
|
71
49
|
for (;;) {
|
|
50
|
+
// oxlint-disable-next-line eslint/no-await-in-loop -- a stream hands over its chunks one after another
|
|
72
51
|
const { done, value } = await reader.read();
|
|
73
|
-
if (done)
|
|
52
|
+
if (done) {
|
|
74
53
|
break;
|
|
54
|
+
}
|
|
75
55
|
if (value) {
|
|
76
56
|
chunks.push(value);
|
|
77
57
|
total += value.byteLength;
|
|
78
58
|
if (total >= maxBytes) {
|
|
79
|
-
|
|
59
|
+
truncated = true;
|
|
80
60
|
break;
|
|
81
61
|
}
|
|
82
62
|
}
|
|
83
63
|
}
|
|
84
|
-
|
|
64
|
+
if (truncated) {
|
|
65
|
+
await reader.cancel().catch(() => null);
|
|
66
|
+
}
|
|
67
|
+
const charset = /charset=(?<charset>[\w-]+)/iu.exec(res.headers.get("content-type") ?? "")?.groups?.charset;
|
|
85
68
|
try {
|
|
86
|
-
return new TextDecoder(charset ??
|
|
69
|
+
return new TextDecoder(charset ?? "utf-8").decode(Buffer.concat(chunks));
|
|
87
70
|
}
|
|
88
71
|
catch {
|
|
89
|
-
return new TextDecoder(
|
|
72
|
+
return new TextDecoder("utf-8").decode(Buffer.concat(chunks));
|
|
90
73
|
}
|
|
91
|
-
}
|
|
92
|
-
|
|
93
|
-
const
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
74
|
+
};
|
|
75
|
+
const fetchHop = async (walk, current) => {
|
|
76
|
+
const { url, options, readBody, timeoutMs, started, chain } = walk;
|
|
77
|
+
const controller = new AbortController();
|
|
78
|
+
const timer = setTimeout(() => controller.abort(), timeoutMs);
|
|
79
|
+
try {
|
|
80
|
+
const res = await fetch(current, {
|
|
81
|
+
method: "GET",
|
|
82
|
+
redirect: "manual",
|
|
83
|
+
signal: controller.signal,
|
|
84
|
+
headers: {
|
|
85
|
+
"user-agent": options.userAgent ?? DEFAULT_USER_AGENT,
|
|
86
|
+
accept: "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
|
|
87
|
+
"accept-language": "en,*;q=0.5",
|
|
88
|
+
...options.headers,
|
|
89
|
+
},
|
|
90
|
+
});
|
|
91
|
+
const headers = pickHeaders(res.headers);
|
|
92
|
+
if (REDIRECT_STATUSES.has(res.status) && headers["location"]) {
|
|
93
|
+
chain.push({ url: current, status: res.status });
|
|
94
|
+
let nextUrl;
|
|
95
|
+
try {
|
|
96
|
+
nextUrl = new URL(headers["location"], current).href;
|
|
97
|
+
}
|
|
98
|
+
catch {
|
|
99
|
+
return {
|
|
100
|
+
done: result(url, current, res.status, headers, null, chain, started, `Invalid Location header: ${headers["location"]}`),
|
|
101
|
+
};
|
|
102
|
+
}
|
|
103
|
+
await res.body?.cancel().catch(() => null);
|
|
104
|
+
if (chain.some((h) => h.url === nextUrl) || nextUrl === current) {
|
|
105
|
+
return {
|
|
106
|
+
done: result(url, nextUrl, res.status, headers, null, chain, started, "Redirect loop"),
|
|
107
|
+
};
|
|
108
|
+
}
|
|
109
|
+
return { next: nextUrl };
|
|
110
|
+
}
|
|
111
|
+
let body = null;
|
|
112
|
+
if (readBody) {
|
|
113
|
+
body = await readLimited(res, options.maxBodyBytes ?? 5_000_000);
|
|
114
|
+
}
|
|
115
|
+
else {
|
|
116
|
+
await res.body?.cancel().catch(() => null);
|
|
117
|
+
}
|
|
118
|
+
return {
|
|
119
|
+
done: result(url, current, res.status, headers, body, chain, started),
|
|
120
|
+
};
|
|
98
121
|
}
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
122
|
+
catch (error) {
|
|
123
|
+
let message = String(error);
|
|
124
|
+
if (error instanceof Error) {
|
|
125
|
+
message =
|
|
126
|
+
error.name === "AbortError"
|
|
127
|
+
? `Timed out after ${timeoutMs} ms`
|
|
128
|
+
: error.message;
|
|
129
|
+
}
|
|
130
|
+
return {
|
|
131
|
+
done: result(url, current, null, {}, null, chain, started, message),
|
|
132
|
+
};
|
|
133
|
+
}
|
|
134
|
+
finally {
|
|
135
|
+
clearTimeout(timer);
|
|
136
|
+
}
|
|
137
|
+
};
|
|
138
|
+
/** GET a URL, following redirects manually so the chain is recorded. */
|
|
139
|
+
export const fetchPage = async (url, options = {}, readBody = true) => {
|
|
140
|
+
const maxRedirects = options.maxRedirects ?? 10;
|
|
141
|
+
const walk = {
|
|
142
|
+
url,
|
|
143
|
+
options,
|
|
144
|
+
readBody,
|
|
145
|
+
timeoutMs: options.timeoutMs ?? 15_000,
|
|
146
|
+
started: performance.now(),
|
|
147
|
+
chain: [],
|
|
111
148
|
};
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
149
|
+
let current = url;
|
|
150
|
+
for (let hop = 0; hop <= maxRedirects; hop += 1) {
|
|
151
|
+
// oxlint-disable-next-line eslint/no-await-in-loop -- each request goes where the one before it redirected
|
|
152
|
+
const outcome = await fetchHop(walk, current);
|
|
153
|
+
if ("done" in outcome) {
|
|
154
|
+
return outcome.done;
|
|
155
|
+
}
|
|
156
|
+
current = outcome.next;
|
|
157
|
+
}
|
|
158
|
+
return result(url, current, null, {}, null, walk.chain, walk.started, `More than ${maxRedirects} redirects`);
|
|
159
|
+
};
|
|
160
|
+
/** Status/redirect check without downloading the body. */
|
|
161
|
+
export const probeUrl = (url, options = {}) => fetchPage(url, options, false);
|
|
162
|
+
export const toResponseInfo = (r) => {
|
|
117
163
|
const info = {
|
|
118
164
|
status: r.status,
|
|
119
165
|
finalUrl: r.finalUrl,
|
|
@@ -122,14 +168,19 @@ export function toResponseInfo(r) {
|
|
|
122
168
|
contentType: r.contentType,
|
|
123
169
|
headers: r.headers,
|
|
124
170
|
};
|
|
125
|
-
if (r.error)
|
|
171
|
+
if (r.error) {
|
|
126
172
|
info.error = r.error;
|
|
173
|
+
}
|
|
127
174
|
return info;
|
|
128
|
-
}
|
|
129
|
-
export
|
|
130
|
-
const ct = r.contentType?.toLowerCase() ??
|
|
131
|
-
if (ct)
|
|
132
|
-
return ct.includes(
|
|
133
|
-
|
|
134
|
-
|
|
175
|
+
};
|
|
176
|
+
export const isHtmlResponse = (r) => {
|
|
177
|
+
const ct = r.contentType?.toLowerCase() ?? "";
|
|
178
|
+
if (ct) {
|
|
179
|
+
return ct.includes("text/html") || ct.includes("application/xhtml");
|
|
180
|
+
}
|
|
181
|
+
if (!r.body) {
|
|
182
|
+
return false;
|
|
183
|
+
}
|
|
184
|
+
return /<html|<!doctype/iu.test(r.body.slice(0, 2000));
|
|
185
|
+
};
|
|
135
186
|
//# sourceMappingURL=fetch.js.map
|
package/dist/core/fetch.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"fetch.js","sourceRoot":"","sources":["../../src/core/fetch.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,eAAe,CAAC;AAGxC,MAAM,CAAC,MAAM,kBAAkB,GAAG,8CAA8C,OAAO,wDAAwD,CAAC;AAEhJ,MAAM,YAAY,GAAG,
|
|
1
|
+
{"version":3,"file":"fetch.js","sourceRoot":"","sources":["../../src/core/fetch.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,eAAe,CAAC;AAGxC,MAAM,CAAC,MAAM,kBAAkB,GAAG,8CAA8C,OAAO,wDAAwD,CAAC;AAEhJ,MAAM,YAAY,GAAG;IACnB,cAAc;IACd,cAAc;IACd,MAAM;IACN,kBAAkB;IAClB,eAAe;IACf,eAAe;IACf,UAAU;IACV,gBAAgB;IAChB,MAAM;CACP,CAAC;AAuBF,MAAM,iBAAiB,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC;AAE7D,MAAM,WAAW,GAAG,CAAC,OAAgB,EAAE,EAAE;IACvC,MAAM,GAAG,GAA2B,EAAE,CAAC;IAEvC,KAAK,MAAM,IAAI,IAAI,YAAY,EAAE,CAAC;QAChC,MAAM,KAAK,GAAG,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QAEhC,IAAI,KAAK,KAAK,IAAI,EAAE,CAAC;YACnB,GAAG,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC;QACpB,CAAC;IACH,CAAC;IAED,OAAO,GAAG,CAAC;AACb,CAAC,CAAC;AAEF,MAAM,MAAM,GAAG,CACb,YAAoB,EACpB,QAAgB,EAChB,MAAqB,EACrB,OAA+B,EAC/B,IAAmB,EACnB,KAAoB,EACpB,OAAe,EACf,KAAc,EACD,EAAE;IACf,MAAM,CAAC,GAAgB;QACrB,YAAY;QACZ,QAAQ;QACR,MAAM;QACN,OAAO;QACP,IAAI;QACJ,aAAa,EAAE,KAAK;QACpB,MAAM,EAAE,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,GAAG,EAAE,GAAG,OAAO,CAAC;QAC/C,WAAW,EAAE,OAAO,CAAC,cAAc,CAAC,IAAI,IAAI;KAC7C,CAAC;IAEF,IAAI,KAAK,EAAE,CAAC;QACV,CAAC,CAAC,KAAK,GAAG,KAAK,CAAC;IAClB,CAAC;IAED,OAAO,CAAC,CAAC;AACX,CAAC,CAAC;AAEF,MAAM,WAAW,GAAG,KAAK,EACvB,GAAa,EACb,QAAgB,EACC,EAAE;IACnB,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC;QACd,OAAO,EAAE,CAAC;IACZ,CAAC;IAED,MAAM,MAAM,GAAG,GAAG,CAAC,IAAI,CAAC,SAAS,EAAE,CAAC;IACpC,MAAM,MAAM,GAAiB,EAAE,CAAC;IAChC,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,IAAI,SAAS,GAAG,KAAK,CAAC;IAEtB,SAAS,CAAC;QACR,uGAAuG;QACvG,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,MAAM,MAAM,CAAC,IAAI,EAAE,CAAC;QAE5C,IAAI,IAAI,EAAE,CAAC;YACT,MAAM;QACR,CAAC;QAED,IAAI,KAAK,EAAE,CAAC;YACV,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACnB,KAAK,IAAI,KAAK,CAAC,UAAU,CAAC;YAE1B,IAAI,KAAK,IAAI,QAAQ,EAAE,CAAC;gBACtB,SAAS,GAAG,IAAI,CAAC;gBACjB,MAAM;YACR,CAAC;QACH,CAAC;IACH,CAAC;IAED,IAAI,SAAS,EAAE,CAAC;QACd,MAAM,MAAM,CAAC,MAAM,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,CAAC;IAC1C,CAAC;IAED,MAAM,OAAO,GAAG,8BAA8B,CAAC,IAAI,CACjD,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,IAAI,EAAE,CACtC,EAAE,MAAM,EAAE,OAAO,CAAC;IAEnB,IAAI,CAAC;QACH,OAAO,IAAI,WAAW,CAAC,OAAO,IAAI,OAAO,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC;IAC3E,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,WAAW,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC;IAChE,CAAC;AACH,CAAC,CAAC;AAeF,MAAM,QAAQ,GAAG,KAAK,EACpB,IAAkB,EAClB,OAAe,EACM,EAAE;IACvB,MAAM,EAAE,GAAG,EAAE,OAAO,EAAE,QAAQ,EAAE,SAAS,EAAE,OAAO,EAAE,KAAK,EAAE,GAAG,IAAI,CAAC;IACnE,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;IACzC,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,UAAU,CAAC,KAAK,EAAE,EAAE,SAAS,CAAC,CAAC;IAE9D,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,OAAO,EAAE;YAC/B,MAAM,EAAE,KAAK;YACb,QAAQ,EAAE,QAAQ;YAClB,MAAM,EAAE,UAAU,CAAC,MAAM;YACzB,OAAO,EAAE;gBACP,YAAY,EAAE,OAAO,CAAC,SAAS,IAAI,kBAAkB;gBACrD,MAAM,EACJ,iEAAiE;gBACnE,iBAAiB,EAAE,YAAY;gBAC/B,GAAG,OAAO,CAAC,OAAO;aACnB;SACF,CAAC,CAAC;QAEH,MAAM,OAAO,GAAG,WAAW,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;QAEzC,IAAI,iBAAiB,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,OAAO,CAAC,UAAU,CAAC,EAAE,CAAC;YAC7D,KAAK,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,OAAO,EAAE,MAAM,EAAE,GAAG,CAAC,MAAM,EAAE,CAAC,CAAC;YACjD,IAAI,OAAe,CAAC;YAEpB,IAAI,CAAC;gBACH,OAAO,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,UAAU,CAAC,EAAE,OAAO,CAAC,CAAC,IAAI,CAAC;YACvD,CAAC;YAAC,MAAM,CAAC;gBACP,OAAO;oBACL,IAAI,EAAE,MAAM,CACV,GAAG,EACH,OAAO,EACP,GAAG,CAAC,MAAM,EACV,OAAO,EACP,IAAI,EACJ,KAAK,EACL,OAAO,EACP,4BAA4B,OAAO,CAAC,UAAU,CAAC,EAAE,CAClD;iBACF,CAAC;YACJ,CAAC;YAED,MAAM,GAAG,CAAC,IAAI,EAAE,MAAM,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,CAAC;YAE3C,IAAI,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,KAAK,OAAO,CAAC,IAAI,OAAO,KAAK,OAAO,EAAE,CAAC;gBAChE,OAAO;oBACL,IAAI,EAAE,MAAM,CACV,GAAG,EACH,OAAO,EACP,GAAG,CAAC,MAAM,EACV,OAAO,EACP,IAAI,EACJ,KAAK,EACL,OAAO,EACP,eAAe,CAChB;iBACF,CAAC;YACJ,CAAC;YAED,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,CAAC;QAC3B,CAAC;QAED,IAAI,IAAI,GAAkB,IAAI,CAAC;QAE/B,IAAI,QAAQ,EAAE,CAAC;YACb,IAAI,GAAG,MAAM,WAAW,CAAC,GAAG,EAAE,OAAO,CAAC,YAAY,IAAI,SAAS,CAAC,CAAC;QACnE,CAAC;aAAM,CAAC;YACN,MAAM,GAAG,CAAC,IAAI,EAAE,MAAM,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,CAAC;QAC7C,CAAC;QAED,OAAO;YACL,IAAI,EAAE,MAAM,CAAC,GAAG,EAAE,OAAO,EAAE,GAAG,CAAC,MAAM,EAAE,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,OAAO,CAAC;SACtE,CAAC;IACJ,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAI,OAAO,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC;QAE5B,IAAI,KAAK,YAAY,KAAK,EAAE,CAAC;YAC3B,OAAO;gBACL,KAAK,CAAC,IAAI,KAAK,YAAY;oBACzB,CAAC,CAAC,mBAAmB,SAAS,KAAK;oBACnC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC;QACtB,CAAC;QAED,OAAO;YACL,IAAI,EAAE,MAAM,CAAC,GAAG,EAAE,OAAO,EAAE,IAAI,EAAE,EAAE,EAAE,IAAI,EAAE,KAAK,EAAE,OAAO,EAAE,OAAO,CAAC;SACpE,CAAC;IACJ,CAAC;YAAS,CAAC;QACT,YAAY,CAAC,KAAK,CAAC,CAAC;IACtB,CAAC;AACH,CAAC,CAAC;AAEF,wEAAwE;AACxE,MAAM,CAAC,MAAM,SAAS,GAAG,KAAK,EAC5B,GAAW,EACX,OAAO,GAAiB,EAAE,EAC1B,QAAQ,GAAG,IAAI,EACO,EAAE;IACxB,MAAM,YAAY,GAAG,OAAO,CAAC,YAAY,IAAI,EAAE,CAAC;IAEhD,MAAM,IAAI,GAAiB;QACzB,GAAG;QACH,OAAO;QACP,QAAQ;QACR,SAAS,EAAE,OAAO,CAAC,SAAS,IAAI,MAAM;QACtC,OAAO,EAAE,WAAW,CAAC,GAAG,EAAE;QAC1B,KAAK,EAAE,EAAE;KACV,CAAC;IAEF,IAAI,OAAO,GAAG,GAAG,CAAC;IAElB,KAAK,IAAI,GAAG,GAAG,CAAC,EAAE,GAAG,IAAI,YAAY,EAAE,GAAG,IAAI,CAAC,EAAE,CAAC;QAChD,2GAA2G;QAC3G,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;QAE9C,IAAI,MAAM,IAAI,OAAO,EAAE,CAAC;YACtB,OAAO,OAAO,CAAC,IAAI,CAAC;QACtB,CAAC;QAED,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC;IACzB,CAAC;IAED,OAAO,MAAM,CACX,GAAG,EACH,OAAO,EACP,IAAI,EACJ,EAAE,EACF,IAAI,EACJ,IAAI,CAAC,KAAK,EACV,IAAI,CAAC,OAAO,EACZ,aAAa,YAAY,YAAY,CACtC,CAAC;AACJ,CAAC,CAAC;AAEF,0DAA0D;AAC1D,MAAM,CAAC,MAAM,QAAQ,GAAG,CACtB,GAAW,EACX,OAAO,GAAiB,EAAE,EACJ,EAAE,CAAC,SAAS,CAAC,GAAG,EAAE,OAAO,EAAE,KAAK,CAAC,CAAC;AAE1D,MAAM,CAAC,MAAM,cAAc,GAAG,CAAC,CAAc,EAAgB,EAAE;IAC7D,MAAM,IAAI,GAAiB;QACzB,MAAM,EAAE,CAAC,CAAC,MAAM;QAChB,QAAQ,EAAE,CAAC,CAAC,QAAQ;QACpB,aAAa,EAAE,CAAC,CAAC,aAAa;QAC9B,MAAM,EAAE,CAAC,CAAC,MAAM;QAChB,WAAW,EAAE,CAAC,CAAC,WAAW;QAC1B,OAAO,EAAE,CAAC,CAAC,OAAO;KACnB,CAAC;IAEF,IAAI,CAAC,CAAC,KAAK,EAAE,CAAC;QACZ,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC;IACvB,CAAC;IAED,OAAO,IAAI,CAAC;AACd,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,cAAc,GAAG,CAAC,CAAc,EAAW,EAAE;IACxD,MAAM,EAAE,GAAG,CAAC,CAAC,WAAW,EAAE,WAAW,EAAE,IAAI,EAAE,CAAC;IAE9C,IAAI,EAAE,EAAE,CAAC;QACP,OAAO,EAAE,CAAC,QAAQ,CAAC,WAAW,CAAC,IAAI,EAAE,CAAC,QAAQ,CAAC,mBAAmB,CAAC,CAAC;IACtE,CAAC;IAED,IAAI,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;QACZ,OAAO,KAAK,CAAC;IACf,CAAC;IAED,OAAO,mBAAmB,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC,CAAC;AACzD,CAAC,CAAC"}
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
import type { AnyNode, Document, Element } from "domhandler";
|
|
2
|
+
export type { AnyNode, Document, Element } from "domhandler";
|
|
3
|
+
/** Parse a full HTML document with the WHATWG algorithm (parse5), scripting enabled like a browser. */
|
|
4
|
+
export declare const parseDocument: (html: string) => Document;
|
|
5
|
+
/** Descendants of `root` matching a CSS selector, in document order. */
|
|
6
|
+
export declare const queryAll: (root: AnyNode, selector: string) => Element[];
|
|
7
|
+
/** First descendant of `root` matching a CSS selector. */
|
|
8
|
+
export declare const queryOne: (root: AnyNode, selector: string) => Element | null;
|
|
9
|
+
export declare const attr: (el: Element, name: string) => string | undefined;
|
|
10
|
+
export declare const hasAttr: (el: Element, name: string) => boolean;
|
|
11
|
+
/** DOM `textContent`: concatenated text of all descendants, comments excluded. */
|
|
12
|
+
export declare const text: (node: AnyNode | AnyNode[]) => string;
|
|
13
|
+
export declare const childElements: (el: Element) => Element[];
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
import { selectAll, selectOne } from "css-select";
|
|
2
|
+
import { isTag } from "domhandler";
|
|
3
|
+
import { textContent } from "domutils";
|
|
4
|
+
import { parse } from "parse5";
|
|
5
|
+
import { adapter } from "parse5-htmlparser2-tree-adapter";
|
|
6
|
+
/** Parse a full HTML document with the WHATWG algorithm (parse5), scripting enabled like a browser. */
|
|
7
|
+
export const parseDocument = (html) => parse(html, { scriptingEnabled: true, treeAdapter: adapter });
|
|
8
|
+
/** Descendants of `root` matching a CSS selector, in document order. */
|
|
9
|
+
export const queryAll = (root, selector) => selectAll(selector, root);
|
|
10
|
+
/** First descendant of `root` matching a CSS selector. */
|
|
11
|
+
export const queryOne = (root, selector) => selectOne(selector, root);
|
|
12
|
+
export const attr = (el, name) => Object.hasOwn(el.attribs, name) ? el.attribs[name] : undefined;
|
|
13
|
+
export const hasAttr = (el, name) => Object.hasOwn(el.attribs, name);
|
|
14
|
+
/** DOM `textContent`: concatenated text of all descendants, comments excluded. */
|
|
15
|
+
export const text = (node) => textContent(node);
|
|
16
|
+
export const childElements = (el) => el.children.filter(isTag);
|
|
17
|
+
//# sourceMappingURL=dom.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"dom.js","sourceRoot":"","sources":["../../../src/core/html/dom.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AAClD,OAAO,EAAE,KAAK,EAAE,MAAM,YAAY,CAAC;AAEnC,OAAO,EAAE,WAAW,EAAE,MAAM,UAAU,CAAC;AACvC,OAAO,EAAE,KAAK,EAAE,MAAM,QAAQ,CAAC;AAC/B,OAAO,EAAE,OAAO,EAAE,MAAM,iCAAiC,CAAC;AAI1D,uGAAuG;AACvG,MAAM,CAAC,MAAM,aAAa,GAAG,CAAC,IAAY,EAAY,EAAE,CACtD,KAAK,CAAC,IAAI,EAAE,EAAE,gBAAgB,EAAE,IAAI,EAAE,WAAW,EAAE,OAAO,EAAE,CAAC,CAAC;AAEhE,wEAAwE;AACxE,MAAM,CAAC,MAAM,QAAQ,GAAG,CAAC,IAAa,EAAE,QAAgB,EAAa,EAAE,CACrE,SAAS,CAAmB,QAAQ,EAAE,IAAI,CAAC,CAAC;AAE9C,0DAA0D;AAC1D,MAAM,CAAC,MAAM,QAAQ,GAAG,CAAC,IAAa,EAAE,QAAgB,EAAkB,EAAE,CAC1E,SAAS,CAAmB,QAAQ,EAAE,IAAI,CAAC,CAAC;AAE9C,MAAM,CAAC,MAAM,IAAI,GAAG,CAAC,EAAW,EAAE,IAAY,EAAsB,EAAE,CACpE,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;AAEjE,MAAM,CAAC,MAAM,OAAO,GAAG,CAAC,EAAW,EAAE,IAAY,EAAW,EAAE,CAC5D,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC;AAElC,kFAAkF;AAClF,MAAM,CAAC,MAAM,IAAI,GAAG,CAAC,IAAyB,EAAU,EAAE,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC;AAE7E,MAAM,CAAC,MAAM,aAAa,GAAG,CAAC,EAAW,EAAa,EAAE,CACtD,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC"}
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import type { ExtractedPage } from
|
|
1
|
+
import type { ExtractedPage } from "../types.js";
|
|
2
2
|
export interface ExtractOptions {
|
|
3
3
|
/** Absolute URL of the document; relative links are resolved against it (or against <base href>). */
|
|
4
4
|
url: string;
|
|
@@ -7,4 +7,4 @@ export interface ExtractOptions {
|
|
|
7
7
|
/** Host used to classify links as internal. Defaults to the host of `url`. */
|
|
8
8
|
siteHost?: string;
|
|
9
9
|
}
|
|
10
|
-
export declare
|
|
10
|
+
export declare const extractPage: (html: string, options: ExtractOptions) => ExtractedPage;
|