@rivi-software/agentic-seo-check 0.1.1 → 0.3.0

package/dist/core/crawl.js.map

@@ -1 +1 @@
-{"version":3,"file":"crawl.js","sourceRoot":"","sources":["../../src/core/crawl.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAC;AACxD,OAAO,EACL,kBAAkB,EAClB,SAAS,EACT,cAAc,EACd,QAAQ,EACR,cAAc,GACf,MAAM,YAAY,CAAC;AAEpB,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAChD,OAAO,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAC5C,OAAO,EAAE,WAAW,EAAE,MAAM,sBAAsB,CAAC;AAEnD,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAO5C,OAAO,EACL,MAAM,EACN,UAAU,EACV,aAAa,EACb,YAAY,EACZ,QAAQ,GACT,MAAM,UAAU,CAAC;AAsDlB;;;;GAIG;AACH,MAAM,QAAQ,GAAG,KAAK,EACpB,KAAkB,EAClB,WAAmB,EACnB,OAAe,EACf,IAAwC,EACzB,EAAE;IACjB,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,WAAW,CAAC,CAAC;IACvC,MAAM,OAAO,GAAG,IAAI,GAAG,EAAiB,CAAC;IAEzC,MAAM,MAAM,GAAG,CAAC,IAAe,EAAQ,EAAE;QACvC,MAAM,IAAI,GAAG,CAAC,KAAK,IAAI,EAAE;YACvB,IAAI,CAAC;gBACH,MAAM,IAAI,CAAC,IAAI,CAAC,CAAC;YACnB,CAAC;YAAC,MAAM,CAAC;gBACP,oEAAoE;YACtE,CAAC;YAED,IAAI,OAAO,GAAG,CAAC,EAAE,CAAC;gBAChB,MAAM,KAAK,CAAC,OAAO,CAAC,CAAC;YACvB,CAAC;QACH,CAAC,CAAC,EAAE,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC;QAEzC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;IACpB,CAAC,CAAC;IAEF,SAAS,CAAC;QACR,OAAO,OAAO,CAAC,IAAI,GAAG,KAAK,EAAE,CAAC;YAC5B,MAAM,IAAI,GAAG,KAAK,CAAC,KAAK,EAAE,CAAC;YAE3B,IAAI,CAAC,IAAI,EAAE,CAAC;gBACV,MAAM;YACR,CAAC;YAED,MAAM,CAAC,IAAI,CAAC,CAAC;QACf,CAAC;QAED,IAAI,OAAO,CAAC,IAAI,KAAK,CAAC,EAAE,CAAC;YACvB,OAAO;QACT,CAAC;QAED,6IAA6I;QAC7I,MAAM,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IAC9B,CAAC;AACH,CAAC,CAAC;AAEF,+DAA+D;AAC/D,MAAM,SAAS,GAAG,CAChB,OAAiC,EACjC,MAAc,EACd,MAAc,EACR,EAAE;IACR,IAAI,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;IAE9B,IAAI,CAAC,GAAG,EAAE,CAAC;QACT,GAAG,GAAG,IAAI,GAAG,EAAE,CAAC;QAChB,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;IAC3B,CAAC;IAED,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;AAClB,CAAC,CAAC;AAEF,MAAM,SAAS,GAAG,KAAK,EACrB,QAAgB,EAChB,SAAuB,EACA,EAAE;IACzB,MAAM,MAAM,GAAiB,EAAE,CAAC;IAChC,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,QAAQ,CAAC,CAAC;IAE9B,MAAM,UAAU,GAAG,IAAI,GAAG,CACxB,sBAAsB,IAAI,CAAC,GAAG,EAAE,CAAC,QAAQ,CAAC,EAAE,CAAC,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EACzF,GAAG,CAAC,MAAM,CACX,CAAC,IAAI,CAAC;IAEP,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,UAAU,EAAE,SAAS,CAAC,CAAC;IACnD,MAAM,CAAC,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,UAAU,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,CAAC;IAE9E,IAAI,WAAW,CAAC,QAAQ,CAAC,EAAE,CAAC;QAC1B,OAAO,MAAM,CAAC;IAChB,CAAC;IAED,IAAI,GAAG,CAAC,QAAQ,KAAK,QAAQ,EAAE,CAAC;QAC9B,MAAM,OAAO,GAAG,UAAU,GAAG,CAAC,IAAI,GAAG,CAAC;QACtC,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,OAAO,EAAE,EAAE,GAAG,SAAS,EAAE,SAAS,EAAE,MAAM,EAAE,CAAC,CAAC;QACzE,MAAM,CAAC,YAAY,GAAG,GAAG,CAAC,KAAK;YAC7B,CAAC,CAAC,IAAI;YACN,CAAC,CAAC;gBACE,GAAG,EAAE,OAAO;gBACZ,MAAM,EAAE,GAAG,CAAC,MAAM;gBAClB,QAAQ,EAAE,GAAG,CAAC,QAAQ;gBACtB,gBAAgB,EAAE,GAAG,CAAC,QAAQ,CAAC,UAAU,CAAC,UAAU,CAAC;aACtD,CAAC;IACR,CAAC;IAED,MAAM,IAAI,GAAG,GAAG,CAAC,QAAQ,CAAC;IAC1B,MAAM,WAAW,GAAG,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,OAAO,IAAI,EAAE,CAAC;IAC7E,MAAM,UAAU,GAAG,GAAG,GAAG,CAAC,QAAQ,KAAK,WAAW,GAAG,CAAC;IACtD,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,UAAU,EAAE,EAAE,GAAG,SAAS,EAAE,SAAS,EAAE,MAAM,EAAE,CAAC,CAAC;IAC5E,MAAM,CAAC,WAAW,GAAG,GAAG,CAAC,KAAK;QAC5B,CAAC,CAAC,IAAI;QACN,CAAC,CAAC;YACE,GAAG,EAAE,UAAU;YACf,MAAM,EAAE,GAAG,CAAC,MAAM;YAClB,QAAQ,EAAE,GAAG,CAAC,QAAQ;YACtB,eAAe,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC,KAAK,IAAI;SACvD,CAAC;IAEN,OAAO,MAAM,CAAC;AAChB,CAAC,CAAC;AAEF,2EAA2E;AAC3E,MAAM,cAAc,GAAG,GAAS,EAAE;IAChC,kBAAkB;AACpB,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,SAAS,GAAG,KAAK,EAC5B,UAAkB,EAClB,OAAqB,EACC,EAAE;IACxB,MAAM,OAAO,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC;IAClC,MAAM,QAAQ,GAAG,OAAO,CAAC,UAAU,IAAI,cAAc,CAAC;IACtD,MAAM,QAAQ,GAAG,YAAY,CAAC,UAAU,CAAC,IAAI,UAAU,CAAC;IACxD,MAAM,EAAE,MAAM,EAAE,GAAG,IAAI,GAAG,CAAC,QAAQ,CAAC,CAAC;IACrC,MAAM,SAAS,GAAG,MAAM,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC;IACzC,MAAM,SAAS,GAAG,OAAO,CAAC,KAAK,CAAC,SAAS,IAAI,kBAAkB,CAAC;IAEhE,QAAQ,CAAC,EAAE,KAAK,EAAE,QAAQ,EAAE,OAAO,EAAE,YAAY,MAAM,aAAa,EAAE,CAAC,CAAC;IACxE,MAAM,MAAM,GAAG,MAAM,aAAa,CAAC,MAAM,EAAE,SAAS,EAAE,OAAO,CAAC,KAAK,CAAC,CAAC;IACrE,MAAM,MAAM,GAAG,OAAO,CAAC,aAAa,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC;IAElE,QAAQ,CAAC,EAAE,KAAK,EAAE,SAAS,EAAE,OAAO,EAAE,mBAAmB,EAAE,CAAC,CAAC;IAE7D,MAAM,iBAAiB,GAAG;QACxB,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,MAAM,CAAC,IAAI,CAAC,QAAQ,EAAE,IAAI,GAAG,CAAC,cAAc,EAAE,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC;KAC5E,CAAC;IAEF,MAAM,QAAQ,GAAG,MAAM,YAAY,CAAC,iBAAiB,EAAE,OAAO,CAAC,KAAK,CAAC,CAAC;IAEtE,MAAM,OAAO,GAAG,IAAI,GAAG,CAAS,CAAC,QAAQ,CAAC,CAAC,CAAC;IAC5C,MAAM,UAAU,GAAG,IAAI,GAAG,CAAS,CAAC,QAAQ,CAAC,CAAC,CAAC;IAC/C,MAAM,UAAU,GAAG,IAAI,GAAG,EAAqB,CAAC;IAChD,MAAM,WAAW,GAAG,IAAI,GAAG,EAAuB,CAAC;IACnD,MAAM,eAAe,GAAG,IAAI,GAAG,EAAuB,CAAC;IACvD,MAAM,OAAO,GAAiB,EAAE,CAAC;IACjC,MAAM,OAAO,GAAa,EAAE,CAAC;IAC7B,IAAI,UAAU,GAAG,CAAC,CAAC;IAEnB,MAAM,OAAO,GAAG,CAAC,GAAW,EAAW,EAAE;QACvC,IACE,OAAO,CAAC,eAAe,CAAC,MAAM;YAC9B,CAAC,OAAO,CAAC,eAAe,CAAC,IAAI,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EACnD,CAAC;YACD,OAAO,KAAK,CAAC;QACf,CAAC;QAED,IAAI,OAAO,CAAC,eAAe,CAAC,IAAI,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC;YACvD,OAAO,KAAK,CAAC;QACf,CAAC;QAED,OAAO,IAAI,CAAC;IACd,CAAC,CAAC;IAEF,MAAM,KAAK,GAAgB,CAAC,EAAE,GAAG,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC,CAAC;IAEzD,wFAAwF;IACxF,MAAM,WAAW,GAAG,CAClB,KAAiB,EACjB,IAAY,EACZ,KAAa,EACP,EAAE;QACR,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,YAAY,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;YAExD,IAAI,CAAC,MAAM,EAAE,CAAC;gBACZ,SAAS;YACX,CAAC;YAED,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,CAAC;gBACnB,SAAS,CAAC,eAAe,EAAE,MAAM,EAAE,IAAI,CAAC,CAAC;gBACzC,SAAS;YACX,CAAC;YAED,SAAS,CAAC,WAAW,EAAE,MAAM,EAAE,IAAI,CAAC,CAAC;YAErC,IACE,CAAC,aAAa,CAAC,MAAM,CAAC;gBACtB,CAAC,OAAO,CAAC,MAAM,CAAC;gBAChB,UAAU,CAAC,GAAG,CAAC,MAAM,CAAC,EACtB,CAAC;gBACD,SAAS;YACX,CAAC;YAED,UAAU,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;YAEvB,IAAI,KAAK,GAAG,CAAC,GAAG,OAAO,CAAC,QAAQ,EAAE,CAAC;gBACjC,SAAS;YACX,CAAC;YAED,IAAI,OAAO,CAAC,IAAI,IAAI,OAAO,CAAC,QAAQ,EAAE,CAAC;gBACrC,UAAU,IAAI,CAAC,CAAC;gBAChB,SAAS;YACX,CAAC;YAED,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;YACpB,KAAK,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,KAAK,GAAG,CAAC,EAAE,CAAC,CAAC;QAChD,CAAC;IACH,CAAC,CAAC;IAEF;;;OAGG;IACH,MAAM,mBAAmB,GAAG,CAAC,GAAgB,EAAE,QAAgB,EAAW,EAAE;QAC1E,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC;YAC9B,UAAU,CAAC,GAAG,CAAC,QAAQ,EAAE;gBACvB,GAAG,EAAE,GAAG,CAAC,QAAQ;gBACjB,MAAM,EAAE,GAAG,CAAC,MAAM;gBAClB,QAAQ,EAAE,GAAG,CAAC,QAAQ;gBACtB,aAAa,EAAE,EAAE;gBACjB,KAAK,EAAE,GAAG,CAAC,KAAK;aACjB,CAAC,CAAC;QACL,CAAC;QAED,IAAI,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,KAAK,QAAQ,CAAC,EAAE,CAAC;YACrE,OAAO,KAAK,CAAC;QACf,CAAC;QAED,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;QAEtB,OAAO,UAAU,CAAC,QAAQ,EAAE,SAAS,EAAE,OAAO,CAAC,iBAAiB,CAAC,CAAC;IACpE,CAAC,CAAC;IAEF,iGAAiG;IACjG,MAAM,gBAAgB,GAAG,CACvB,IAAe,EACf,GAAgB,EAChB,QAAgB,EACV,EAAE;QACR,MAAM,OAAO,GACX,GAAG,CAAC,MAAM,KAAK,IAAI;YACnB,GAAG,CAAC,MAAM,GAAG,GAAG;YAChB,CAAC,GAAG,CAAC,KAAK;YACV,GAAG,CAAC,IAAI,KAAK,IAAI;YACjB,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC;QAEvB,8FAA8F;QAC9F,IAAI,OAAO,IAAI,CAAC,IAAI,CAAC,KAAK,KAAK,CAAC,IAAI,GAAG,CAAC,MAAM,KAAK,IAAI,CAAC,EAAE,CAAC;YACzD,OAAO;QACT,CAAC;QAED,MAAM,IAAI,GAAG,WAAW,CAAC,GAAG,CAAC,IAAI,IAAI,EAAE,EAAE;YACvC,GAAG,EAAE,GAAG,CAAC,QAAQ;YACjB,QAAQ,EAAE,SAAS;YACnB,iBAAiB,EAAE,OAAO,CAAC,iBAAiB;SAC7C,CAAC,CAAC;QAEH,OAAO,CAAC,IAAI,CAAC;YACX,GAAG,EAAE,QAAQ;YACb,YAAY,EAAE,IAAI,CAAC,GAAG;YACtB,IAAI;YACJ,QAAQ,EAAE,cAAc,CAAC,GAAG,CAAC;YAC7B,KAAK,EAAE,IAAI,CAAC,KAAK;SAClB,CAAC,CAAC;IACL,CAAC,CAAC;IAEF,MAAM,WAAW,GAAG,KAAK,EAAE,IAAe,EAAiB,EAAE;QAC3D,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;YACtB,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YAEvB,OAAO;QACT,CAAC;QAED,MAAM,GAAG,GAAG,MAAM,SAAS,CAAC,IAAI,CAAC,GAAG,EAAE,OAAO,CAAC,KAAK,CAAC,CAAC;QACrD,MAAM,YAAY,GAAG,IAAI,CAAC,GAAG,CAAC;QAC9B,MAAM,QAAQ,GAAG,YAAY,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,GAAG,CAAC,QAAQ,CAAC;QAE5D,UAAU,CAAC,GAAG,CAAC,YAAY,EAAE;YAC3B,GAAG,EAAE,IAAI,CAAC,GAAG;YACb,MAAM,EAAE,GAAG,CAAC,MAAM;YAClB,QAAQ,EAAE,GAAG,CAAC,QAAQ;YACtB,aAAa,EAAE,GAAG,CAAC,aAAa;YAChC,KAAK,EAAE,GAAG,CAAC,KAAK;SACjB,CAAC,CAAC;QAEH,IAAI,QAAQ,KAAK,YAAY,IAAI,CAAC,mBAAmB,CAAC,GAAG,EAAE,QAAQ,CAAC,EAAE,CAAC;YACrE,OAAO;QACT,CAAC;QAED,QAAQ,CAAC;YACP,KAAK,EAAE,OAAO;YACd,OAAO,EAAE,GAAG,GAAG,CAAC,MAAM,IAAI,KAAK,IAAI,GAAG,CAAC,QAAQ,EAAE;YACjD,OAAO,EAAE,OAAO,CAAC,MAAM,GAAG,CAAC;YAC3B,UAAU,EAAE,UAAU,CAAC,IAAI;YAC3B,GAAG,EAAE,GAAG,CAAC,QAAQ;SAClB,CAAC,CAAC;QAEH,IAAI,GAAG,CAAC,KAAK,IAAI,GAAG,CAAC,IAAI,KAAK,IAAI,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,EAAE,CAAC;YAC3D,gBAAgB,CAAC,IAAI,EAAE,GAAG,EAAE,QAAQ,CAAC,CAAC;YAEtC,OAAO;QACT,CAAC;QAED,MAAM,IAAI,GAAG,WAAW,CAAC,GAAG,CAAC,IAAI,EAAE;YACjC,GAAG,EAAE,GAAG,CAAC,QAAQ;YACjB,QAAQ,EAAE,SAAS;YACnB,iBAAiB,EAAE,OAAO,CAAC,iBAAiB;SAC7C,CAAC,CAAC;QAEH,OAAO,CAAC,IAAI,CAAC;YACX,GAAG,EAAE,QAAQ;YACb,YAAY,EAAE,IAAI,CAAC,GAAG;YACtB,IAAI;YACJ,QAAQ,EAAE,cAAc,CAAC,GAAG,CAAC;YAC7B,KAAK,EAAE,IAAI,CAAC,KAAK;SAClB,CAAC,CAAC;QAEH,WAAW,CAAC,IAAI,CAAC,KAAK,EAAE,QAAQ,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC;IAChD,CAAC,CAAC;IAEF,MAAM,QAAQ,CAAC,KAAK,EAAE,OAAO,CAAC,WAAW,EAAE,OAAO,CAAC,OAAO,EAAE,WAAW,CAAC,CAAC;IAEzE,iGAAiG;IACjG,QAAQ,CAAC,EAAE,KAAK,EAAE,OAAO,EAAE,OAAO,EAAE,gCAAgC,EAAE,CAAC,CAAC;IAExE,MAAM,OAAO,GAAG,CAAC,GAAG,WAAW,CAAC,IAAI,EAAE,CAAC;SACpC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,MAAM,CAAC,CAAC,CAAC,CAAC;SAC9C,KAAK,CAAC,CAAC,EAAE,OAAO,CAAC,aAAa,CAAC,CAAC;IAEnC,MAAM,YAAY,GAAG,QAAQ,CAAC,IAAI;SAC/B,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,YAAY,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;SAChC,MAAM,CACL,CAAC,CAAC,EAAE,EAAE,CACJ,UAAU,CAAC,CAAC,EAAE,SAAS,EAAE,OAAO,CAAC,iBAAiB,CAAC;QACnD,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC;QAClB,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC;QACpB,MAAM,CAAC,CAAC,CAAC,CACZ;SACA,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,OAAO,CAAC,aAAa,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC;IAEjE,KAAK,MAAM,MAAM,IAAI,CAAC,GAAG,WAAW,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;QACvE,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;YAC9B,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QACvB,CAAC;IACH,CAAC;IAED,MAAM,aAAa,CACjB,CAAC,GAAG,OAAO,EAAE,GAAG,YAAY,CAAC,EAC7B,OAAO,CAAC,WAAW,EACnB,KAAK,EAAE,CAAC,EAAE,EAAE;QACV,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,CAAC,EAAE,OAAO,CAAC,KAAK,CAAC,CAAC;QAC7C,UAAU,CAAC,GAAG,CAAC,CAAC,EAAE;YAChB,GAAG,EAAE,CAAC;YACN,MAAM,EAAE,GAAG,CAAC,MAAM;YAClB,QAAQ,EAAE,GAAG,CAAC,QAAQ;YACtB,aAAa,EAAE,GAAG,CAAC,aAAa;YAChC,KAAK,EAAE,GAAG,CAAC,KAAK;SACjB,CAAC,CAAC;QAEH,IAAI,OAAO,CAAC,OAAO,EAAE,CAAC;YACpB,MAAM,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;QAC/B,CAAC;IACH,CAAC,CACF,CAAC;IAEF,IAAI,cAAkD,CAAC;IAEvD,IAAI,OAAO,CAAC,kBAAkB,EAAE,CAAC;QAC/B,QAAQ,CAAC;YACP,KAAK,EAAE,OAAO;YACd,OAAO,EAAE,YAAY,IAAI,CAAC,GAAG,CAAC,eAAe,CAAC,IAAI,EAAE,OAAO,CAAC,aAAa,CAAC,iBAAiB;SAC5F,CAAC,CAAC;QACH,MAAM,MAAM,GAAG,IAAI,GAAG,EAAqB,CAAC;QAC5C,cAAc,GAAG,MAAM,CAAC;QACxB,MAAM,OAAO,GAAG,CAAC,GAAG,eAAe,CAAC,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,OAAO,CAAC,aAAa,CAAC,CAAC;QAC5E,MAAM,aAAa,CACjB,OAAO,EACP,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,WAAW,EAAE,CAAC,CAAC,EAChC,KAAK,EAAE,CAAC,EAAE,EAAE;YACV,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,CAAC,EAAE;gBAC5B,GAAG,OAAO,CAAC,KAAK;gBAChB,SAAS,EAAE,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,KAAK,CAAC,SAAS,IAAI,MAAM,EAAE,MAAM,CAAC;aAC/D,CAAC,CAAC;YAEH,MAAM,CAAC,GAAG,CAAC,CAAC,EAAE;gBACZ,GAAG,EAAE,CAAC;gBACN,MAAM,EAAE,GAAG,CAAC,MAAM;gBAClB,QAAQ,EAAE,GAAG,CAAC,QAAQ;gBACtB,aAAa,EAAE,GAAG,CAAC,aAAa;gBAChC,KAAK,EAAE,GAAG,CAAC,KAAK;aACjB,CAAC,CAAC;QACL,CAAC,CACF,CAAC;IACJ,CAAC;IAED,QAAQ,CAAC,EAAE,KAAK,EAAE,QAAQ,EAAE,OAAO,EAAE,qCAAqC,EAAE,CAAC,CAAC;IAC9E,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,QAAQ,EAAE,OAAO,CAAC,KAAK,CAAC,CAAC;IAExD,MAAM,KAAK,GAAe;QACxB,YAAY,EAAE,OAAO,CAAC,MAAM;QAC5B,eAAe,EAAE,UAAU,CAAC,IAAI;QAChC,YAAY,EAAE,UAAU,CAAC,IAAI,GAAG,CAAC,cAAc,EAAE,IAAI,IAAI,CAAC,CAAC;QAC3D,QAAQ,EAAE,OAAO,CAAC,QAAQ;QAC1B,QAAQ,EAAE,OAAO,CAAC,QAAQ;QAC1B,YAAY,EAAE,UAAU,GAAG,CAAC;QAC5B,UAAU,EAAE,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,GAAG,EAAE,GAAG,OAAO,CAAC;QACnD,eAAe,EAAE,OAAO,CAAC,aAAa;KACvC,CAAC;IAEF,OAAO;QACL,QAAQ;QACR,OAAO;QACP,UAAU;QACV,WAAW;QACX,cAAc;QACd,eAAe,EAAE,OAAO,CAAC,kBAAkB,CAAC,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,SAAS;QACzE,MAAM,EAAE,MAAM,CAAC,IAAI;QACnB,YAAY,EAAE,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI;QACtD,eAAe,EAAE,OAAO;QACxB,QAAQ;QACR,MAAM;QACN,KAAK;KACN,CAAC;AACJ,CAAC,CAAC"}
+{"version":3,"file":"crawl.js","sourceRoot":"","sources":["../../src/core/crawl.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAC;AACxD,OAAO,EACL,kBAAkB,EAClB,SAAS,EACT,cAAc,EACd,QAAQ,EACR,cAAc,GACf,MAAM,YAAY,CAAC;AAEpB,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAChD,OAAO,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAC5C,OAAO,EAAE,WAAW,EAAE,MAAM,sBAAsB,CAAC;AACnD,OAAO,EAAE,WAAW,EAAE,gBAAgB,EAAE,MAAM,kBAAkB,CAAC;AAGjE,OAAO,EAAE,oBAAoB,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAOlE,OAAO,EACL,MAAM,EACN,UAAU,EACV,aAAa,EACb,YAAY,EACZ,QAAQ,GACT,MAAM,UAAU,CAAC;AAyDlB,+EAA+E;AAC/E,MAAM,gBAAgB,GAAG,GAAG,CAAC;AAO7B;;;;GAIG;AACH,MAAM,QAAQ,GAAG,KAAK,EACpB,KAAkB,EAClB,WAAmB,EACnB,OAAe,EACf,IAAwC,EACzB,EAAE;IACjB,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,WAAW,CAAC,CAAC;IACvC,MAAM,OAAO,GAAG,IAAI,GAAG,EAAiB,CAAC;IAEzC,MAAM,MAAM,GAAG,CAAC,IAAe,EAAQ,EAAE;QACvC,MAAM,IAAI,GAAG,CAAC,KAAK,IAAI,EAAE;YACvB,IAAI,CAAC;gBACH,MAAM,IAAI,CAAC,IAAI,CAAC,CAAC;YACnB,CAAC;YAAC,MAAM,CAAC;gBACP,oEAAoE;YACtE,CAAC;YAED,IAAI,OAAO,GAAG,CAAC,EAAE,CAAC;gBAChB,MAAM,KAAK,CAAC,OAAO,CAAC,CAAC;YACvB,CAAC;QACH,CAAC,CAAC,EAAE,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC;QAEzC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;IACpB,CAAC,CAAC;IAEF,SAAS,CAAC;QACR,OAAO,OAAO,CAAC,IAAI,GAAG,KAAK,EAAE,CAAC;YAC5B,MAAM,IAAI,GAAG,KAAK,CAAC,KAAK,EAAE,CAAC;YAE3B,IAAI,CAAC,IAAI,EAAE,CAAC;gBACV,MAAM;YACR,CAAC;YAED,MAAM,CAAC,IAAI,CAAC,CAAC;QACf,CAAC;QAED,IAAI,OAAO,CAAC,IAAI,KAAK,CAAC,EAAE,CAAC;YACvB,OAAO;QACT,CAAC;QAED,6IAA6I;QAC7I,MAAM,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IAC9B,CAAC;AACH,CAAC,CAAC;AAEF,+DAA+D;AAC/D,MAAM,SAAS,GAAG,CAChB,OAAiC,EACjC,MAAc,EACd,MAAc,EACR,EAAE;IACR,IAAI,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;IAE9B,IAAI,CAAC,GAAG,EAAE,CAAC;QACT,GAAG,GAAG,IAAI,GAAG,EAAE,CAAC;QAChB,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;IAC3B,CAAC;IAED,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;AAClB,CAAC,CAAC;AAEF,MAAM,SAAS,GAAG,KAAK,EACrB,QAAgB,EAChB,SAAuB,EACA,EAAE;IACzB,MAAM,MAAM,GAAiB,EAAE,CAAC;IAChC,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,QAAQ,CAAC,CAAC;IAE9B,MAAM,UAAU,GAAG,IAAI,GAAG,CACxB,sBAAsB,IAAI,CAAC,GAAG,EAAE,CAAC,QAAQ,CAAC,EAAE,CAAC,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EACzF,GAAG,CAAC,MAAM,CACX,CAAC,IAAI,CAAC;IAEP,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,UAAU,EAAE,SAAS,CAAC,CAAC;IACnD,MAAM,CAAC,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,UAAU,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,CAAC;IAE9E,IAAI,WAAW,CAAC,QAAQ,CAAC,EAAE,CAAC;QAC1B,OAAO,MAAM,CAAC;IAChB,CAAC;IAED,IAAI,GAAG,CAAC,QAAQ,KAAK,QAAQ,EAAE,CAAC;QAC9B,MAAM,OAAO,GAAG,UAAU,GAAG,CAAC,IAAI,GAAG,CAAC;QACtC,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,OAAO,EAAE,EAAE,GAAG,SAAS,EAAE,SAAS,EAAE,MAAM,EAAE,CAAC,CAAC;QACzE,MAAM,CAAC,YAAY,GAAG,GAAG,CAAC,KAAK;YAC7B,CAAC,CAAC,IAAI;YACN,CAAC,CAAC;gBACE,GAAG,EAAE,OAAO;gBACZ,MAAM,EAAE,GAAG,CAAC,MAAM;gBAClB,QAAQ,EAAE,GAAG,CAAC,QAAQ;gBACtB,gBAAgB,EAAE,GAAG,CAAC,QAAQ,CAAC,UAAU,CAAC,UAAU,CAAC;aACtD,CAAC;IACR,CAAC;IAED,MAAM,IAAI,GAAG,GAAG,CAAC,QAAQ,CAAC;IAC1B,MAAM,WAAW,GAAG,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,OAAO,IAAI,EAAE,CAAC;IAC7E,MAAM,UAAU,GAAG,GAAG,GAAG,CAAC,QAAQ,KAAK,WAAW,GAAG,CAAC;IACtD,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,UAAU,EAAE,EAAE,GAAG,SAAS,EAAE,SAAS,EAAE,MAAM,EAAE,CAAC,CAAC;IAC5E,MAAM,CAAC,WAAW,GAAG,GAAG,CAAC,KAAK;QAC5B,CAAC,CAAC,IAAI;QACN,CAAC,CAAC;YACE,GAAG,EAAE,UAAU;YACf,MAAM,EAAE,GAAG,CAAC,MAAM;YAClB,QAAQ,EAAE,GAAG,CAAC,QAAQ;YACtB,eAAe,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC,KAAK,IAAI;SACvD,CAAC;IAEN,OAAO,MAAM,CAAC;AAChB,CAAC,CAAC;AAEF,gGAAgG;AAChG,MAAM,eAAe,GAAG,CACtB,OAAqB,EACrB,SAAiB,EACjB,iBAA0B,EAChB,EAAE;IACZ,MAAM,GAAG,GAAG,IAAI,GAAG,EAAU,CAAC;IAE9B,KAAK,MAAM,CAAC,IAAI,OAAO,EAAE,CAAC;QACxB,KAAK,MAAM,GAAG,IAAI;YAChB,CAAC,CAAC,IAAI,CAAC,SAAS;YAChB,GAAG,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC;SACrC,EAAE,CAAC;YACF,MAAM,MAAM,GAAG,GAAG,CAAC,CAAC,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;YAE9C,IAAI,MAAM,IAAI,UAAU,CAAC,MAAM,EAAE,SAAS,EAAE,iBAAiB,CAAC,EAAE,CAAC;gBAC/D,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;YAClB,CAAC;QACH,CAAC;IACH,CAAC;IAED,OAAO,CAAC,GAAG,GAAG,CAAC,CAAC;AAClB,CAAC,CAAC;AAEF,2EAA2E;AAC3E,MAAM,cAAc,GAAG,GAAS,EAAE;IAChC,kBAAkB;AACpB,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,SAAS,GAAG,KAAK,EAC5B,UAAkB,EAClB,OAAqB,EACC,EAAE;IACxB,MAAM,OAAO,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC;IAClC,MAAM,QAAQ,GAAG,OAAO,CAAC,UAAU,IAAI,cAAc,CAAC;IACtD,MAAM,QAAQ,GAAG,YAAY,CAAC,UAAU,CAAC,IAAI,UAAU,CAAC;IACxD,MAAM,EAAE,MAAM,EAAE,GAAG,IAAI,GAAG,CAAC,QAAQ,CAAC,CAAC;IACrC,MAAM,SAAS,GAAG,MAAM,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC;IACzC,MAAM,SAAS,GAAG,OAAO,CAAC,KAAK,CAAC,SAAS,IAAI,kBAAkB,CAAC;IAEhE,QAAQ,CAAC,EAAE,KAAK,EAAE,QAAQ,EAAE,OAAO,EAAE,YAAY,MAAM,aAAa,EAAE,CAAC,CAAC;IACxE,MAAM,MAAM,GAAG,MAAM,aAAa,CAAC,MAAM,EAAE,SAAS,EAAE,OAAO,CAAC,KAAK,CAAC,CAAC;IACrE,MAAM,MAAM,GAAG,OAAO,CAAC,aAAa,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC;IAElE,MAAM,UAAU,GAAG,OAAO,CAAC,UAAU,IAAI,IAAI,CAAC;IAE9C,IAAI,QAAQ,GAAgB;QAC1B,OAAO,EAAE,EAAE;QACX,IAAI,EAAE,EAAE;QACR,QAAQ,EAAE,CAAC;QACX,MAAM,EAAE,EAAE;QACV,QAAQ,EAAE,oBAAoB,EAAE;KACjC,CAAC;IAEF,IAAI,UAAU,EAAE,CAAC;QACf,QAAQ,CAAC,EAAE,KAAK,EAAE,SAAS,EAAE,OAAO,EAAE,mBAAmB,EAAE,CAAC,CAAC;QAE7D,MAAM,iBAAiB,GAAG;YACxB,GAAG,IAAI,GAAG,CAAC;gBACT,GAAG,MAAM,CAAC,IAAI,CAAC,QAAQ;gBACvB,IAAI,GAAG,CAAC,cAAc,EAAE,MAAM,CAAC,CAAC,IAAI;aACrC,CAAC;SACH,CAAC;QAEF,QAAQ,GAAG,MAAM,YAAY,CAAC,iBAAiB,EAAE,OAAO,CAAC,KAAK,EAAE;YAC9D,QAAQ,EAAE,SAAS;YACnB,iBAAiB,EAAE,OAAO,CAAC,iBAAiB;SAC7C,CAAC,CAAC;IACL,CAAC;IAED,MAAM,OAAO,GAAG,IAAI,GAAG,CAAS,CAAC,QAAQ,CAAC,CAAC,CAAC;IAC5C,MAAM,UAAU,GAAG,IAAI,GAAG,CAAS,CAAC,QAAQ,CAAC,CAAC,CAAC;IAC/C,MAAM,UAAU,GAAG,IAAI,GAAG,EAAqB,CAAC;IAChD,MAAM,WAAW,GAAG,IAAI,GAAG,EAAuB,CAAC;IACnD,MAAM,eAAe,GAAG,IAAI,GAAG,EAAuB,CAAC;IACvD,MAAM,OAAO,GAAiB,EAAE,CAAC;IACjC,yFAAyF;IACzF,MAAM,QAAQ,GAAG,IAAI,GAAG,EAAU,CAAC;IACnC,MAAM,OAAO,GAAa,EAAE,CAAC;IAC7B,IAAI,UAAU,GAAG,CAAC,CAAC;IAEnB,MAAM,OAAO,GAAG,CAAC,GAAW,EAAW,EAAE;QACvC,IACE,OAAO,CAAC,eAAe,CAAC,MAAM;YAC9B,CAAC,OAAO,CAAC,eAAe,CAAC,IAAI,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EACnD,CAAC;YACD,OAAO,KAAK,CAAC;QACf,CAAC;QAED,IAAI,OAAO,CAAC,eAAe,CAAC,IAAI,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC;YACvD,OAAO,KAAK,CAAC;QACf,CAAC;QAED,OAAO,IAAI,CAAC;IACd,CAAC,CAAC;IAEF,MAAM,KAAK,GAAgB,CAAC,EAAE,GAAG,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC,CAAC;IAEzD,wFAAwF;IACxF,MAAM,WAAW,GAAG,CAClB,KAAiB,EACjB,IAAY,EACZ,KAAa,EACP,EAAE;QACR,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,YAAY,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;YAExD,IAAI,CAAC,MAAM,EAAE,CAAC;gBACZ,SAAS;YACX,CAAC;YAED,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,CAAC;gBACnB,SAAS,CAAC,eAAe,EAAE,MAAM,EAAE,IAAI,CAAC,CAAC;gBACzC,SAAS;YACX,CAAC;YAED,SAAS,CAAC,WAAW,EAAE,MAAM,EAAE,IAAI,CAAC,CAAC;YAErC,IACE,CAAC,aAAa,CAAC,MAAM,CAAC;gBACtB,CAAC,OAAO,CAAC,MAAM,CAAC;gBAChB,UAAU,CAAC,GAAG,CAAC,MAAM,CAAC,EACtB,CAAC;gBACD,SAAS;YACX,CAAC;YAED,UAAU,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;YAEvB,IAAI,KAAK,GAAG,CAAC,GAAG,OAAO,CAAC,QAAQ,EAAE,CAAC;gBACjC,SAAS;YACX,CAAC;YAED,IAAI,OAAO,CAAC,IAAI,IAAI,OAAO,CAAC,QAAQ,EAAE,CAAC;gBACrC,UAAU,IAAI,CAAC,CAAC;gBAChB,SAAS;YACX,CAAC;YAED,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;YACpB,KAAK,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,KAAK,GAAG,CAAC,EAAE,CAAC,CAAC;QAChD,CAAC;IACH,CAAC,CAAC;IAEF;;;OAGG;IACH,MAAM,mBAAmB,GAAG,CAAC,GAAgB,EAAE,QAAgB,EAAW,EAAE;QAC1E,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC;YAC9B,UAAU,CAAC,GAAG,CAAC,QAAQ,EAAE;gBACvB,GAAG,EAAE,GAAG,CAAC,QAAQ;gBACjB,MAAM,EAAE,GAAG,CAAC,MAAM;gBAClB,QAAQ,EAAE,GAAG,CAAC,QAAQ;gBACtB,aAAa,EAAE,EAAE;gBACjB,KAAK,EAAE,GAAG,CAAC,KAAK;aACjB,CAAC,CAAC;QACL,CAAC;QAED,IAAI,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,KAAK,QAAQ,CAAC,EAAE,CAAC;YACrE,OAAO,KAAK,CAAC;QACf,CAAC;QAED,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;QAEtB,OAAO,UAAU,CAAC,QAAQ,EAAE,SAAS,EAAE,OAAO,CAAC,iBAAiB,CAAC,CAAC;IACpE,CAAC,CAAC;IAEF,iGAAiG;IACjG,MAAM,gBAAgB,GAAG,CACvB,IAAe,EACf,GAAgB,EAChB,QAAgB,EACV,EAAE;QACR,MAAM,OAAO,GACX,GAAG,CAAC,MAAM,KAAK,IAAI;YACnB,GAAG,CAAC,MAAM,GAAG,GAAG;YAChB,CAAC,GAAG,CAAC,KAAK;YACV,GAAG,CAAC,IAAI,KAAK,IAAI;YACjB,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC;QAEvB,8FAA8F;QAC9F,IAAI,OAAO,IAAI,CAAC,IAAI,CAAC,KAAK,KAAK,CAAC,IAAI,GAAG,CAAC,MAAM,KAAK,IAAI,CAAC,EAAE,CAAC;YACzD,OAAO;QACT,CAAC;QAED,IAAI,QAAQ,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC;YAC3B,OAAO;QACT,CAAC;QAED,QAAQ,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;QAEvB,MAAM,IAAI,GAAG,WAAW,CAAC,GAAG,CAAC,IAAI,IAAI,EAAE,EAAE;YACvC,GAAG,EAAE,GAAG,CAAC,QAAQ;YACjB,QAAQ,EAAE,SAAS;YACnB,iBAAiB,EAAE,OAAO,CAAC,iBAAiB;SAC7C,CAAC,CAAC;QAEH,OAAO,CAAC,IAAI,CAAC;YACX,GAAG,EAAE,QAAQ;YACb,YAAY,EAAE,IAAI,CAAC,GAAG;YACtB,IAAI;YACJ,QAAQ,EAAE,cAAc,CAAC,GAAG,CAAC;YAC7B,KAAK,EAAE,IAAI,CAAC,KAAK;SAClB,CAAC,CAAC;IACL,CAAC,CAAC;IAEF,MAAM,WAAW,GAAG,KAAK,EAAE,IAAe,EAAiB,EAAE;QAC3D,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;YACtB,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YAEvB,OAAO;QACT,CAAC;QAED,MAAM,GAAG,GAAG,MAAM,SAAS,CAAC,IAAI,CAAC,GAAG,EAAE,OAAO,CAAC,KAAK,CAAC,CAAC;QACrD,MAAM,YAAY,GAAG,IAAI,CAAC,GAAG,CAAC;QAC9B,MAAM,QAAQ,GAAG,YAAY,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,GAAG,CAAC,QAAQ,CAAC;QAE5D,UAAU,CAAC,GAAG,CAAC,YAAY,EAAE;YAC3B,GAAG,EAAE,IAAI,CAAC,GAAG;YACb,MAAM,EAAE,GAAG,CAAC,MAAM;YAClB,QAAQ,EAAE,GAAG,CAAC,QAAQ;YACtB,aAAa,EAAE,GAAG,CAAC,aAAa;YAChC,KAAK,EAAE,GAAG,CAAC,KAAK;SACjB,CAAC,CAAC;QAEH,IAAI,QAAQ,KAAK,YAAY,IAAI,CAAC,mBAAmB,CAAC,GAAG,EAAE,QAAQ,CAAC,EAAE,CAAC;YACrE,OAAO;QACT,CAAC;QAED,QAAQ,CAAC;YACP,KAAK,EAAE,OAAO;YACd,OAAO,EAAE,GAAG,GAAG,CAAC,MAAM,IAAI,KAAK,IAAI,GAAG,CAAC,QAAQ,EAAE;YACjD,OAAO,EAAE,OAAO,CAAC,MAAM,GAAG,CAAC;YAC3B,UAAU,EAAE,UAAU,CAAC,IAAI;YAC3B,GAAG,EAAE,GAAG,CAAC,QAAQ;SAClB,CAAC,CAAC;QAEH,IAAI,GAAG,CAAC,KAAK,IAAI,GAAG,CAAC,IAAI,KAAK,IAAI,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,EAAE,CAAC;YAC3D,gBAAgB,CAAC,IAAI,EAAE,GAAG,EAAE,QAAQ,CAAC,CAAC;YAEtC,OAAO;QACT,CAAC;QAED,0FAA0F;QAC1F,iCAAiC;QACjC,IAAI,QAAQ,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC;YAC3B,OAAO;QACT,CAAC;QAED,QAAQ,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;QAEvB,MAAM,IAAI,GAAG,WAAW,CAAC,GAAG,CAAC,IAAI,EAAE;YACjC,GAAG,EAAE,GAAG,CAAC,QAAQ;YACjB,QAAQ,EAAE,SAAS;YACnB,iBAAiB,EAAE,OAAO,CAAC,iBAAiB;SAC7C,CAAC,CAAC;QAEH,OAAO,CAAC,IAAI,CAAC;YACX,GAAG,EAAE,QAAQ;YACb,YAAY,EAAE,IAAI,CAAC,GAAG;YACtB,IAAI;YACJ,QAAQ,EAAE,cAAc,CAAC,GAAG,CAAC;YAC7B,KAAK,EAAE,IAAI,CAAC,KAAK;SAClB,CAAC,CAAC;QAEH,WAAW,CAAC,IAAI,CAAC,KAAK,EAAE,QAAQ,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC;IAChD,CAAC,CAAC;IAEF,MAAM,QAAQ,CAAC,KAAK,EAAE,OAAO,CAAC,WAAW,EAAE,OAAO,CAAC,OAAO,EAAE,WAAW,CAAC,CAAC;IAEzE,iGAAiG;IACjG,QAAQ,CAAC,EAAE,KAAK,EAAE,OAAO,EAAE,OAAO,EAAE,gCAAgC,EAAE,CAAC,CAAC;IAExE,MAAM,OAAO,GAAG,CAAC,GAAG,WAAW,CAAC,IAAI,EAAE,CAAC;SACpC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,MAAM,CAAC,CAAC,CAAC,CAAC;SAC9C,KAAK,CAAC,CAAC,EAAE,OAAO,CAAC,aAAa,CAAC,CAAC;IAEnC,MAAM,YAAY,GAAG,QAAQ,CAAC,IAAI;SAC/B,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,YAAY,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;SAChC,MAAM,CACL,CAAC,CAAC,EAAE,EAAE,CACJ,UAAU,CAAC,CAAC,EAAE,SAAS,EAAE,OAAO,CAAC,iBAAiB,CAAC;QACnD,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC;QAClB,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC;QACpB,MAAM,CAAC,CAAC,CAAC,CACZ;SACA,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,OAAO,CAAC,aAAa,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC;IAEjE,wFAAwF;IACxF,MAAM,aAAa,GAAG,eAAe,CACnC,OAAO,EACP,SAAS,EACT,OAAO,CAAC,iBAAiB,CAC1B;SACE,MAAM,CACL,CAAC,CAAC,EAAE,EAAE,CACJ,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC;QAClB,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC;QACpB,CAAC,YAAY,CAAC,QAAQ,CAAC,CAAC,CAAC;QACzB,MAAM,CAAC,CAAC,CAAC,CACZ;SACA,KAAK,CACJ,CAAC,EACD,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,OAAO,CAAC,aAAa,GAAG,OAAO,CAAC,MAAM,GAAG,YAAY,CAAC,MAAM,CAAC,CAC1E,CAAC;IAEJ,KAAK,MAAM,MAAM,IAAI,CAAC,GAAG,WAAW,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;QACvE,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;YAC9B,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QACvB,CAAC;IACH,CAAC;IAED,MAAM,aAAa,CACjB,CAAC,GAAG,OAAO,EAAE,GAAG,YAAY,EAAE,GAAG,aAAa,CAAC,EAC/C,OAAO,CAAC,WAAW,EACnB,KAAK,EAAE,CAAC,EAAE,EAAE;QACV,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,CAAC,EAAE,OAAO,CAAC,KAAK,CAAC,CAAC;QAC7C,UAAU,CAAC,GAAG,CAAC,CAAC,EAAE;YAChB,GAAG,EAAE,CAAC;YACN,MAAM,EAAE,GAAG,CAAC,MAAM;YAClB,QAAQ,EAAE,GAAG,CAAC,QAAQ;YACtB,aAAa,EAAE,GAAG,CAAC,aAAa;YAChC,KAAK,EAAE,GAAG,CAAC,KAAK;SACjB,CAAC,CAAC;QAEH,IAAI,OAAO,CAAC,OAAO,EAAE,CAAC;YACpB,MAAM,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;QAC/B,CAAC;IACH,CAAC,CACF,CAAC;IAEF,IAAI,cAAkD,CAAC;IAEvD,IAAI,OAAO,CAAC,kBAAkB,EAAE,CAAC;QAC/B,QAAQ,CAAC;YACP,KAAK,EAAE,OAAO;YACd,OAAO,EAAE,YAAY,IAAI,CAAC,GAAG,CAAC,eAAe,CAAC,IAAI,EAAE,OAAO,CAAC,aAAa,CAAC,iBAAiB;SAC5F,CAAC,CAAC;QACH,MAAM,MAAM,GAAG,IAAI,GAAG,EAAqB,CAAC;QAC5C,cAAc,GAAG,MAAM,CAAC;QACxB,MAAM,OAAO,GAAG,CAAC,GAAG,eAAe,CAAC,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,OAAO,CAAC,aAAa,CAAC,CAAC;QAC5E,MAAM,aAAa,CACjB,OAAO,EACP,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,WAAW,EAAE,CAAC,CAAC,EAChC,KAAK,EAAE,CAAC,EAAE,EAAE;YACV,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,CAAC,EAAE;gBAC5B,GAAG,OAAO,CAAC,KAAK;gBAChB,SAAS,EAAE,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,KAAK,CAAC,SAAS,IAAI,MAAM,EAAE,MAAM,CAAC;aAC/D,CAAC,CAAC;YAEH,MAAM,CAAC,GAAG,CAAC,CAAC,EAAE;gBACZ,GAAG,EAAE,CAAC;gBACN,MAAM,EAAE,GAAG,CAAC,MAAM;gBAClB,QAAQ,EAAE,GAAG,CAAC,QAAQ;gBACtB,aAAa,EAAE,GAAG,CAAC,aAAa;gBAChC,KAAK,EAAE,GAAG,CAAC,KAAK;aACjB,CAAC,CAAC;QACL,CAAC,CACF,CAAC;IACJ,CAAC;IAED,IAAI,MAAM,GAAiB,EAAE,CAAC;IAC9B,IAAI,MAAoC,CAAC;IAEzC,IAAI,UAAU,EAAE,CAAC;QACf,MAAM,YAAY,GAAG;YACnB,KAAK,EAAE,OAAO,CAAC,KAAK;YACpB,WAAW,EAAE,OAAO,CAAC,WAAW;YAChC,OAAO,EAAE,OAAO,CAAC,OAAO;YACxB,MAAM;YACN,QAAQ;YACR,iBAAiB,EAAE,OAAO,CAAC,iBAAiB;YAC5C,aAAa,EAAE,OAAO,CAAC,kBAAkB;SAC1C,CAAC;QAEF,QAAQ,CAAC,EAAE,KAAK,EAAE,OAAO,EAAE,OAAO,EAAE,iBAAiB,EAAE,CAAC,CAAC;QACzD,MAAM,GAAG,MAAM,WAAW,CACxB,OAAO,EACP,IAAI,CAAC,GAAG,CAAC,gBAAgB,EAAE,OAAO,CAAC,aAAa,CAAC,EACjD,YAAY,CACb,CAAC;QAEF,QAAQ,CAAC;YACP,KAAK,EAAE,QAAQ;YACf,OAAO,EAAE,mDAAmD;SAC7D,CAAC,CAAC;QACH,MAAM,GAAG,MAAM,SAAS,CAAC,QAAQ,EAAE,OAAO,CAAC,KAAK,CAAC,CAAC;QAClD,MAAM,CAAC,WAAW,GAAG,MAAM,gBAAgB,CACzC,OAAO,EACP,QAAQ,EACR,YAAY,CACb,CAAC;IACJ,CAAC;IAED,MAAM,KAAK,GAAe;QACxB,YAAY,EAAE,OAAO,CAAC,MAAM;QAC5B,eAAe,EAAE,UAAU,CAAC,IAAI;QAChC,YAAY,EAAE,UAAU,CAAC,IAAI,GAAG,CAAC,cAAc,EAAE,IAAI,IAAI,CAAC,CAAC;QAC3D,QAAQ,EAAE,OAAO,CAAC,QAAQ;QAC1B,QAAQ,EAAE,OAAO,CAAC,QAAQ;QAC1B,YAAY,EAAE,UAAU,GAAG,CAAC;QAC5B,UAAU,EAAE,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,GAAG,EAAE,GAAG,OAAO,CAAC;QACnD,eAAe,EAAE,OAAO,CAAC,aAAa;KACvC,CAAC;IAEF,OAAO;QACL,QAAQ;QACR,OAAO;QACP,UAAU;QACV,WAAW;QACX,cAAc;QACd,eAAe,EAAE,OAAO,CAAC,kBAAkB,CAAC,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,SAAS;QACzE,MAAM,EAAE,MAAM,CAAC,IAAI;QACnB,UAAU,EAAE,MAAM,CAAC,IAAI;QACvB,YAAY,EAAE,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI;QACtD,eAAe,EAAE,OAAO;QACxB,QAAQ;QACR,MAAM;QACN,KAAK;QACL,WAAW,EAAE,MAAM,EAAE,WAAW;QAChC,YAAY,EAAE,MAAM,EAAE,YAAY;KACnC,CAAC;AACJ,CAAC,CAAC"}

package/dist/core/fetch.d.ts

@@ -7,6 +7,11 @@ export interface FetchOptions {
     maxRedirects?: number;
     /** Stop reading the body after this many bytes. */
     maxBodyBytes?: number;
+    /**
+     * Return the body as bytes (`bytes`) instead of text (`body` is then ""), for files that are not
+     * text, such as a gzip sitemap.
+     */
+    keepBytes?: boolean;
 }
 export interface FetchResult {
     requestedUrl: string;
@@ -18,6 +23,10 @@ export interface FetchResult {
     timeMs: number;
     contentType: string | null;
     error?: string;
+    /** The body stopped at `maxBodyBytes`; the rest was not read. */
+    truncated?: boolean;
+    /** The body's bytes, with `keepBytes`. */
+    bytes?: Buffer;
 }
 /** GET a URL, following redirects manually so the chain is recorded. */
 export declare const fetchPage: (url: string, options?: FetchOptions, readBody?: boolean) => Promise<FetchResult>;

package/dist/core/fetch.js

@@ -2,6 +2,7 @@ import { VERSION } from "../version.js";
 export const DEFAULT_USER_AGENT = `Mozilla/5.0 (compatible; agentic-seo-check/${VERSION}; +https://github.com/rivi-software/agentic-seo-check)`;
 const KEPT_HEADERS = [
     "content-type",
+    "content-encoding",
     "x-robots-tag",
     "link",
     "content-language",
@@ -10,6 +11,7 @@ const KEPT_HEADERS = [
     "location",
     "content-length",
     "vary",
+    "content-security-policy",
 ];
 const REDIRECT_STATUSES = new Set([301, 302, 303, 307, 308]);
 const pickHeaders = (headers) => {
@@ -40,7 +42,7 @@ const result = (requestedUrl, finalUrl, status, headers, body, chain, started, e
 };
 const readLimited = async (res, maxBytes) => {
     if (!res.body) {
-        return "";
+        return { bytes: Buffer.alloc(0), truncated: false };
     }
     const reader = res.body.getReader();
     const chunks = [];
@@ -64,12 +66,17 @@ const readLimited = async (res, maxBytes) => {
     if (truncated) {
         await reader.cancel().catch(() => null);
     }
-    const charset = /charset=(?<charset>[\w-]+)/iu.exec(res.headers.get("content-type") ?? "")?.groups?.charset;
+    return { bytes: Buffer.concat(chunks), truncated };
+};
+/** The body as text, in the charset the Content-Type header names (UTF-8 otherwise). */
+const decodeBody = (bytes, contentType) => {
+    const charset = /charset=(?<charset>[\w-]+)/iu.exec(contentType ?? "")?.groups
+        ?.charset;
     try {
-        return new TextDecoder(charset ?? "utf-8").decode(Buffer.concat(chunks));
+        return new TextDecoder(charset ?? "utf-8").decode(bytes);
     }
     catch {
-        return new TextDecoder("utf-8").decode(Buffer.concat(chunks));
+        return new TextDecoder("utf-8").decode(bytes);
     }
 };
 const fetchHop = async (walk, current) => {
@@ -108,16 +115,25 @@ const fetchHop = async (walk, current) => {
             }
             return { next: nextUrl };
         }
-        let body = null;
-        if (readBody) {
-            body = await readLimited(res, options.maxBodyBytes ?? 5_000_000);
-        }
-        else {
+        if (!readBody) {
             await res.body?.cancel().catch(() => null);
+            return {
+                done: result(url, current, res.status, headers, null, chain, started),
+            };
         }
-        return {
-            done: result(url, current, res.status, headers, body, chain, started),
-        };
+        const read = await readLimited(res, options.maxBodyBytes ?? 5_000_000);
+        const done = result(url, current, res.status, headers, 
+        // The caller reads `bytes`; decoding them too would hold the body a second time.
+        options.keepBytes
+            ? ""
+            : decodeBody(read.bytes, res.headers.get("content-type")), chain, started);
+        if (read.truncated) {
+            done.truncated = true;
+        }
+        if (options.keepBytes) {
+            done.bytes = read.bytes;
+        }
+        return { done };
     }
     catch (error) {
         let message = String(error);

package/dist/core/fetch.js.map

@@ -1 +1 @@
-{"version":3,"file":"fetch.js","sourceRoot":"","sources":["../../src/core/fetch.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,eAAe,CAAC;AAGxC,MAAM,CAAC,MAAM,kBAAkB,GAAG,8CAA8C,OAAO,wDAAwD,CAAC;AAEhJ,MAAM,YAAY,GAAG;IACnB,cAAc;IACd,cAAc;IACd,MAAM;IACN,kBAAkB;IAClB,eAAe;IACf,eAAe;IACf,UAAU;IACV,gBAAgB;IAChB,MAAM;CACP,CAAC;AAuBF,MAAM,iBAAiB,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC;AAE7D,MAAM,WAAW,GAAG,CAAC,OAAgB,EAAE,EAAE;IACvC,MAAM,GAAG,GAA2B,EAAE,CAAC;IAEvC,KAAK,MAAM,IAAI,IAAI,YAAY,EAAE,CAAC;QAChC,MAAM,KAAK,GAAG,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QAEhC,IAAI,KAAK,KAAK,IAAI,EAAE,CAAC;YACnB,GAAG,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC;QACpB,CAAC;IACH,CAAC;IAED,OAAO,GAAG,CAAC;AACb,CAAC,CAAC;AAEF,MAAM,MAAM,GAAG,CACb,YAAoB,EACpB,QAAgB,EAChB,MAAqB,EACrB,OAA+B,EAC/B,IAAmB,EACnB,KAAoB,EACpB,OAAe,EACf,KAAc,EACD,EAAE;IACf,MAAM,CAAC,GAAgB;QACrB,YAAY;QACZ,QAAQ;QACR,MAAM;QACN,OAAO;QACP,IAAI;QACJ,aAAa,EAAE,KAAK;QACpB,MAAM,EAAE,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,GAAG,EAAE,GAAG,OAAO,CAAC;QAC/C,WAAW,EAAE,OAAO,CAAC,cAAc,CAAC,IAAI,IAAI;KAC7C,CAAC;IAEF,IAAI,KAAK,EAAE,CAAC;QACV,CAAC,CAAC,KAAK,GAAG,KAAK,CAAC;IAClB,CAAC;IAED,OAAO,CAAC,CAAC;AACX,CAAC,CAAC;AAEF,MAAM,WAAW,GAAG,KAAK,EACvB,GAAa,EACb,QAAgB,EACC,EAAE;IACnB,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC;QACd,OAAO,EAAE,CAAC;IACZ,CAAC;IAED,MAAM,MAAM,GAAG,GAAG,CAAC,IAAI,CAAC,SAAS,EAAE,CAAC;IACpC,MAAM,MAAM,GAAiB,EAAE,CAAC;IAChC,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,IAAI,SAAS,GAAG,KAAK,CAAC;IAEtB,SAAS,CAAC;QACR,uGAAuG;QACvG,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,MAAM,MAAM,CAAC,IAAI,EAAE,CAAC;QAE5C,IAAI,IAAI,EAAE,CAAC;YACT,MAAM;QACR,CAAC;QAED,IAAI,KAAK,EAAE,CAAC;YACV,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACnB,KAAK,IAAI,KAAK,CAAC,UAAU,CAAC;YAE1B,IAAI,KAAK,IAAI,QAAQ,EAAE,CAAC;gBACtB,SAAS,GAAG,IAAI,CAAC;gBACjB,MAAM;YACR,CAAC;QACH,CAAC;IACH,CAAC;IAED,IAAI,SAAS,EAAE,CAAC;QACd,MAAM,MAAM,CAAC,MAAM,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,CAAC;IAC1C,CAAC;IAED,MAAM,OAAO,GAAG,8BAA8B,CAAC,IAAI,CACjD,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,IAAI,EAAE,CACtC,EAAE,MAAM,EAAE,OAAO,CAAC;IAEnB,IAAI,CAAC;QACH,OAAO,IAAI,WAAW,CAAC,OAAO,IAAI,OAAO,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC;IAC3E,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,WAAW,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC;IAChE,CAAC;AACH,CAAC,CAAC;AAeF,MAAM,QAAQ,GAAG,KAAK,EACpB,IAAkB,EAClB,OAAe,EACM,EAAE;IACvB,MAAM,EAAE,GAAG,EAAE,OAAO,EAAE,QAAQ,EAAE,SAAS,EAAE,OAAO,EAAE,KAAK,EAAE,GAAG,IAAI,CAAC;IACnE,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;IACzC,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,UAAU,CAAC,KAAK,EAAE,EAAE,SAAS,CAAC,CAAC;IAE9D,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,OAAO,EAAE;YAC/B,MAAM,EAAE,KAAK;YACb,QAAQ,EAAE,QAAQ;YAClB,MAAM,EAAE,UAAU,CAAC,MAAM;YACzB,OAAO,EAAE;gBACP,YAAY,EAAE,OAAO,CAAC,SAAS,IAAI,kBAAkB;gBACrD,MAAM,EACJ,iEAAiE;gBACnE,iBAAiB,EAAE,YAAY;gBAC/B,GAAG,OAAO,CAAC,OAAO;aACnB;SACF,CAAC,CAAC;QAEH,MAAM,OAAO,GAAG,WAAW,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;QAEzC,IAAI,iBAAiB,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,OAAO,CAAC,UAAU,CAAC,EAAE,CAAC;YAC7D,KAAK,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,OAAO,EAAE,MAAM,EAAE,GAAG,CAAC,MAAM,EAAE,CAAC,CAAC;YACjD,IAAI,OAAe,CAAC;YAEpB,IAAI,CAAC;gBACH,OAAO,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,UAAU,CAAC,EAAE,OAAO,CAAC,CAAC,IAAI,CAAC;YACvD,CAAC;YAAC,MAAM,CAAC;gBACP,OAAO;oBACL,IAAI,EAAE,MAAM,CACV,GAAG,EACH,OAAO,EACP,GAAG,CAAC,MAAM,EACV,OAAO,EACP,IAAI,EACJ,KAAK,EACL,OAAO,EACP,4BAA4B,OAAO,CAAC,UAAU,CAAC,EAAE,CAClD;iBACF,CAAC;YACJ,CAAC;YAED,MAAM,GAAG,CAAC,IAAI,EAAE,MAAM,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,CAAC;YAE3C,IAAI,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,KAAK,OAAO,CAAC,IAAI,OAAO,KAAK,OAAO,EAAE,CAAC;gBAChE,OAAO;oBACL,IAAI,EAAE,MAAM,CACV,GAAG,EACH,OAAO,EACP,GAAG,CAAC,MAAM,EACV,OAAO,EACP,IAAI,EACJ,KAAK,EACL,OAAO,EACP,eAAe,CAChB;iBACF,CAAC;YACJ,CAAC;YAED,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,CAAC;QAC3B,CAAC;QAED,IAAI,IAAI,GAAkB,IAAI,CAAC;QAE/B,IAAI,QAAQ,EAAE,CAAC;YACb,IAAI,GAAG,MAAM,WAAW,CAAC,GAAG,EAAE,OAAO,CAAC,YAAY,IAAI,SAAS,CAAC,CAAC;QACnE,CAAC;aAAM,CAAC;YACN,MAAM,GAAG,CAAC,IAAI,EAAE,MAAM,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,CAAC;QAC7C,CAAC;QAED,OAAO;YACL,IAAI,EAAE,MAAM,CAAC,GAAG,EAAE,OAAO,EAAE,GAAG,CAAC,MAAM,EAAE,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,OAAO,CAAC;SACtE,CAAC;IACJ,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAI,OAAO,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC;QAE5B,IAAI,KAAK,YAAY,KAAK,EAAE,CAAC;YAC3B,OAAO;gBACL,KAAK,CAAC,IAAI,KAAK,YAAY;oBACzB,CAAC,CAAC,mBAAmB,SAAS,KAAK;oBACnC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC;QACtB,CAAC;QAED,OAAO;YACL,IAAI,EAAE,MAAM,CAAC,GAAG,EAAE,OAAO,EAAE,IAAI,EAAE,EAAE,EAAE,IAAI,EAAE,KAAK,EAAE,OAAO,EAAE,OAAO,CAAC;SACpE,CAAC;IACJ,CAAC;YAAS,CAAC;QACT,YAAY,CAAC,KAAK,CAAC,CAAC;IACtB,CAAC;AACH,CAAC,CAAC;AAEF,wEAAwE;AACxE,MAAM,CAAC,MAAM,SAAS,GAAG,KAAK,EAC5B,GAAW,EACX,OAAO,GAAiB,EAAE,EAC1B,QAAQ,GAAG,IAAI,EACO,EAAE;IACxB,MAAM,YAAY,GAAG,OAAO,CAAC,YAAY,IAAI,EAAE,CAAC;IAEhD,MAAM,IAAI,GAAiB;QACzB,GAAG;QACH,OAAO;QACP,QAAQ;QACR,SAAS,EAAE,OAAO,CAAC,SAAS,IAAI,MAAM;QACtC,OAAO,EAAE,WAAW,CAAC,GAAG,EAAE;QAC1B,KAAK,EAAE,EAAE;KACV,CAAC;IAEF,IAAI,OAAO,GAAG,GAAG,CAAC;IAElB,KAAK,IAAI,GAAG,GAAG,CAAC,EAAE,GAAG,IAAI,YAAY,EAAE,GAAG,IAAI,CAAC,EAAE,CAAC;QAChD,2GAA2G;QAC3G,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;QAE9C,IAAI,MAAM,IAAI,OAAO,EAAE,CAAC;YACtB,OAAO,OAAO,CAAC,IAAI,CAAC;QACtB,CAAC;QAED,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC;IACzB,CAAC;IAED,OAAO,MAAM,CACX,GAAG,EACH,OAAO,EACP,IAAI,EACJ,EAAE,EACF,IAAI,EACJ,IAAI,CAAC,KAAK,EACV,IAAI,CAAC,OAAO,EACZ,aAAa,YAAY,YAAY,CACtC,CAAC;AACJ,CAAC,CAAC;AAEF,0DAA0D;AAC1D,MAAM,CAAC,MAAM,QAAQ,GAAG,CACtB,GAAW,EACX,OAAO,GAAiB,EAAE,EACJ,EAAE,CAAC,SAAS,CAAC,GAAG,EAAE,OAAO,EAAE,KAAK,CAAC,CAAC;AAE1D,MAAM,CAAC,MAAM,cAAc,GAAG,CAAC,CAAc,EAAgB,EAAE;IAC7D,MAAM,IAAI,GAAiB;QACzB,MAAM,EAAE,CAAC,CAAC,MAAM;QAChB,QAAQ,EAAE,CAAC,CAAC,QAAQ;QACpB,aAAa,EAAE,CAAC,CAAC,aAAa;QAC9B,MAAM,EAAE,CAAC,CAAC,MAAM;QAChB,WAAW,EAAE,CAAC,CAAC,WAAW;QAC1B,OAAO,EAAE,CAAC,CAAC,OAAO;KACnB,CAAC;IAEF,IAAI,CAAC,CAAC,KAAK,EAAE,CAAC;QACZ,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC;IACvB,CAAC;IAED,OAAO,IAAI,CAAC;AACd,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,cAAc,GAAG,CAAC,CAAc,EAAW,EAAE;IACxD,MAAM,EAAE,GAAG,CAAC,CAAC,WAAW,EAAE,WAAW,EAAE,IAAI,EAAE,CAAC;IAE9C,IAAI,EAAE,EAAE,CAAC;QACP,OAAO,EAAE,CAAC,QAAQ,CAAC,WAAW,CAAC,IAAI,EAAE,CAAC,QAAQ,CAAC,mBAAmB,CAAC,CAAC;IACtE,CAAC;IAED,IAAI,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;QACZ,OAAO,KAAK,CAAC;IACf,CAAC;IAED,OAAO,mBAAmB,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC,CAAC;AACzD,CAAC,CAAC"}
+{"version":3,"file":"fetch.js","sourceRoot":"","sources":["../../src/core/fetch.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,eAAe,CAAC;AAGxC,MAAM,CAAC,MAAM,kBAAkB,GAAG,8CAA8C,OAAO,wDAAwD,CAAC;AAEhJ,MAAM,YAAY,GAAG;IACnB,cAAc;IACd,kBAAkB;IAClB,cAAc;IACd,MAAM;IACN,kBAAkB;IAClB,eAAe;IACf,eAAe;IACf,UAAU;IACV,gBAAgB;IAChB,MAAM;IACN,yBAAyB;CAC1B,CAAC;AAgCF,MAAM,iBAAiB,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC;AAE7D,MAAM,WAAW,GAAG,CAAC,OAAgB,EAAE,EAAE;IACvC,MAAM,GAAG,GAA2B,EAAE,CAAC;IAEvC,KAAK,MAAM,IAAI,IAAI,YAAY,EAAE,CAAC;QAChC,MAAM,KAAK,GAAG,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QAEhC,IAAI,KAAK,KAAK,IAAI,EAAE,CAAC;YACnB,GAAG,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC;QACpB,CAAC;IACH,CAAC;IAED,OAAO,GAAG,CAAC;AACb,CAAC,CAAC;AAEF,MAAM,MAAM,GAAG,CACb,YAAoB,EACpB,QAAgB,EAChB,MAAqB,EACrB,OAA+B,EAC/B,IAAmB,EACnB,KAAoB,EACpB,OAAe,EACf,KAAc,EACD,EAAE;IACf,MAAM,CAAC,GAAgB;QACrB,YAAY;QACZ,QAAQ;QACR,MAAM;QACN,OAAO;QACP,IAAI;QACJ,aAAa,EAAE,KAAK;QACpB,MAAM,EAAE,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,GAAG,EAAE,GAAG,OAAO,CAAC;QAC/C,WAAW,EAAE,OAAO,CAAC,cAAc,CAAC,IAAI,IAAI;KAC7C,CAAC;IAEF,IAAI,KAAK,EAAE,CAAC;QACV,CAAC,CAAC,KAAK,GAAG,KAAK,CAAC;IAClB,CAAC;IAED,OAAO,CAAC,CAAC;AACX,CAAC,CAAC;AAOF,MAAM,WAAW,GAAG,KAAK,EACvB,GAAa,EACb,QAAgB,EACG,EAAE;IACrB,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC;QACd,OAAO,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,KAAK,EAAE,CAAC;IACtD,CAAC;IAED,MAAM,MAAM,GAAG,GAAG,CAAC,IAAI,CAAC,SAAS,EAAE,CAAC;IACpC,MAAM,MAAM,GAAiB,EAAE,CAAC;IAChC,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,IAAI,SAAS,GAAG,KAAK,CAAC;IAEtB,SAAS,CAAC;QACR,uGAAuG;QACvG,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,MAAM,MAAM,CAAC,IAAI,EAAE,CAAC;QAE5C,IAAI,IAAI,EAAE,CAAC;YACT,MAAM;QACR,CAAC;QAED,IAAI,KAAK,EAAE,CAAC;YACV,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACnB,KAAK,IAAI,KAAK,CAAC,UAAU,CAAC;YAE1B,IAAI,KAAK,IAAI,QAAQ,EAAE,CAAC;gBACtB,SAAS,GAAG,IAAI,CAAC;gBACjB,MAAM;YACR,CAAC;QACH,CAAC;IACH,CAAC;IAED,IAAI,SAAS,EAAE,CAAC;QACd,MAAM,MAAM,CAAC,MAAM,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,CAAC;IAC1C,CAAC;IAED,OAAO,EAAE,KAAK,EAAE,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,SAAS,EAAE,CAAC;AACrD,CAAC,CAAC;AAEF,wFAAwF;AACxF,MAAM,UAAU,GAAG,CAAC,KAAa,EAAE,WAA0B,EAAU,EAAE;IACvE,MAAM,OAAO,GAAG,8BAA8B,CAAC,IAAI,CAAC,WAAW,IAAI,EAAE,CAAC,EAAE,MAAM;QAC5E,EAAE,OAAO,CAAC;IAEZ,IAAI,CAAC;QACH,OAAO,IAAI,WAAW,CAAC,OAAO,IAAI,OAAO,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IAC3D,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,WAAW,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IAChD,CAAC;AACH,CAAC,CAAC;AAeF,MAAM,QAAQ,GAAG,KAAK,EACpB,IAAkB,EAClB,OAAe,EACM,EAAE;IACvB,MAAM,EAAE,GAAG,EAAE,OAAO,EAAE,QAAQ,EAAE,SAAS,EAAE,OAAO,EAAE,KAAK,EAAE,GAAG,IAAI,CAAC;IACnE,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;IACzC,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,UAAU,CAAC,KAAK,EAAE,EAAE,SAAS,CAAC,CAAC;IAE9D,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,OAAO,EAAE;YAC/B,MAAM,EAAE,KAAK;YACb,QAAQ,EAAE,QAAQ;YAClB,MAAM,EAAE,UAAU,CAAC,MAAM;YACzB,OAAO,EAAE;gBACP,YAAY,EAAE,OAAO,CAAC,SAAS,IAAI,kBAAkB;gBACrD,MAAM,EACJ,iEAAiE;gBACnE,iBAAiB,EAAE,YAAY;gBAC/B,GAAG,OAAO,CAAC,OAAO;aACnB;SACF,CAAC,CAAC;QAEH,MAAM,OAAO,GAAG,WAAW,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;QAEzC,IAAI,iBAAiB,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,OAAO,CAAC,UAAU,CAAC,EAAE,CAAC;YAC7D,KAAK,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,OAAO,EAAE,MAAM,EAAE,GAAG,CAAC,MAAM,EAAE,CAAC,CAAC;YACjD,IAAI,OAAe,CAAC;YAEpB,IAAI,CAAC;gBACH,OAAO,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,UAAU,CAAC,EAAE,OAAO,CAAC,CAAC,IAAI,CAAC;YACvD,CAAC;YAAC,MAAM,CAAC;gBACP,OAAO;oBACL,IAAI,EAAE,MAAM,CACV,GAAG,EACH,OAAO,EACP,GAAG,CAAC,MAAM,EACV,OAAO,EACP,IAAI,EACJ,KAAK,EACL,OAAO,EACP,4BAA4B,OAAO,CAAC,UAAU,CAAC,EAAE,CAClD;iBACF,CAAC;YACJ,CAAC;YAED,MAAM,GAAG,CAAC,IAAI,EAAE,MAAM,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,CAAC;YAE3C,IAAI,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,KAAK,OAAO,CAAC,IAAI,OAAO,KAAK,OAAO,EAAE,CAAC;gBAChE,OAAO;oBACL,IAAI,EAAE,MAAM,CACV,GAAG,EACH,OAAO,EACP,GAAG,CAAC,MAAM,EACV,OAAO,EACP,IAAI,EACJ,KAAK,EACL,OAAO,EACP,eAAe,CAChB;iBACF,CAAC;YACJ,CAAC;YAED,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,CAAC;QAC3B,CAAC;QAED,IAAI,CAAC,QAAQ,EAAE,CAAC;YACd,MAAM,GAAG,CAAC,IAAI,EAAE,MAAM,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,CAAC;YAE3C,OAAO;gBACL,IAAI,EAAE,MAAM,CAAC,GAAG,EAAE,OAAO,EAAE,GAAG,CAAC,MAAM,EAAE,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,OAAO,CAAC;aACtE,CAAC;QACJ,CAAC;QAED,MAAM,IAAI,GAAG,MAAM,WAAW,CAAC,GAAG,EAAE,OAAO,CAAC,YAAY,IAAI,SAAS,CAAC,CAAC;QAEvE,MAAM,IAAI,GAAG,MAAM,CACjB,GAAG,EACH,OAAO,EACP,GAAG,CAAC,MAAM,EACV,OAAO;QACP,iFAAiF;QACjF,OAAO,CAAC,SAAS;YACf,CAAC,CAAC,EAAE;YACJ,CAAC,CAAC,UAAU,CAAC,IAAI,CAAC,KAAK,EAAE,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,CAAC,EAC3D,KAAK,EACL,OAAO,CACR,CAAC;QAEF,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;YACnB,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC;QACxB,CAAC;QAED,IAAI,OAAO,CAAC,SAAS,EAAE,CAAC;YACtB,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC;QAC1B,CAAC;QAED,OAAO,EAAE,IAAI,EAAE,CAAC;IAClB,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAI,OAAO,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC;QAE5B,IAAI,KAAK,YAAY,KAAK,EAAE,CAAC;YAC3B,OAAO;gBACL,KAAK,CAAC,IAAI,KAAK,YAAY;oBACzB,CAAC,CAAC,mBAAmB,SAAS,KAAK;oBACnC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC;QACtB,CAAC;QAED,OAAO;YACL,IAAI,EAAE,MAAM,CAAC,GAAG,EAAE,OAAO,EAAE,IAAI,EAAE,EAAE,EAAE,IAAI,EAAE,KAAK,EAAE,OAAO,EAAE,OAAO,CAAC;SACpE,CAAC;IACJ,CAAC;YAAS,CAAC;QACT,YAAY,CAAC,KAAK,CAAC,CAAC;IACtB,CAAC;AACH,CAAC,CAAC;AAEF,wEAAwE;AACxE,MAAM,CAAC,MAAM,SAAS,GAAG,KAAK,EAC5B,GAAW,EACX,OAAO,GAAiB,EAAE,EAC1B,QAAQ,GAAG,IAAI,EACO,EAAE;IACxB,MAAM,YAAY,GAAG,OAAO,CAAC,YAAY,IAAI,EAAE,CAAC;IAEhD,MAAM,IAAI,GAAiB;QACzB,GAAG;QACH,OAAO;QACP,QAAQ;QACR,SAAS,EAAE,OAAO,CAAC,SAAS,IAAI,MAAM;QACtC,OAAO,EAAE,WAAW,CAAC,GAAG,EAAE;QAC1B,KAAK,EAAE,EAAE;KACV,CAAC;IAEF,IAAI,OAAO,GAAG,GAAG,CAAC;IAElB,KAAK,IAAI,GAAG,GAAG,CAAC,EAAE,GAAG,IAAI,YAAY,EAAE,GAAG,IAAI,CAAC,EAAE,CAAC;QAChD,2GAA2G;QAC3G,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;QAE9C,IAAI,MAAM,IAAI,OAAO,EAAE,CAAC;YACtB,OAAO,OAAO,CAAC,IAAI,CAAC;QACtB,CAAC;QAED,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC;IACzB,CAAC;IAED,OAAO,MAAM,CACX,GAAG,EACH,OAAO,EACP,IAAI,EACJ,EAAE,EACF,IAAI,EACJ,IAAI,CAAC,KAAK,EACV,IAAI,CAAC,OAAO,EACZ,aAAa,YAAY,YAAY,CACtC,CAAC;AACJ,CAAC,CAAC;AAEF,0DAA0D;AAC1D,MAAM,CAAC,MAAM,QAAQ,GAAG,CACtB,GAAW,EACX,OAAO,GAAiB,EAAE,EACJ,EAAE,CAAC,SAAS,CAAC,GAAG,EAAE,OAAO,EAAE,KAAK,CAAC,CAAC;AAE1D,MAAM,CAAC,MAAM,cAAc,GAAG,CAAC,CAAc,EAAgB,EAAE;IAC7D,MAAM,IAAI,GAAiB;QACzB,MAAM,EAAE,CAAC,CAAC,MAAM;QAChB,QAAQ,EAAE,CAAC,CAAC,QAAQ;QACpB,aAAa,EAAE,CAAC,CAAC,aAAa;QAC9B,MAAM,EAAE,CAAC,CAAC,MAAM;QAChB,WAAW,EAAE,CAAC,CAAC,WAAW;QAC1B,OAAO,EAAE,CAAC,CAAC,OAAO;KACnB,CAAC;IAEF,IAAI,CAAC,CAAC,KAAK,EAAE,CAAC;QACZ,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC;IACvB,CAAC;IAED,OAAO,IAAI,CAAC;AACd,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,cAAc,GAAG,CAAC,CAAc,EAAW,EAAE;IACxD,MAAM,EAAE,GAAG,CAAC,CAAC,WAAW,EAAE,WAAW,EAAE,IAAI,EAAE,CAAC;IAE9C,IAAI,EAAE,EAAE,CAAC;QACP,OAAO,EAAE,CAAC,QAAQ,CAAC,WAAW,CAAC,IAAI,EAAE,CAAC,QAAQ,CAAC,mBAAmB,CAAC,CAAC;IACtE,CAAC;IAED,IAAI,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;QACZ,OAAO,KAAK,CAAC;IACf,CAAC;IAED,OAAO,mBAAmB,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC,CAAC;AACzD,CAAC,CAAC"}

package/dist/core/html/answer.d.ts

@@ -0,0 +1,4 @@
+import type { AnswerSignals } from "../types.js";
+import type { Document } from "./dom.js";
+/** Reads what the AEO rules need from a parsed document; `bodyText` is its visible body text. */
+export declare const extractAnswerSignals: (doc: Document, bodyText: string) => AnswerSignals;

package/dist/core/html/answer.js

@@ -0,0 +1,198 @@
+import { hasChildren, isTag, isText } from "domhandler";
+import { attr, childElements, HIDDEN_ELEMENTS, queryAll, queryOne, text, visibleText, } from "./dom.js";
+import { collapseWhitespace, wordCount } from "./text.js";
+/** Meta names whose content holds robots directives for one crawler or for all of them. */
+const ROBOTS_META_NAMES = new Set([
+    "robots",
+    "googlebot",
+    "googlebot-news",
+    "bingbot",
+]);
+/** Containers FAQ widgets put a question in when it is not a heading. */
+const QUESTION_CONTAINERS = new Set(["summary", "dt"]);
+/** Elements an answer usually starts with. */
+const ANSWER_BLOCKS = new Set([
+    "p",
+    "li",
+    "dd",
+    "blockquote",
+    "td",
+    "figcaption",
+    "pre",
+]);
+/** Page chrome: lists and tables here are navigation, not content. */
+const CHROME = new Set(["nav", "header", "footer", "aside"]);
+/** Roots JavaScript frameworks render into; empty in the HTML when the page renders client-side. */
+const APP_ROOTS = [
+    "#root",
+    "#app",
+    "#__next",
+    "#__nuxt",
+    "#___gatsby",
+    "#svelte",
+    "app-root",
+    "[data-reactroot]",
+    "[ng-app]",
+];
+/*
+ * Written dates in the visible text. Every quantifier is bounded and every alternative is a fixed
+ * word, so each regular expression stays linear on long page text.
+ */
+const DATE_PATTERNS = [
+    /\b\d{4}-\d{2}-\d{2}\b/u,
+    /\b\d{1,2}[./-]\d{1,2}[./-]\d{4}\b/u,
+    /\b(?:jan(?:uary)?|feb(?:ruary)?|mar(?:ch)?|apr(?:il)?|may|june?|july?|aug(?:ust)?|sep(?:t(?:ember)?)?|oct(?:ober)?|nov(?:ember)?|dec(?:ember)?)\.? \d{1,2}(?:st|nd|rd|th)?,? \d{4}\b/iu,
+    /\b\d{1,2}(?:st|nd|rd|th)?\.? (?:jan(?:uary)?|feb(?:ruary)?|mar(?:ch)?|apr(?:il)?|may|june?|july?|aug(?:ust)?|sep(?:t(?:ember)?)?|oct(?:ober)?|nov(?:ember)?|dec(?:ember)?)\.?,? \d{4}\b/iu,
+];
+const MAX_QUESTIONS = 100;
+/** A question ends with a question mark; a long "heading" is a paragraph styled as one. */
+const isQuestion = (value) => value.length > 0 && value.length <= 300 && /[?？]$/u.test(value);
+const isHeading = (el) => /^h[1-6]$/u.test(el.name);
+const inChrome = (el) => {
+    for (let p = el.parent; p && isTag(p); p = p.parent) {
+        if (CHROME.has(p.name)) {
+            return true;
+        }
+    }
+    return false;
+};
+/**
+ * What a heading, `<summary>` or `<dt>` does to the question being read: "subsection" goes on with
+ * its section, "content" is ordinary text, a question starts a new section and null ends it.
+ */
+const headingEffect = (el, current) => {
+    const heading = isHeading(el);
+    const label = collapseWhitespace(visibleText(el));
+    const question = isQuestion(label);
+    const level = heading ? Number(el.name.slice(1)) : 0;
+    if (!question) {
+        // "Why choose us?" answered by one card per reason, each under its own subheading, is answered.
+        if (heading && current && current.level > 0 && level > current.level) {
+            return "subsection";
+        }
+        // A <summary> or <dt> ends a section only when it asks a question itself.
+        return heading ? null : "content";
+    }
+    return {
+        level,
+        text: label.slice(0, 200),
+        sectionWords: 0,
+        firstBlockWords: null,
+    };
+};
+/** Question headings in document order, with how much text follows each before the next one. */
+const findQuestions = (body) => {
+    const questions = [];
+    let current = null;
+    const walk = (nodes) => {
+        for (const node of nodes) {
+            if (isText(node)) {
+                if (current) {
+                    current.sectionWords += wordCount(node.data);
+                }
+                continue;
+            }
+            if (!isTag(node)) {
+                if (hasChildren(node)) {
+                    walk(node.children);
+                }
+                continue;
+            }
+            if (HIDDEN_ELEMENTS.has(node.name)) {
+                continue;
+            }
+            if (isHeading(node) || QUESTION_CONTAINERS.has(node.name)) {
+                const effect = headingEffect(node, current);
+                if (effect === "subsection") {
+                    continue;
+                }
+                if (effect !== "content") {
+                    current = effect;
+                    if (current && questions.length < MAX_QUESTIONS) {
+                        questions.push(current);
+                    }
+                    continue;
+                }
+            }
+            if (current &&
+                current.firstBlockWords === null &&
+                ANSWER_BLOCKS.has(node.name)) {
+                current.firstBlockWords = wordCount(visibleText(node));
+            }
+            walk(node.children);
+        }
+    };
+    walk(body.children);
+    return questions;
+};
+/** A short selector for a `data-nosnippet` element: its tag plus its id or first class. */
+const nosnippetSelector = (el) => {
+    const id = attr(el, "id")?.trim();
+    const firstClass = attr(el, "class")?.trim().split(/\s+/u)[0];
+    let { name } = el;
+    if (id) {
+        name += `#${id}`;
+    }
+    else if (firstClass) {
+        name += `.${firstClass}`;
+    }
+    return `${name.slice(0, 80)}[data-nosnippet]`;
+};
+const hasNosnippetAncestor = (el) => {
+    for (let p = el.parent; p && isTag(p); p = p.parent) {
+        if (Object.hasOwn(p.attribs, "data-nosnippet")) {
+            return true;
+        }
+    }
+    return false;
+};
+const emptyAppRoot = (body) => {
+    for (const selector of APP_ROOTS) {
+        const el = queryOne(body, selector);
+        if (el && wordCount(visibleText(el)) === 0) {
+            return selector;
+        }
+    }
+    return null;
+};
+const emptySignals = (robotsMeta) => ({
+    robotsMeta,
+    questions: [],
+    lists: 0,
+    tables: 0,
+    nosnippetElements: 0,
+    nosnippetWords: 0,
+    nosnippetSelectors: [],
+    emptyAppRoot: null,
+    noscriptAsksForJavaScript: false,
+    timeElements: 0,
+    mentionsDate: false,
+});
+/** Reads what the AEO rules need from a parsed document; `bodyText` is its visible body text. */
+export const extractAnswerSignals = (doc, bodyText) => {
+    const robotsMeta = queryAll(doc, "meta[name]").flatMap((el) => {
+        const name = (attr(el, "name") ?? "").trim().toLowerCase();
+        const content = collapseWhitespace(attr(el, "content") ?? "").toLowerCase();
+        return ROBOTS_META_NAMES.has(name) && content ? [{ name, content }] : [];
+    });
+    const body = queryOne(doc, "body");
+    if (!body) {
+        return emptySignals(robotsMeta);
+    }
+    const nosnippet = queryAll(body, "[data-nosnippet]").filter((el) => !hasNosnippetAncestor(el));
+    return {
+        robotsMeta,
+        questions: findQuestions(body),
+        lists: queryAll(body, "ul, ol").filter((el) => childElements(el).filter((c) => c.name === "li").length >= 2 &&
+            !inChrome(el)).length,
+        tables: queryAll(body, "table").filter((el) => queryAll(el, "tr").length >= 2 && !inChrome(el)).length,
+        nosnippetElements: nosnippet.length,
+        nosnippetWords: nosnippet.reduce((sum, el) => sum + wordCount(visibleText(el)), 0),
+        nosnippetSelectors: nosnippet.slice(0, 10).map(nosnippetSelector),
+        emptyAppRoot: emptyAppRoot(body),
+        noscriptAsksForJavaScript: queryAll(body, "noscript").some((el) => /javascript/iu.test(text(el))),
+        timeElements: queryAll(body, "time").length,
+        mentionsDate: DATE_PATTERNS.some((re) => re.test(bodyText)),
+    };
+};
+//# sourceMappingURL=answer.js.map

package/dist/core/html/answer.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"answer.js","sourceRoot":"","sources":["../../../src/core/html/answer.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,YAAY,CAAC;AAGxD,OAAO,EACL,IAAI,EACJ,aAAa,EACb,eAAe,EACf,QAAQ,EACR,QAAQ,EACR,IAAI,EACJ,WAAW,GACZ,MAAM,UAAU,CAAC;AAElB,OAAO,EAAE,kBAAkB,EAAE,SAAS,EAAE,MAAM,WAAW,CAAC;AAE1D,2FAA2F;AAC3F,MAAM,iBAAiB,GAAG,IAAI,GAAG,CAAC;IAChC,QAAQ;IACR,WAAW;IACX,gBAAgB;IAChB,SAAS;CACV,CAAC,CAAC;AAEH,yEAAyE;AACzE,MAAM,mBAAmB,GAAG,IAAI,GAAG,CAAC,CAAC,SAAS,EAAE,IAAI,CAAC,CAAC,CAAC;AAEvD,8CAA8C;AAC9C,MAAM,aAAa,GAAG,IAAI,GAAG,CAAC;IAC5B,GAAG;IACH,IAAI;IACJ,IAAI;IACJ,YAAY;IACZ,IAAI;IACJ,YAAY;IACZ,KAAK;CACN,CAAC,CAAC;AAEH,sEAAsE;AACtE,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,CAAC,KAAK,EAAE,QAAQ,EAAE,QAAQ,EAAE,OAAO,CAAC,CAAC,CAAC;AAE7D,oGAAoG;AACpG,MAAM,SAAS,GAAG;IAChB,OAAO;IACP,MAAM;IACN,SAAS;IACT,SAAS;IACT,YAAY;IACZ,SAAS;IACT,UAAU;IACV,kBAAkB;IAClB,UAAU;CACX,CAAC;AAEF;;;GAGG;AACH,MAAM,aAAa,GAAG;IACpB,wBAAwB;IACxB,oCAAoC;IACpC,wLAAwL;IACxL,2LAA2L;CAC5L,CAAC;AAEF,MAAM,aAAa,GAAG,GAAG,CAAC;AAE1B,2FAA2F;AAC3F,MAAM,UAAU,GAAG,CAAC,KAAa,EAAW,EAAE,CAC5C,KAAK,CAAC,MAAM,GAAG,CAAC,IAAI,KAAK,CAAC,MAAM,IAAI,GAAG,IAAI,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;AAElE,MAAM,SAAS,GAAG,CAAC,EAAW,EAAW,EAAE,CAAC,WAAW,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,CAAC;AAEtE,MAAM,QAAQ,GAAG,CAAC,EAAW,EAAW,EAAE;IACxC,KAAK,IAAI,CAAC,GAAG,EAAE,CAAC,MAAM,EAAE,CAAC,IAAI,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,CAAC;QACpD,IAAI,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC;YACvB,OAAO,IAAI,CAAC;QACd,CAAC;IACH,CAAC;IAED,OAAO,KAAK,CAAC;AACf,CAAC,CAAC;AAEF;;;GAGG;AACH,MAAM,aAAa,GAAG,CACpB,EAAW,EACX,OAA+B,EACoB,EAAE;IACrD,MAAM,OAAO,GAAG,SAAS,CAAC,EAAE,CAAC,CAAC;IAC9B,MAAM,KAAK,GAAG,kBAAkB,CAAC,WAAW,CAAC,EAAE,CAAC,CAAC,CAAC;IAClD,MAAM,QAAQ,GAAG,UAAU,CAAC,KAAK,CAAC,CAAC;IACnC,MAAM,KAAK,GAAG,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAErD,IAAI,CAAC,QAAQ,EAAE,CAAC;QACd,gGAAgG;QAChG,IAAI,OAAO,IAAI,OAAO,IAAI,OAAO,CAAC,KAAK,GAAG,CAAC,IAAI,KAAK,GAAG,OAAO,CAAC,KAAK,EAAE,CAAC;YACrE,OAAO,YAAY,CAAC;QACtB,CAAC;QAED,0EAA0E;QAC1E,OAAO,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,SAAS,CAAC;IACpC,CAAC;IAED,OAAO;QACL,KAAK;QACL,IAAI,EAAE,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC;QACzB,YAAY,EAAE,CAAC;QACf,eAAe,EAAE,IAAI;KACtB,CAAC;AACJ,CAAC,CAAC;AAEF,gGAAgG;AAChG,MAAM,aAAa,GAAG,CAAC,IAAa,EAAqB,EAAE;IACzD,MAAM,SAAS,GAAsB,EAAE,CAAC;IACxC,IAAI,OAAO,GAA2B,IAAI,CAAC;IAE3C,MAAM,IAAI,GAAG,CAAC,KAAgB,EAAQ,EAAE;QACtC,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,IAAI,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC;gBACjB,IAAI,OAAO,EAAE,CAAC;oBACZ,OAAO,CAAC,YAAY,IAAI,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;gBAC/C,CAAC;gBAED,SAAS;YACX,CAAC;YAED,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC;gBACjB,IAAI,WAAW,CAAC,IAAI,CAAC,EAAE,CAAC;oBACtB,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;gBACtB,CAAC;gBAED,SAAS;YACX,CAAC;YAED,IAAI,eAAe,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;gBACnC,SAAS;YACX,CAAC;YAED,IAAI,SAAS,CAAC,IAAI,CAAC,IAAI,mBAAmB,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;gBAC1D,MAAM,MAAM,GAAG,aAAa,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;gBAE5C,IAAI,MAAM,KAAK,YAAY,EAAE,CAAC;oBAC5B,SAAS;gBACX,CAAC;gBAED,IAAI,MAAM,KAAK,SAAS,EAAE,CAAC;oBACzB,OAAO,GAAG,MAAM,CAAC;oBAEjB,IAAI,OAAO,IAAI,SAAS,CAAC,MAAM,GAAG,aAAa,EAAE,CAAC;wBAChD,SAAS,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;oBAC1B,CAAC;oBAED,SAAS;gBACX,CAAC;YACH,CAAC;YAED,IACE,OAAO;gBACP,OAAO,CAAC,eAAe,KAAK,IAAI;gBAChC,aAAa,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,EAC5B,CAAC;gBACD,OAAO,CAAC,eAAe,GAAG,SAAS,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC,CAAC;YACzD,CAAC;YAED,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QACtB,CAAC;IACH,CAAC,CAAC;IAEF,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;IAEpB,OAAO,SAAS,CAAC;AACnB,CAAC,CAAC;AAEF,2FAA2F;AAC3F,MAAM,iBAAiB,GAAG,CAAC,EAAW,EAAU,EAAE;IAChD,MAAM,EAAE,GAAG,IAAI,CAAC,EAAE,EAAE,IAAI,CAAC,EAAE,IAAI,EAAE,CAAC;IAClC,MAAM,UAAU,GAAG,IAAI,CAAC,EAAE,EAAE,OAAO,CAAC,EAAE,IAAI,EAAE,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;IAE9D,IAAI,EAAE,IAAI,EAAE,GAAG,EAAE,CAAC;IAElB,IAAI,EAAE,EAAE,CAAC;QACP,IAAI,IAAI,IAAI,EAAE,EAAE,CAAC;IACnB,CAAC;SAAM,IAAI,UAAU,EAAE,CAAC;QACtB,IAAI,IAAI,IAAI,UAAU,EAAE,CAAC;IAC3B,CAAC;IAED,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,kBAAkB,CAAC;AAChD,CAAC,CAAC;AAEF,MAAM,oBAAoB,GAAG,CAAC,EAAW,EAAW,EAAE;IACpD,KAAK,IAAI,CAAC,GAAG,EAAE,CAAC,MAAM,EAAE,CAAC,IAAI,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,CAAC;QACpD,IAAI,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,OAAO,EAAE,gBAAgB,CAAC,EAAE,CAAC;YAC/C,OAAO,IAAI,CAAC;QACd,CAAC;IACH,CAAC;IAED,OAAO,KAAK,CAAC;AACf,CAAC,CAAC;AAEF,MAAM,YAAY,GAAG,CAAC,IAAa,EAAiB,EAAE;IACpD,KAAK,MAAM,QAAQ,IAAI,SAAS,EAAE,CAAC;QACjC,MAAM,EAAE,GAAG,QAAQ,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;QAEpC,IAAI,EAAE,IAAI,SAAS,CAAC,WAAW,CAAC,EAAE,CAAC,CAAC,KAAK,CAAC,EAAE,CAAC;YAC3C,OAAO,QAAQ,CAAC;QAClB,CAAC;IACH,CAAC;IAED,OAAO,IAAI,CAAC;AACd,CAAC,CAAC;AAEF,MAAM,YAAY,GAAG,CAAC,UAAuC,EAAE,EAAE,CAAC,CAAC;IACjE,UAAU;IACV,SAAS,EAAE,EAAE;IACb,KAAK,EAAE,CAAC;IACR,MAAM,EAAE,CAAC;IACT,iBAAiB,EAAE,CAAC;IACpB,cAAc,EAAE,CAAC;IACjB,kBAAkB,EAAE,EAAE;IACtB,YAAY,EAAE,IAAI;IAClB,yBAAyB,EAAE,KAAK;IAChC,YAAY,EAAE,CAAC;IACf,YAAY,EAAE,KAAK;CACpB,CAAC,CAAC;AAEH,iGAAiG;AACjG,MAAM,CAAC,MAAM,oBAAoB,GAAG,CAClC,GAAa,EACb,QAAgB,EACD,EAAE;IACjB,MAAM,UAAU,GAAG,QAAQ,CAAC,GAAG,EAAE,YAAY,CAAC,CAAC,OAAO,CAAC,CAAC,EAAE,EAAE,EAAE;QAC5D,MAAM,IAAI,GAAG,CAAC,IAAI,CAAC,EAAE,EAAE,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;QAC3D,MAAM,OAAO,GAAG,kBAAkB,CAAC,IAAI,CAAC,EAAE,EAAE,SAAS,CAAC,IAAI,EAAE,CAAC,CAAC,WAAW,EAAE,CAAC;QAE5E,OAAO,iBAAiB,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,OAAO,CAAC,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;IAC3E,CAAC,CAAC,CAAC;IAEH,MAAM,IAAI,GAAG,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC;IAEnC,IAAI,CAAC,IAAI,EAAE,CAAC;QACV,OAAO,YAAY,CAAC,UAAU,CAAC,CAAC;IAClC,CAAC;IAED,MAAM,SAAS,GAAG,QAAQ,CAAC,IAAI,EAAE,kBAAkB,CAAC,CAAC,MAAM,CACzD,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,oBAAoB,CAAC,EAAE,CAAC,CAClC,CAAC;IAEF,OAAO;QACL,UAAU;QACV,SAAS,EAAE,aAAa,CAAC,IAAI,CAAC;QAC9B,KAAK,EAAE,QAAQ,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC,MAAM,CACpC,CAAC,EAAE,EAAE,EAAE,CACL,aAAa,CAAC,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,IAAI,CAAC,CAAC,MAAM,IAAI,CAAC;YAC5D,CAAC,QAAQ,CAAC,EAAE,CAAC,CAChB,CAAC,MAAM;QACR,MAAM,EAAE,QAAQ,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC,MAAM,CACpC,CAAC,EAAE,EAAE,EAAE,CAAC,QAAQ,CAAC,EAAE,EAAE,IAAI,CAAC,CAAC,MAAM,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC,CACxD,CAAC,MAAM;QACR,iBAAiB,EAAE,SAAS,CAAC,MAAM;QACnC,cAAc,EAAE,SAAS,CAAC,MAAM,CAC9B,CAAC,GAAG,EAAE,EAAE,EAAE,EAAE,CAAC,GAAG,GAAG,SAAS,CAAC,WAAW,CAAC,EAAE,CAAC,CAAC,EAC7C,CAAC,CACF;QACD,kBAAkB,EAAE,SAAS,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,iBAAiB,CAAC;QACjE,YAAY,EAAE,YAAY,CAAC,IAAI,CAAC;QAChC,yBAAyB,EAAE,QAAQ,CAAC,IAAI,EAAE,UAAU,CAAC,CAAC,IAAI,CAAC,CAAC,EAAE,EAAE,EAAE,CAChE,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAC9B;QACD,YAAY,EAAE,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC,MAAM;QAC3C,YAAY,EAAE,aAAa,CAAC,IAAI,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;KAC5D,CAAC;AACJ,CAAC,CAAC"}

package/dist/core/html/dom.d.ts

@@ -1,5 +1,11 @@
 import type { AnyNode, Document, Element } from "domhandler";
 export type { AnyNode, Document, Element } from "domhandler";
+/** Elements whose content is never visible page text. */
+export declare const HIDDEN_ELEMENTS: Set<string>;
+/** The text a reader sees under `root`: hidden elements left out, whitespace collapsed. */
+export declare const visibleText: (root: AnyNode) => string;
+/** For each <img> under `root`, the number of visible words that come before it in the document. */
+export declare const wordsBeforeImages: (root: AnyNode) => Map<Element, number>;
 /** Parse a full HTML document with the WHATWG algorithm (parse5), scripting enabled like a browser. */
 export declare const parseDocument: (html: string) => Document;
 /** Descendants of `root` matching a CSS selector, in document order. */

package/dist/core/html/dom.js

@@ -1,8 +1,99 @@
 import { selectAll, selectOne } from "css-select";
-import { isTag } from "domhandler";
+import { hasChildren, isTag, isText } from "domhandler";
 import { textContent } from "domutils";
 import { parse } from "parse5";
 import { adapter } from "parse5-htmlparser2-tree-adapter";
+import { collapseWhitespace, wordCount } from "./text.js";
+/** Elements whose content is never visible page text. */
+export const HIDDEN_ELEMENTS = new Set([
+    "script",
+    "style",
+    "noscript",
+    "template",
+    "svg",
+    "iframe",
+    "head",
+]);
+/** Elements that break words apart; padded with spaces so adjacent words don't fuse. */
+const BLOCK_ELEMENTS = new Set([
+    "p",
+    "div",
+    "li",
+    "h1",
+    "h2",
+    "h3",
+    "h4",
+    "h5",
+    "h6",
+    "br",
+    "tr",
+    "td",
+    "th",
+    "section",
+    "article",
+    "header",
+    "footer",
+    "nav",
+    "blockquote",
+    "pre",
+    "dd",
+    "dt",
+    "figcaption",
+]);
+/** The text a reader sees under `root`: hidden elements left out, whitespace collapsed. */
+export const visibleText = (root) => {
+    const parts = [];
+    const walk = (nodes) => {
+        for (const node of nodes) {
+            if (isText(node)) {
+                parts.push(node.data);
+            }
+            else if (isTag(node)) {
+                if (HIDDEN_ELEMENTS.has(node.name)) {
+                    continue;
+                }
+                const block = BLOCK_ELEMENTS.has(node.name);
+                if (block) {
+                    parts.push(" ");
+                }
+                walk(node.children);
+                if (block) {
+                    parts.push(" ");
+                }
+            }
+            else if (hasChildren(node)) {
+                walk(node.children);
+            }
+        }
+    };
+    walk(hasChildren(root) ? root.children : [root]);
+    return collapseWhitespace(parts.join(""));
+};
+/** For each <img> under `root`, the number of visible words that come before it in the document. */
+export const wordsBeforeImages = (root) => {
+    const out = new Map();
+    let words = 0;
+    const walk = (nodes) => {
+        for (const node of nodes) {
+            if (isText(node)) {
+                words += wordCount(node.data);
+            }
+            else if (isTag(node)) {
+                if (node.name === "img") {
+                    out.set(node, words);
+                }
+                else if (!HIDDEN_ELEMENTS.has(node.name)) {
+                    walk(node.children);
+                }
+            }
+            else if (hasChildren(node)) {
+                walk(node.children);
+            }
+        }
+    };
+    walk(hasChildren(root) ? root.children : [root]);
+    return out;
+};
 /** Parse a full HTML document with the WHATWG algorithm (parse5), scripting enabled like a browser. */
 export const parseDocument = (html) => parse(html, { scriptingEnabled: true, treeAdapter: adapter });
 /** Descendants of `root` matching a CSS selector, in document order. */

package/dist/core/html/dom.js.map

@@ -1 +1 @@
-{"version":3,"file":"dom.js","sourceRoot":"","sources":["../../../src/core/html/dom.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AAClD,OAAO,EAAE,KAAK,EAAE,MAAM,YAAY,CAAC;AAEnC,OAAO,EAAE,WAAW,EAAE,MAAM,UAAU,CAAC;AACvC,OAAO,EAAE,KAAK,EAAE,MAAM,QAAQ,CAAC;AAC/B,OAAO,EAAE,OAAO,EAAE,MAAM,iCAAiC,CAAC;AAI1D,uGAAuG;AACvG,MAAM,CAAC,MAAM,aAAa,GAAG,CAAC,IAAY,EAAY,EAAE,CACtD,KAAK,CAAC,IAAI,EAAE,EAAE,gBAAgB,EAAE,IAAI,EAAE,WAAW,EAAE,OAAO,EAAE,CAAC,CAAC;AAEhE,wEAAwE;AACxE,MAAM,CAAC,MAAM,QAAQ,GAAG,CAAC,IAAa,EAAE,QAAgB,EAAa,EAAE,CACrE,SAAS,CAAmB,QAAQ,EAAE,IAAI,CAAC,CAAC;AAE9C,0DAA0D;AAC1D,MAAM,CAAC,MAAM,QAAQ,GAAG,CAAC,IAAa,EAAE,QAAgB,EAAkB,EAAE,CAC1E,SAAS,CAAmB,QAAQ,EAAE,IAAI,CAAC,CAAC;AAE9C,MAAM,CAAC,MAAM,IAAI,GAAG,CAAC,EAAW,EAAE,IAAY,EAAsB,EAAE,CACpE,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;AAEjE,MAAM,CAAC,MAAM,OAAO,GAAG,CAAC,EAAW,EAAE,IAAY,EAAW,EAAE,CAC5D,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC;AAElC,kFAAkF;AAClF,MAAM,CAAC,MAAM,IAAI,GAAG,CAAC,IAAyB,EAAU,EAAE,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC;AAE7E,MAAM,CAAC,MAAM,aAAa,GAAG,CAAC,EAAW,EAAa,EAAE,CACtD,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC"}
+{"version":3,"file":"dom.js","sourceRoot":"","sources":["../../../src/core/html/dom.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AAClD,OAAO,EAAE,WAAW,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,YAAY,CAAC;AAExD,OAAO,EAAE,WAAW,EAAE,MAAM,UAAU,CAAC;AACvC,OAAO,EAAE,KAAK,EAAE,MAAM,QAAQ,CAAC;AAC/B,OAAO,EAAE,OAAO,EAAE,MAAM,iCAAiC,CAAC;AAE1D,OAAO,EAAE,kBAAkB,EAAE,SAAS,EAAE,MAAM,WAAW,CAAC;AAI1D,yDAAyD;AACzD,MAAM,CAAC,MAAM,eAAe,GAAG,IAAI,GAAG,CAAC;IACrC,QAAQ;IACR,OAAO;IACP,UAAU;IACV,UAAU;IACV,KAAK;IACL,QAAQ;IACR,MAAM;CACP,CAAC,CAAC;AAEH,wFAAwF;AACxF,MAAM,cAAc,GAAG,IAAI,GAAG,CAAC;IAC7B,GAAG;IACH,KAAK;IACL,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,SAAS;IACT,SAAS;IACT,QAAQ;IACR,QAAQ;IACR,KAAK;IACL,YAAY;IACZ,KAAK;IACL,IAAI;IACJ,IAAI;IACJ,YAAY;CACb,CAAC,CAAC;AAEH,2FAA2F;AAC3F,MAAM,CAAC,MAAM,WAAW,GAAG,CAAC,IAAa,EAAU,EAAE;IACnD,MAAM,KAAK,GAAa,EAAE,CAAC;IAE3B,MAAM,IAAI,GAAG,CAAC,KAAgB,EAAQ,EAAE;QACtC,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,IAAI,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC;gBACjB,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACxB,CAAC;iBAAM,IAAI,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC;gBACvB,IAAI,eAAe,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;oBACnC,SAAS;gBACX,CAAC;gBAED,MAAM,KAAK,GAAG,cAAc,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;gBAE5C,IAAI,KAAK,EAAE,CAAC;oBACV,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;gBAClB,CAAC;gBAED,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;gBAEpB,IAAI,KAAK,EAAE,CAAC;oBACV,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;gBAClB,CAAC;YACH,CAAC;iBAAM,IAAI,WAAW,CAAC,IAAI,CAAC,EAAE,CAAC;gBAC7B,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;YACtB,CAAC;QACH,CAAC;IACH,CAAC,CAAC;IAEF,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;IAEjD,OAAO,kBAAkB,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC;AAC5C,CAAC,CAAC;AAEF,oGAAoG;AACpG,MAAM,CAAC,MAAM,iBAAiB,GAAG,CAAC,IAAa,EAAwB,EAAE;IACvE,MAAM,GAAG,GAAG,IAAI,GAAG,EAAmB,CAAC;IACvC,IAAI,KAAK,GAAG,CAAC,CAAC;IAEd,MAAM,IAAI,GAAG,CAAC,KAAgB,EAAQ,EAAE;QACtC,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,IAAI,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC;gBACjB,KAAK,IAAI,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAChC,CAAC;iBAAM,IAAI,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC;gBACvB,IAAI,IAAI,CAAC,IAAI,KAAK,KAAK,EAAE,CAAC;oBACxB,GAAG,CAAC,GAAG,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;gBACvB,CAAC;qBAAM,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;oBAC3C,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;gBACtB,CAAC;YACH,CAAC;iBAAM,IAAI,WAAW,CAAC,IAAI,CAAC,EAAE,CAAC;gBAC7B,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;YACtB,CAAC;QACH,CAAC;IACH,CAAC,CAAC;IAEF,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;IAEjD,OAAO,GAAG,CAAC;AACb,CAAC,CAAC;AAEF,uGAAuG;AACvG,MAAM,CAAC,MAAM,aAAa,GAAG,CAAC,IAAY,EAAY,EAAE,CACtD,KAAK,CAAC,IAAI,EAAE,EAAE,gBAAgB,EAAE,IAAI,EAAE,WAAW,EAAE,OAAO,EAAE,CAAC,CAAC;AAEhE,wEAAwE;AACxE,MAAM,CAAC,MAAM,QAAQ,GAAG,CAAC,IAAa,EAAE,QAAgB,EAAa,EAAE,CACrE,SAAS,CAAmB,QAAQ,EAAE,IAAI,CAAC,CAAC;AAE9C,0DAA0D;AAC1D,MAAM,CAAC,MAAM,QAAQ,GAAG,CAAC,IAAa,EAAE,QAAgB,EAAkB,EAAE,CAC1E,SAAS,CAAmB,QAAQ,EAAE,IAAI,CAAC,CAAC;AAE9C,MAAM,CAAC,MAAM,IAAI,GAAG,CAAC,EAAW,EAAE,IAAY,EAAsB,EAAE,CACpE,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;AAEjE,MAAM,CAAC,MAAM,OAAO,GAAG,CAAC,EAAW,EAAE,IAAY,EAAW,EAAE,CAC5D,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC;AAElC,kFAAkF;AAClF,MAAM,CAAC,MAAM,IAAI,GAAG,CAAC,IAAyB,EAAU,EAAE,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC;AAE7E,MAAM,CAAC,MAAM,aAAa,GAAG,CAAC,EAAW,EAAa,EAAE,CACtD,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC"}