@rivi-software/agentic-seo-check 0.0.0-stage → 0.1.0

package/dist/core/crawl.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"crawl.js","sourceRoot":"","sources":["../../src/core/crawl.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAChD,OAAO,EAAE,kBAAkB,EAAE,SAAS,EAAE,cAAc,EAAE,QAAQ,EAAE,cAAc,EAAqB,MAAM,YAAY,CAAC;AACxH,OAAO,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAC5C,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAC5C,OAAO,EAAE,WAAW,EAAE,MAAM,sBAAsB,CAAC;AAGnD,OAAO,EAAE,MAAM,EAAE,UAAU,EAAE,aAAa,EAAE,YAAY,EAAE,QAAQ,EAAE,MAAM,UAAU,CAAC;AACrF,OAAO,EAAE,aAAa,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAC;AA8CxD,MAAM,CAAC,KAAK,UAAU,SAAS,CAAC,UAAkB,EAAE,OAAqB;IACvE,MAAM,OAAO,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC;IAClC,MAAM,QAAQ,GAAG,OAAO,CAAC,UAAU,IAAI,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;IACzD,MAAM,QAAQ,GAAG,YAAY,CAAC,UAAU,CAAC,IAAI,UAAU,CAAC;IACxD,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC;IACxC,MAAM,SAAS,GAAG,MAAM,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC;IACzC,MAAM,SAAS,GAAG,OAAO,CAAC,KAAK,CAAC,SAAS,IAAI,kBAAkB,CAAC;IAEhE,QAAQ,CAAC,EAAE,KAAK,EAAE,QAAQ,EAAE,OAAO,EAAE,YAAY,MAAM,aAAa,EAAE,CAAC,CAAC;IACxE,MAAM,MAAM,GAAG,MAAM,aAAa,CAAC,MAAM,EAAE,SAAS,EAAE,OAAO,CAAC,KAAK,CAAC,CAAC;IACrE,MAAM,MAAM,GAAG,OAAO,CAAC,aAAa,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC;IAElE,QAAQ,CAAC,EAAE,KAAK,EAAE,SAAS,EAAE,OAAO,EAAE,mBAAmB,EAAE,CAAC,CAAC;IAC7D,MAAM,iBAAiB,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,MAAM,CAAC,IAAI,CAAC,QAAQ,EAAE,IAAI,GAAG,CAAC,cAAc,EAAE,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IACxG,MAAM,QAAQ,GAAG,MAAM,YAAY,CAAC,iBAAiB,EAAE,OAAO,CAAC,KAAK,CAAC,CAAC;IAEtE,MAAM,OAAO,GAAG,IAAI,GAAG,CAAS,CAAC,QAAQ,CAAC,CAAC,CAAC;IAC5C,MAAM,UAAU,GAAG,IAAI,GAAG,CAAS,CAAC,QAAQ,CAAC,CAAC,CAAC;IAC/C,MAAM,UAAU,GAAG,IAAI,GAAG,EAAqB,CAAC;IAChD,MAAM,WAAW,GAAG,IAAI,GAAG,EAAuB,CAAC;IACnD,MAAM,eAAe,GAAG,IAAI,GAAG,EAAuB,CAAC;IACvD,MAAM,OAAO,GAAiB,EAAE,CAAC;IACjC,MAAM,OAAO,GAAa,EAAE,CAAC;IAC7B,IAAI,UAAU,GAAG,CAAC,CAAC;IAEnB,MAAM,OAAO,GAAG,CAAC,GAAW,EAAW,EAAE;QACvC,IAAI,OAAO,CAAC,eAAe,CAAC,MAAM,IAAI,CAAC,OAAO,CAAC,eAAe,CAAC,IAAI,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YAAE,OAAO,KAAK,CAAC;QACxG,IAAI,OAAO,CAAC,eAAe,CAAC,IAAI,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YAAE,OAAO,KAAK,CAAC;QACrE,OAAO,IAAI,CAAC;IACd,CAAC,CAAC;IAEF,MAAM,KAAK,GAAgB,CAAC,EAAE,GAAG,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC,CAAC;IAEzD,MAAM,WAAW,GAAG,KAAK,EAAE,IAAe,EAAiB,EAAE;QAC3D,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;YACtB,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACvB,OAAO;QACT,CAAC;QACD,MAAM,GAAG,GAAG,MAAM,SAAS,CAAC,IAAI,CAAC,GAAG,EAAE,OAAO,CAAC,KAAK,CAAC,CAAC;QACrD,MAAM,YAAY,GAAG,IAAI,CAAC,GAAG,CAAC;QAC9B,MAAM,QAAQ,GAAG,YAAY,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,GAAG,CAAC,QAAQ,CAAC;QAC5D,UAAU,CAAC,GAAG,CAAC,YAAY,EAAE,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,EAAE,MAAM,EAAE,GAAG,CAAC,MAAM,EAAE,QAAQ,EAAE,GAAG,CAAC,QAAQ,EAAE,aAAa,EAAE,GAAG,CAAC,aAAa,EAAE,KAAK,EAAE,GAAG,CAAC,KAAK,EAAE,CAAC,CAAC;QAChJ,IAAI,QAAQ,KAAK,YAAY,EAAE,CAAC;YAC9B,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,QAAQ,CAAC;gBAAE,UAAU,CAAC,GAAG,CAAC,QAAQ,EAAE,EAAE,GAAG,EAAE,GAAG,CAAC,QAAQ,EAAE,MAAM,EAAE,GAAG,CAAC,MAAM,EAAE,QAAQ,EAAE,GAAG,CAAC,QAAQ,EAAE,aAAa,EAAE,EAAE,EAAE,KAAK,EAAE,GAAG,CAAC,KAAK,EAAE,CAAC,CAAC;YAChK,IAAI,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,KAAK,QAAQ,CAAC;gBAAE,OAAO;YAC7E,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;YACtB,IAAI,CAAC,UAAU,CAAC,QAAQ,EAAE,SAAS,EAAE,OAAO,CAAC,iBAAiB,CAAC;gBAAE,OAAO,CAAC,sBAAsB;QACjG,CAAC;QACD,QAAQ,CAAC,EAAE,KAAK,EAAE,OAAO,EAAE,OAAO,EAAE,GAAG,GAAG,CAAC,MAAM,IAAI,KAAK,IAAI,GAAG,CAAC,QAAQ,EAAE,EAAE,OAAO,EAAE,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,UAAU,EAAE,UAAU,CAAC,IAAI,EAAE,GAAG,EAAE,GAAG,CAAC,QAAQ,EAAE,CAAC,CAAC;QAC7J,IAAI,GAAG,CAAC,KAAK,IAAI,OAAO,GAAG,CAAC,IAAI,KAAK,QAAQ,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,EAAE,CAAC;YACtE,IAAI,GAAG,CAAC,MAAM,KAAK,IAAI,IAAI,GAAG,CAAC,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,IAAI,OAAO,GAAG,CAAC,IAAI,KAAK,QAAQ,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC;gBAAE,OAAO,CAAC,iBAAiB;YAC5I,8FAA8F;YAC9F,IAAI,IAAI,CAAC,KAAK,KAAK,CAAC,IAAI,GAAG,CAAC,MAAM,KAAK,IAAI,EAAE,CAAC;gBAC5C,MAAM,IAAI,GAAG,WAAW,CAAC,GAAG,CAAC,IAAI,IAAI,EAAE,EAAE,EAAE,GAAG,EAAE,GAAG,CAAC,QAAQ,EAAE,QAAQ,EAAE,SAAS,EAAE,iBAAiB,EAAE,OAAO,CAAC,iBAAiB,EAAE,CAAC,CAAC;gBACnI,OAAO,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,QAAQ,EAAE,YAAY,EAAE,IAAI,CAAC,GAAG,EAAE,IAAI,EAAE,QAAQ,EAAE,cAAc,CAAC,GAAG,CAAC,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE,CAAC,CAAC;YAClH,CAAC;YACD,OAAO;QACT,CAAC;QACD,MAAM,IAAI,GAAG,WAAW,CAAC,GAAG,CAAC,IAAI,EAAE,EAAE,GAAG,EAAE,GAAG,CAAC,QAAQ,EAAE,QAAQ,EAAE,SAAS,EAAE,iBAAiB,EAAE,OAAO,CAAC,iBAAiB,EAAE,CAAC,CAAC;QAC7H,OAAO,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,QAAQ,EAAE,YAAY,EAAE,IAAI,CAAC,GAAG,EAAE,IAAI,EAAE,QAAQ,EAAE,cAAc,CAAC,GAAG,CAAC,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE,CAAC,CAAC;QAEhH,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;YAC9B,IAAI,CAAC,IAAI,CAAC,GAAG;gBAAE,SAAS;YACxB,MAAM,MAAM,GAAG,YAAY,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACtC,IAAI,CAAC,MAAM;gBAAE,SAAS;YACtB,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,CAAC;gBACnB,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,MAAM,CAAC;oBAAE,eAAe,CAAC,GAAG,CAAC,MAAM,EAAE,IAAI,GAAG,EAAE,CAAC,CAAC;gBACzE,eAAe,CAAC,GAAG,CAAC,MAAM,CAAE,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;gBAC3C,SAAS;YACX,CAAC;YACD,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,MAAM,CAAC;gBAAE,WAAW,CAAC,GAAG,CAAC,MAAM,EAAE,IAAI,GAAG,EAAE,CAAC,CAAC;YACjE,WAAW,CAAC,GAAG,CAAC,MAAM,CAAE,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;YACvC,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC;gBAAE,SAAS;YACzD,IAAI,UAAU,CAAC,GAAG,CAAC,MAAM,CAAC;gBAAE,SAAS;YACrC,UAAU,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;YACvB,IAAI,IAAI,CAAC,KAAK,GAAG,CAAC,GAAG,OAAO,CAAC,QAAQ;gBAAE,SAAS;YAChD,IAAI,OAAO,CAAC,IAAI,IAAI,OAAO,CAAC,QAAQ,EAAE,CAAC;gBACrC,UAAU,EAAE,CAAC;gBACb,SAAS;YACX,CAAC;YACD,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;YACpB,KAAK,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,GAAG,CAAC,EAAE,CAAC,CAAC;QACrD,CAAC;IACH,CAAC,CAAC;IAEF,MAAM,QAAQ,CAAC,KAAK,EAAE,OAAO,CAAC,WAAW,EAAE,OAAO,CAAC,OAAO,EAAE,WAAW,CAAC,CAAC;IAEzE,iGAAiG;IACjG,QAAQ,CAAC,EAAE,KAAK,EAAE,OAAO,EAAE,OAAO,EAAE,gCAAgC,EAAE,CAAC,CAAC;IACxE,MAAM,OAAO,GAAG,CAAC,GAAG,WAAW,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,OAAO,CAAC,aAAa,CAAC,CAAC;IACvH,MAAM,YAAY,GAAG,QAAQ,CAAC,IAAI;SAC/B,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,YAAY,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;SAChC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,UAAU,CAAC,CAAC,EAAE,SAAS,EAAE,OAAO,CAAC,iBAAiB,CAAC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,MAAM,CAAC,CAAC,CAAC,CAAC;SAC7H,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,OAAO,CAAC,aAAa,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC;IACjE,KAAK,MAAM,CAAC,IAAI,CAAC,GAAG,WAAW,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;QAAE,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC;YAAE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAC7G,MAAM,aAAa,CAAC,CAAC,GAAG,OAAO,EAAE,GAAG,YAAY,CAAC,EAAE,OAAO,CAAC,WAAW,EAAE,KAAK,EAAE,CAAC,EAAE,EAAE;QAClF,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,CAAC,EAAE,OAAO,CAAC,KAAK,CAAC,CAAC;QAC7C,UAAU,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,GAAG,EAAE,CAAC,EAAE,MAAM,EAAE,GAAG,CAAC,MAAM,EAAE,QAAQ,EAAE,GAAG,CAAC,QAAQ,EAAE,aAAa,EAAE,GAAG,CAAC,aAAa,EAAE,KAAK,EAAE,GAAG,CAAC,KAAK,EAAE,CAAC,CAAC;QAC9H,IAAI,OAAO,CAAC,OAAO;YAAE,MAAM,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;IACpD,CAAC,CAAC,CAAC;IAEH,IAAI,cAAkD,CAAC;IACvD,IAAI,OAAO,CAAC,kBAAkB,EAAE,CAAC;QAC/B,QAAQ,CAAC,EAAE,KAAK,EAAE,OAAO,EAAE,OAAO,EAAE,YAAY,IAAI,CAAC,GAAG,CAAC,eAAe,CAAC,IAAI,EAAE,OAAO,CAAC,aAAa,CAAC,iBAAiB,EAAE,CAAC,CAAC;QAC1H,cAAc,GAAG,IAAI,GAAG,EAAE,CAAC;QAC3B,MAAM,OAAO,GAAG,CAAC,GAAG,eAAe,CAAC,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,OAAO,CAAC,aAAa,CAAC,CAAC;QAC5E,MAAM,aAAa,CAAC,OAAO,EAAE,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,WAAW,EAAE,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,EAAE;YACzE,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,CAAC,EAAE,EAAE,GAAG,OAAO,CAAC,KAAK,EAAE,SAAS,EAAE,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,KAAK,CAAC,SAAS,IAAI,MAAM,EAAE,MAAM,CAAC,EAAE,CAAC,CAAC;YACpH,cAAe,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,GAAG,EAAE,CAAC,EAAE,MAAM,EAAE,GAAG,CAAC,MAAM,EAAE,QAAQ,EAAE,GAAG,CAAC,QAAQ,EAAE,aAAa,EAAE,GAAG,CAAC,aAAa,EAAE,KAAK,EAAE,GAAG,CAAC,KAAK,EAAE,CAAC,CAAC;QACrI,CAAC,CAAC,CAAC;IACL,CAAC;IAED,QAAQ,CAAC,EAAE,KAAK,EAAE,QAAQ,EAAE,OAAO,EAAE,qCAAqC,EAAE,CAAC,CAAC;IAC9E,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,QAAQ,EAAE,OAAO,CAAC,KAAK,CAAC,CAAC;IAExD,MAAM,KAAK,GAAe;QACxB,YAAY,EAAE,OAAO,CAAC,MAAM;QAC5B,eAAe,EAAE,UAAU,CAAC,IAAI;QAChC,YAAY,EAAE,UAAU,CAAC,IAAI,GAAG,CAAC,cAAc,EAAE,IAAI,IAAI,CAAC,CAAC;QAC3D,QAAQ,EAAE,OAAO,CAAC,QAAQ;QAC1B,QAAQ,EAAE,OAAO,CAAC,QAAQ;QAC1B,YAAY,EAAE,UAAU,GAAG,CAAC;QAC5B,UAAU,EAAE,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,GAAG,EAAE,GAAG,OAAO,CAAC;QACnD,eAAe,EAAE,OAAO,CAAC,aAAa;KACvC,CAAC;IAEF,OAAO;QACL,QAAQ;QACR,OAAO;QACP,UAAU;QACV,WAAW;QACX,cAAc;QACd,eAAe,EAAE,OAAO,CAAC,kBAAkB,CAAC,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,SAAS;QACzE,MAAM,EAAE,MAAM,CAAC,IAAI;QACnB,YAAY,EAAE,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI;QACtD,eAAe,EAAE,OAAO;QACxB,QAAQ;QACR,MAAM;QACN,KAAK;KACN,CAAC;AACJ,CAAC;AAED,KAAK,UAAU,QAAQ,CAAC,KAAkB,EAAE,WAAmB,EAAE,OAAe,EAAE,IAAwC;IACxH,IAAI,MAAM,GAAG,CAAC,CAAC;IACf,MAAM,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,EAAE;QAClC,MAAM,IAAI,GAAG,GAAS,EAAE;YACtB,OAAO,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,WAAW,CAAC,IAAI,KAAK,CAAC,MAAM,EAAE,CAAC;gBACzD,MAAM,IAAI,GAAG,KAAK,CAAC,KAAK,EAAG,CAAC;gBAC5B,MAAM,EAAE,CAAC;gBACT,IAAI,CAAC,IAAI,CAAC;qBACP,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC;qBACtB,OAAO,CAAC,GAAG,EAAE;oBACZ,MAAM,EAAE,CAAC;oBACT,IAAI,OAAO,GAAG,CAAC;wBAAE,UAAU,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;;wBACtC,IAAI,EAAE,CAAC;gBACd,CAAC,CAAC,CAAC;YACP,CAAC;YACD,IAAI,MAAM,KAAK,CAAC,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;gBAAE,OAAO,EAAE,CAAC;QACpD,CAAC,CAAC;QACF,IAAI,EAAE,CAAC;IACT,CAAC,CAAC,CAAC;AACL,CAAC;AAED,KAAK,UAAU,SAAS,CAAC,QAAgB,EAAE,SAAuB;IAChE,MAAM,MAAM,GAAiB,EAAE,CAAC;IAChC,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,QAAQ,CAAC,CAAC;IAC9B,MAAM,UAAU,GAAG,IAAI,GAAG,CAAC,sBAAsB,IAAI,CAAC,GAAG,EAAE,CAAC,QAAQ,CAAC,EAAE,CAAC,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC;IACvI,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,UAAU,EAAE,SAAS,CAAC,CAAC;IACnD,MAAM,CAAC,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,UAAU,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,CAAC;IAE9E,IAAI,WAAW,CAAC,QAAQ,CAAC;QAAE,OAAO,MAAM,CAAC;IAEzC,IAAI,GAAG,CAAC,QAAQ,KAAK,QAAQ,EAAE,CAAC;QAC9B,MAAM,OAAO,GAAG,UAAU,GAAG,CAAC,IAAI,GAAG,CAAC;QACtC,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,OAAO,EAAE,EAAE,GAAG,SAAS,EAAE,SAAS,EAAE,MAAM,EAAE,CAAC,CAAC;QACzE,MAAM,CAAC,YAAY,GAAG,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,OAAO,EAAE,MAAM,EAAE,GAAG,CAAC,MAAM,EAAE,QAAQ,EAAE,GAAG,CAAC,QAAQ,EAAE,gBAAgB,EAAE,GAAG,CAAC,QAAQ,CAAC,UAAU,CAAC,UAAU,CAAC,EAAE,CAAC;IAC/J,CAAC;IAED,MAAM,IAAI,GAAG,GAAG,CAAC,QAAQ,CAAC;IAC1B,MAAM,WAAW,GAAG,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,OAAO,IAAI,EAAE,CAAC;IAC7E,MAAM,UAAU,GAAG,GAAG,GAAG,CAAC,QAAQ,KAAK,WAAW,GAAG,CAAC;IACtD,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,UAAU,EAAE,EAAE,GAAG,SAAS,EAAE,SAAS,EAAE,MAAM,EAAE,CAAC,CAAC;IAC5E,MAAM,CAAC,WAAW,GAAG,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,UAAU,EAAE,MAAM,EAAE,GAAG,CAAC,MAAM,EAAE,QAAQ,EAAE,GAAG,CAAC,QAAQ,EAAE,eAAe,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC,KAAK,IAAI,EAAE,CAAC;IAChK,OAAO,MAAM,CAAC;AAChB,CAAC"}

package/dist/core/fetch.d.ts

@@ -0,0 +1,27 @@
+import type { RedirectHop, ResponseInfo } from './types.js';
+export declare const DEFAULT_USER_AGENT: string;
+export interface FetchOptions {
+    timeoutMs?: number;
+    userAgent?: string;
+    headers?: Record<string, string>;
+    maxRedirects?: number;
+    /** Stop reading the body after this many bytes. */
+    maxBodyBytes?: number;
+}
+export interface FetchResult {
+    requestedUrl: string;
+    finalUrl: string;
+    status: number | null;
+    headers: Record<string, string>;
+    body: string | null;
+    redirectChain: RedirectHop[];
+    timeMs: number;
+    contentType: string | null;
+    error?: string;
+}
+/** GET a URL, following redirects manually so the chain is recorded. */
+export declare function fetchPage(url: string, options?: FetchOptions, readBody?: boolean): Promise<FetchResult>;
+/** Status/redirect check without downloading the body. */
+export declare function probeUrl(url: string, options?: FetchOptions): Promise<FetchResult>;
+export declare function toResponseInfo(r: FetchResult): ResponseInfo;
+export declare function isHtmlResponse(r: FetchResult): boolean;

package/dist/core/fetch.js

@@ -0,0 +1,135 @@
+import { VERSION } from '../version.js';
+export const DEFAULT_USER_AGENT = `Mozilla/5.0 (compatible; agentic-seo-check/${VERSION}; +https://github.com/rivi-software/agentic-seo-check)`;
+const KEPT_HEADERS = ['content-type', 'x-robots-tag', 'link', 'content-language', 'cache-control', 'last-modified', 'location', 'content-length', 'vary'];
+const REDIRECT_STATUSES = new Set([301, 302, 303, 307, 308]);
+/** GET a URL, following redirects manually so the chain is recorded. */
+export async function fetchPage(url, options = {}, readBody = true) {
+    const timeoutMs = options.timeoutMs ?? 15_000;
+    const maxRedirects = options.maxRedirects ?? 10;
+    const started = performance.now();
+    const chain = [];
+    let current = url;
+    for (let hop = 0; hop <= maxRedirects; hop++) {
+        const controller = new AbortController();
+        const timer = setTimeout(() => controller.abort(), timeoutMs);
+        try {
+            const res = await fetch(current, {
+                method: 'GET',
+                redirect: 'manual',
+                signal: controller.signal,
+                headers: {
+                    'user-agent': options.userAgent ?? DEFAULT_USER_AGENT,
+                    accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
+                    'accept-language': 'en,*;q=0.5',
+                    ...(options.headers ?? {}),
+                },
+            });
+            const headers = pickHeaders(res.headers);
+            if (REDIRECT_STATUSES.has(res.status) && headers['location']) {
+                chain.push({ url: current, status: res.status });
+                let nextUrl;
+                try {
+                    nextUrl = new URL(headers['location'], current).href;
+                }
+                catch {
+                    return result(url, current, res.status, headers, null, chain, started, `Invalid Location header: ${headers['location']}`);
+                }
+                await res.body?.cancel().catch(() => undefined);
+                if (chain.some((h) => h.url === nextUrl) || nextUrl === current) {
+                    return result(url, nextUrl, res.status, headers, null, chain, started, 'Redirect loop');
+                }
+                current = nextUrl;
+                continue;
+            }
+            let body = null;
+            if (readBody)
+                body = await readLimited(res, options.maxBodyBytes ?? 5_000_000);
+            else
+                await res.body?.cancel().catch(() => undefined);
+            return result(url, current, res.status, headers, body, chain, started);
+        }
+        catch (err) {
+            const message = err instanceof Error ? (err.name === 'AbortError' ? `Timed out after ${timeoutMs} ms` : err.message) : String(err);
+            return result(url, current, null, {}, null, chain, started, message);
+        }
+        finally {
+            clearTimeout(timer);
+        }
+    }
+    return result(url, current, null, {}, null, chain, started, `More than ${maxRedirects} redirects`);
+}
+/** Status/redirect check without downloading the body. */
+export function probeUrl(url, options = {}) {
+    return fetchPage(url, options, false);
+}
+async function readLimited(res, maxBytes) {
+    if (!res.body)
+        return '';
+    const reader = res.body.getReader();
+    const chunks = [];
+    let total = 0;
+    for (;;) {
+        const { done, value } = await reader.read();
+        if (done)
+            break;
+        if (value) {
+            chunks.push(value);
+            total += value.byteLength;
+            if (total >= maxBytes) {
+                await reader.cancel().catch(() => undefined);
+                break;
+            }
+        }
+    }
+    const charset = /charset=([\w-]+)/i.exec(res.headers.get('content-type') ?? '')?.[1];
+    try {
+        return new TextDecoder(charset ?? 'utf-8').decode(Buffer.concat(chunks));
+    }
+    catch {
+        return new TextDecoder('utf-8').decode(Buffer.concat(chunks));
+    }
+}
+function pickHeaders(headers) {
+    const out = {};
+    for (const name of KEPT_HEADERS) {
+        const value = headers.get(name);
+        if (value !== null)
+            out[name] = value;
+    }
+    return out;
+}
+function result(requestedUrl, finalUrl, status, headers, body, chain, started, error) {
+    const r = {
+        requestedUrl,
+        finalUrl,
+        status,
+        headers,
+        body,
+        redirectChain: chain,
+        timeMs: Math.round(performance.now() - started),
+        contentType: headers['content-type'] ?? null,
+    };
+    if (error)
+        r.error = error;
+    return r;
+}
+export function toResponseInfo(r) {
+    const info = {
+        status: r.status,
+        finalUrl: r.finalUrl,
+        redirectChain: r.redirectChain,
+        timeMs: r.timeMs,
+        contentType: r.contentType,
+        headers: r.headers,
+    };
+    if (r.error)
+        info.error = r.error;
+    return info;
+}
+export function isHtmlResponse(r) {
+    const ct = r.contentType?.toLowerCase() ?? '';
+    if (ct)
+        return ct.includes('text/html') || ct.includes('application/xhtml');
+    return typeof r.body === 'string' && /<html|<!doctype/i.test(r.body.slice(0, 2000));
+}
+//# sourceMappingURL=fetch.js.map

package/dist/core/fetch.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"fetch.js","sourceRoot":"","sources":["../../src/core/fetch.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,eAAe,CAAC;AAGxC,MAAM,CAAC,MAAM,kBAAkB,GAAG,8CAA8C,OAAO,wDAAwD,CAAC;AAEhJ,MAAM,YAAY,GAAG,CAAC,cAAc,EAAE,cAAc,EAAE,MAAM,EAAE,kBAAkB,EAAE,eAAe,EAAE,eAAe,EAAE,UAAU,EAAE,gBAAgB,EAAE,MAAM,CAAC,CAAC;AAuB1J,MAAM,iBAAiB,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC;AAE7D,wEAAwE;AACxE,MAAM,CAAC,KAAK,UAAU,SAAS,CAAC,GAAW,EAAE,UAAwB,EAAE,EAAE,QAAQ,GAAG,IAAI;IACtF,MAAM,SAAS,GAAG,OAAO,CAAC,SAAS,IAAI,MAAM,CAAC;IAC9C,MAAM,YAAY,GAAG,OAAO,CAAC,YAAY,IAAI,EAAE,CAAC;IAChD,MAAM,OAAO,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC;IAClC,MAAM,KAAK,GAAkB,EAAE,CAAC;IAChC,IAAI,OAAO,GAAG,GAAG,CAAC;IAElB,KAAK,IAAI,GAAG,GAAG,CAAC,EAAE,GAAG,IAAI,YAAY,EAAE,GAAG,EAAE,EAAE,CAAC;QAC7C,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;QACzC,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,UAAU,CAAC,KAAK,EAAE,EAAE,SAAS,CAAC,CAAC;QAC9D,IAAI,CAAC;YACH,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,OAAO,EAAE;gBAC/B,MAAM,EAAE,KAAK;gBACb,QAAQ,EAAE,QAAQ;gBAClB,MAAM,EAAE,UAAU,CAAC,MAAM;gBACzB,OAAO,EAAE;oBACP,YAAY,EAAE,OAAO,CAAC,SAAS,IAAI,kBAAkB;oBACrD,MAAM,EAAE,iEAAiE;oBACzE,iBAAiB,EAAE,YAAY;oBAC/B,GAAG,CAAC,OAAO,CAAC,OAAO,IAAI,EAAE,CAAC;iBAC3B;aACF,CAAC,CAAC;YACH,MAAM,OAAO,GAAG,WAAW,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;YACzC,IAAI,iBAAiB,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,OAAO,CAAC,UAAU,CAAC,EAAE,CAAC;gBAC7D,KAAK,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,OAAO,EAAE,MAAM,EAAE,GAAG,CAAC,MAAM,EAAE,CAAC,CAAC;gBACjD,IAAI,OAAe,CAAC;gBACpB,IAAI,CAAC;oBACH,OAAO,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,UAAU,CAAC,EAAE,OAAO,CAAC,CAAC,IAAI,CAAC;gBACvD,CAAC;gBAAC,MAAM,CAAC;oBACP,OAAO,MAAM,CAAC,GAAG,EAAE,OAAO,EAAE,GAAG,CAAC,MAAM,EAAE,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,OAAO,EAAE,4BAA4B,OAAO,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;gBAC5H,CAAC;gBACD,MAAM,GAAG,CAAC,IAAI,EAAE,MAAM,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;gBAChD,IAAI,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,KAAK,OAAO,CAAC,IAAI,OAAO,KAAK,OAAO,EAAE,CAAC;oBAChE,OAAO,MAAM,CAAC,GAAG,EAAE,OAAO,EAAE,GAAG,CAAC,MAAM,EAAE,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,OAAO,EAAE,eAAe,CAAC,CAAC;gBAC1F,CAAC;gBACD,OAAO,GAAG,OAAO,CAAC;gBAClB,SAAS;YACX,CAAC;YACD,IAAI,IAAI,GAAkB,IAAI,CAAC;YAC/B,IAAI,QAAQ;gBAAE,IAAI,GAAG,MAAM,WAAW,CAAC,GAAG,EAAE,OAAO,CAAC,YAAY,IAAI,SAAS,CAAC,CAAC;;gBAC1E,MAAM,GAAG,CAAC,IAAI,EAAE,MAAM,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;YACrD,OAAO,MAAM,CAAC,GAAG,EAAE,OAAO,EAAE,GAAG,CAAC,MAAM,EAAE,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,OAAO,CAAC,CAAC;QACzE,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,OAAO,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,IAAI,KAAK,YAAY,CAAC,CAAC,CAAC,mBAAmB,SAAS,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YACnI,OAAO,MAAM,CAAC,GAAG,EAAE,OAAO,EAAE,IAAI,EAAE,EAAE,EAAE,IAAI,EAAE,KAAK,EAAE,OAAO,EAAE,OAAO,CAAC,CAAC;QACvE,CAAC;gBAAS,CAAC;YACT,YAAY,CAAC,KAAK,CAAC,CAAC;QACtB,CAAC;IACH,CAAC;IACD,OAAO,MAAM,CAAC,GAAG,EAAE,OAAO,EAAE,IAAI,EAAE,EAAE,EAAE,IAAI,EAAE,KAAK,EAAE,OAAO,EAAE,aAAa,YAAY,YAAY,CAAC,CAAC;AACrG,CAAC;AAED,0DAA0D;AAC1D,MAAM,UAAU,QAAQ,CAAC,GAAW,EAAE,UAAwB,EAAE;IAC9D,OAAO,SAAS,CAAC,GAAG,EAAE,OAAO,EAAE,KAAK,CAAC,CAAC;AACxC,CAAC;AAED,KAAK,UAAU,WAAW,CAAC,GAAa,EAAE,QAAgB;IACxD,IAAI,CAAC,GAAG,CAAC,IAAI;QAAE,OAAO,EAAE,CAAC;IACzB,MAAM,MAAM,GAAG,GAAG,CAAC,IAAI,CAAC,SAAS,EAAE,CAAC;IACpC,MAAM,MAAM,GAAiB,EAAE,CAAC;IAChC,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,SAAS,CAAC;QACR,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,MAAM,MAAM,CAAC,IAAI,EAAE,CAAC;QAC5C,IAAI,IAAI;YAAE,MAAM;QAChB,IAAI,KAAK,EAAE,CAAC;YACV,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACnB,KAAK,IAAI,KAAK,CAAC,UAAU,CAAC;YAC1B,IAAI,KAAK,IAAI,QAAQ,EAAE,CAAC;gBACtB,MAAM,MAAM,CAAC,MAAM,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;gBAC7C,MAAM;YACR,CAAC;QACH,CAAC;IACH,CAAC;IACD,MAAM,OAAO,GAAG,mBAAmB,CAAC,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,IAAI,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;IACrF,IAAI,CAAC;QACH,OAAO,IAAI,WAAW,CAAC,OAAO,IAAI,OAAO,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC;IAC3E,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,WAAW,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC;IAChE,CAAC;AACH,CAAC;AAED,SAAS,WAAW,CAAC,OAAgB;IACnC,MAAM,GAAG,GAA2B,EAAE,CAAC;IACvC,KAAK,MAAM,IAAI,IAAI,YAAY,EAAE,CAAC;QAChC,MAAM,KAAK,GAAG,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QAChC,IAAI,KAAK,KAAK,IAAI;YAAE,GAAG,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC;IACxC,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED,SAAS,MAAM,CAAC,YAAoB,EAAE,QAAgB,EAAE,MAAqB,EAAE,OAA+B,EAAE,IAAmB,EAAE,KAAoB,EAAE,OAAe,EAAE,KAAc;IACxL,MAAM,CAAC,GAAgB;QACrB,YAAY;QACZ,QAAQ;QACR,MAAM;QACN,OAAO;QACP,IAAI;QACJ,aAAa,EAAE,KAAK;QACpB,MAAM,EAAE,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,GAAG,EAAE,GAAG,OAAO,CAAC;QAC/C,WAAW,EAAE,OAAO,CAAC,cAAc,CAAC,IAAI,IAAI;KAC7C,CAAC;IACF,IAAI,KAAK;QAAE,CAAC,CAAC,KAAK,GAAG,KAAK,CAAC;IAC3B,OAAO,CAAC,CAAC;AACX,CAAC;AAED,MAAM,UAAU,cAAc,CAAC,CAAc;IAC3C,MAAM,IAAI,GAAiB;QACzB,MAAM,EAAE,CAAC,CAAC,MAAM;QAChB,QAAQ,EAAE,CAAC,CAAC,QAAQ;QACpB,aAAa,EAAE,CAAC,CAAC,aAAa;QAC9B,MAAM,EAAE,CAAC,CAAC,MAAM;QAChB,WAAW,EAAE,CAAC,CAAC,WAAW;QAC1B,OAAO,EAAE,CAAC,CAAC,OAAO;KACnB,CAAC;IACF,IAAI,CAAC,CAAC,KAAK;QAAE,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC;IAClC,OAAO,IAAI,CAAC;AACd,CAAC;AAED,MAAM,UAAU,cAAc,CAAC,CAAc;IAC3C,MAAM,EAAE,GAAG,CAAC,CAAC,WAAW,EAAE,WAAW,EAAE,IAAI,EAAE,CAAC;IAC9C,IAAI,EAAE;QAAE,OAAO,EAAE,CAAC,QAAQ,CAAC,WAAW,CAAC,IAAI,EAAE,CAAC,QAAQ,CAAC,mBAAmB,CAAC,CAAC;IAC5E,OAAO,OAAO,CAAC,CAAC,IAAI,KAAK,QAAQ,IAAI,kBAAkB,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC,CAAC;AACtF,CAAC"}

package/dist/core/html/extract.d.ts

@@ -0,0 +1,10 @@
+import type { ExtractedPage } from '../types.js';
+export interface ExtractOptions {
+    /** Absolute URL of the document; relative links are resolved against it (or against <base href>). */
+    url: string;
+    /** Treat subdomains of the site host as internal. */
+    includeSubdomains?: boolean;
+    /** Host used to classify links as internal. Defaults to the host of `url`. */
+    siteHost?: string;
+}
+export declare function extractPage(html: string, options: ExtractOptions): ExtractedPage;

package/dist/core/html/extract.js

@@ -0,0 +1,150 @@
+import { load } from 'cheerio';
+import { isSameSite, resolveUrl, hostOf } from '../url.js';
+import { collapseWhitespace, wordCount } from './text.js';
+import { extractStructuredData } from './structured-data.js';
+export function extractPage(html, options) {
+    const $ = load(html);
+    const hasDoctype = /^\s*<!doctype\s+html/i.test(html);
+    const baseHref = $('base[href]').first().attr('href');
+    const base = (baseHref && resolveUrl(baseHref, options.url)) || options.url;
+    const siteHost = options.siteHost ?? hostOf(options.url) ?? '';
+    const titles = $('title')
+        .toArray()
+        .map((el) => collapseWhitespace($(el).text()));
+    const metaDescriptions = $('meta[name]')
+        .toArray()
+        .filter((el) => ($(el).attr('name') ?? '').trim().toLowerCase() === 'description')
+        .map((el) => collapseWhitespace($(el).attr('content') ?? ''));
+    const metaRobots = $('meta[name]')
+        .toArray()
+        .filter((el) => ['robots', 'googlebot', 'googlebot-news', 'bingbot'].includes(($(el).attr('name') ?? '').trim().toLowerCase()))
+        .map((el) => collapseWhitespace($(el).attr('content') ?? '').toLowerCase())
+        .filter(Boolean);
+    const canonicals = $('link[rel]')
+        .toArray()
+        .filter((el) => relList($(el).attr('rel')).includes('canonical'))
+        .map((el) => ($(el).attr('href') ?? '').trim());
+    const headings = $('h1, h2, h3, h4, h5, h6')
+        .toArray()
+        .map((el) => ({ level: Number(el.tagName.slice(1)), text: collapseWhitespace($(el).text()) }));
+    const og = {};
+    const twitter = {};
+    $('meta[property], meta[name]').each((_, el) => {
+        const key = ($(el).attr('property') ?? $(el).attr('name') ?? '').trim().toLowerCase();
+        const content = collapseWhitespace($(el).attr('content') ?? '');
+        if (!key)
+            return;
+        if (key.startsWith('og:') && !(key in og))
+            og[key.slice(3)] = content;
+        else if (key.startsWith('twitter:') && !(key in twitter))
+            twitter[key.slice(8)] = content;
+    });
+    const images = $('img')
+        .toArray()
+        .map((el) => ({
+        src: $(el).attr('src') ?? $(el).attr('data-src') ?? null,
+        alt: $(el).attr('alt') ?? null,
+        width: $(el).attr('width') ?? null,
+        height: $(el).attr('height') ?? null,
+        loading: $(el).attr('loading') ?? null,
+    }));
+    const links = $('a[href]')
+        .toArray()
+        .map((el) => {
+        const href = ($(el).attr('href') ?? '').trim();
+        // Fragment-only and empty hrefs are not links to other documents.
+        const url = href === '' || href.startsWith('#') ? null : resolveUrl(href, base);
+        const rel = $(el).attr('rel') ?? null;
+        const relTokens = relList(rel);
+        const text = collapseWhitespace($(el).text());
+        const ariaLabel = collapseWhitespace($(el).attr('aria-label') ?? '');
+        const title = collapseWhitespace($(el).attr('title') ?? '');
+        const imgAlt = $(el)
+            .find('img[alt]')
+            .toArray()
+            .map((img) => collapseWhitespace($(img).attr('alt') ?? ''))
+            .find(Boolean);
+        return {
+            href,
+            url,
+            text,
+            rel,
+            internal: url ? isSameSite(url, siteHost, options.includeSubdomains) : false,
+            nofollow: relTokens.includes('nofollow'),
+            hasAccessibleName: Boolean(text || ariaLabel || title || imgAlt),
+        };
+    });
+    const hreflang = $('link[rel]')
+        .toArray()
+        .filter((el) => relList($(el).attr('rel')).includes('alternate') && $(el).attr('hreflang'))
+        .map((el) => {
+        const href = ($(el).attr('href') ?? '').trim();
+        return { lang: ($(el).attr('hreflang') ?? '').trim(), href, url: resolveUrl(href, base) };
+    });
+    const structured = extractStructuredData($);
+    const text = visibleText($);
+    return {
+        url: options.url,
+        hasDoctype,
+        lang: attrOrNull($('html').attr('lang')),
+        charset: detectCharset($),
+        viewport: attrOrNull($('meta[name="viewport" i]').first().attr('content')),
+        titles,
+        title: firstNonEmpty(titles),
+        metaDescriptions,
+        metaDescription: firstNonEmpty(metaDescriptions),
+        metaRobots,
+        canonicals,
+        canonical: canonicals[0] ? resolveUrl(canonicals[0], base) : null,
+        headings,
+        h1s: headings.filter((h) => h.level === 1).map((h) => h.text),
+        og,
+        twitter,
+        images,
+        links,
+        hreflang,
+        structuredData: structured.items,
+        structuredDataErrors: structured.errors,
+        text,
+        wordCount: wordCount(text),
+        htmlBytes: Buffer.byteLength(html, 'utf8'),
+        hasBaseHref: Boolean(baseHref),
+    };
+}
+function relList(rel) {
+    return (rel ?? '')
+        .toLowerCase()
+        .split(/\s+/)
+        .filter(Boolean);
+}
+function attrOrNull(value) {
+    const v = value?.trim();
+    return v ? v : null;
+}
+function firstNonEmpty(values) {
+    for (const v of values)
+        if (v)
+            return v;
+    return null;
+}
+function detectCharset($) {
+    const direct = $('meta[charset]').first().attr('charset');
+    if (direct)
+        return direct.trim();
+    const httpEquiv = $('meta[http-equiv]')
+        .toArray()
+        .find((el) => ($(el).attr('http-equiv') ?? '').toLowerCase() === 'content-type');
+    const content = httpEquiv ? $(httpEquiv).attr('content') : undefined;
+    const match = content?.match(/charset=([\w-]+)/i);
+    return match?.[1] ?? null;
+}
+function visibleText($) {
+    const body = $('body').clone();
+    body.find('script, style, noscript, template, svg, iframe, head').remove();
+    // Insert spaces at block boundaries so words don't fuse.
+    body.find('p, div, li, h1, h2, h3, h4, h5, h6, br, tr, td, th, section, article, header, footer, nav, blockquote, pre, dd, dt, figcaption').each((_, el) => {
+        $(el).prepend(' ').append(' ');
+    });
+    return collapseWhitespace(body.text());
+}
+//# sourceMappingURL=extract.js.map

package/dist/core/html/extract.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"extract.js","sourceRoot":"","sources":["../../../src/core/html/extract.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAmB,MAAM,SAAS,CAAC;AAEhD,OAAO,EAAE,UAAU,EAAE,UAAU,EAAE,MAAM,EAAE,MAAM,WAAW,CAAC;AAC3D,OAAO,EAAE,kBAAkB,EAAE,SAAS,EAAE,MAAM,WAAW,CAAC;AAC1D,OAAO,EAAE,qBAAqB,EAAE,MAAM,sBAAsB,CAAC;AAW7D,MAAM,UAAU,WAAW,CAAC,IAAY,EAAE,OAAuB;IAC/D,MAAM,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC;IACrB,MAAM,UAAU,GAAG,uBAAuB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACtD,MAAM,QAAQ,GAAG,CAAC,CAAC,YAAY,CAAC,CAAC,KAAK,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IACtD,MAAM,IAAI,GAAG,CAAC,QAAQ,IAAI,UAAU,CAAC,QAAQ,EAAE,OAAO,CAAC,GAAG,CAAC,CAAC,IAAI,OAAO,CAAC,GAAG,CAAC;IAC5E,MAAM,QAAQ,GAAG,OAAO,CAAC,QAAQ,IAAI,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC;IAE/D,MAAM,MAAM,GAAG,CAAC,CAAC,OAAO,CAAC;SACtB,OAAO,EAAE;SACT,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,kBAAkB,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;IAEjD,MAAM,gBAAgB,GAAG,CAAC,CAAC,YAAY,CAAC;SACrC,OAAO,EAAE;SACT,MAAM,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,KAAK,aAAa,CAAC;SACjF,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,kBAAkB,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;IAEhE,MAAM,UAAU,GAAG,CAAC,CAAC,YAAY,CAAC;SAC/B,OAAO,EAAE;SACT,MAAM,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,QAAQ,EAAE,WAAW,EAAE,gBAAgB,EAAE,SAAS,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC,CAAC;SAC9H,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,kBAAkB,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,CAAC,CAAC,WAAW,EAAE,CAAC;SAC1E,MAAM,CAAC,OAAO,CAAC,CAAC;IAEnB,MAAM,UAAU,GAAG,CAAC,CAAC,WAAW,CAAC;SAC9B,OAAO,EAAE;SACT,MAAM,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC;SAChE,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;IAElD,MAAM,QAAQ,GAAkB,CAAC,CAAC,wBAAwB,CAAC;SACxD,OAAO,EAAE;SACT,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,EAAE,KAAK,EAAE,MAAM,CAAC,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,kBAAkB,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC;IAEjG,MAAM,EAAE,GAA2B,EAAE,CAAC;IACtC,MAAM,OAAO,GAA2B,EAAE,CAAC;IAC3C,CAAC,CAAC,4BAA4B,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,EAAE,EAAE;QAC7C,MAAM,GAAG,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;QACtF,MAAM,OAAO,GAAG,kBAAkB,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,CAAC,CAAC;QAChE,IAAI,CAAC,GAAG;YAAE,OAAO;QACjB,IAAI,GAAG,CAAC,UAAU,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,GAAG,IAAI,EAAE,CAAC;YAAE,EAAE,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,GAAG,OAAO,CAAC;aACjE,IAAI,GAAG,CAAC,UAAU,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,GAAG,IAAI,OAAO,CAAC;YAAE,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,GAAG,OAAO,CAAC;IAC5F,CAAC,CAAC,CAAC;IAEH,MAAM,MAAM,GAAgB,CAAC,CAAC,KAAK,CAAC;SACjC,OAAO,EAAE;SACT,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC;QACZ,GAAG,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,UAAU,CAAC,IAAI,IAAI;QACxD,GAAG,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,IAAI;QAC9B,KAAK,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,IAAI;QAClC,MAAM,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,IAAI;QACpC,OAAO,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,IAAI;KACvC,CAAC,CAAC,CAAC;IAEN,MAAM,KAAK,GAAe,CAAC,CAAC,SAAS,CAAC;SACnC,OAAO,EAAE;SACT,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE;QACV,MAAM,IAAI,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;QAC/C,kEAAkE;QAClE,MAAM,GAAG,GAAG,IAAI,KAAK,EAAE,IAAI,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,UAAU,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;QAChF,MAAM,GAAG,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,IAAI,CAAC;QACtC,MAAM,SAAS,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC;QAC/B,MAAM,IAAI,GAAG,kBAAkB,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;QAC9C,MAAM,SAAS,GAAG,kBAAkB,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,YAAY,CAAC,IAAI,EAAE,CAAC,CAAC;QACrE,MAAM,KAAK,GAAG,kBAAkB,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC;QAC5D,MAAM,MAAM,GAAG,CAAC,CAAC,EAAE,CAAC;aACjB,IAAI,CAAC,UAAU,CAAC;aAChB,OAAO,EAAE;aACT,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,kBAAkB,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC;aAC1D,IAAI,CAAC,OAAO,CAAC,CAAC;QACjB,OAAO;YACL,IAAI;YACJ,GAAG;YACH,IAAI;YACJ,GAAG;YACH,QAAQ,EAAE,GAAG,CAAC,CAAC,CAAC,UAAU,CAAC,GAAG,EAAE,QAAQ,EAAE,OAAO,CAAC,iBAAiB,CAAC,CAAC,CAAC,CAAC,KAAK;YAC5E,QAAQ,EAAE,SAAS,CAAC,QAAQ,CAAC,UAAU,CAAC;YACxC,iBAAiB,EAAE,OAAO,CAAC,IAAI,IAAI,SAAS,IAAI,KAAK,IAAI,MAAM,CAAC;SACjE,CAAC;IACJ,CAAC,CAAC,CAAC;IAEL,MAAM,QAAQ,GAAmB,CAAC,CAAC,WAAW,CAAC;SAC5C,OAAO,EAAE;SACT,MAAM,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,QAAQ,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;SAC1F,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE;QACV,MAAM,IAAI,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;QAC/C,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,UAAU,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,EAAE,IAAI,EAAE,GAAG,EAAE,UAAU,CAAC,IAAI,EAAE,IAAI,CAAC,EAAE,CAAC;IAC5F,CAAC,CAAC,CAAC;IAEL,MAAM,UAAU,GAAG,qBAAqB,CAAC,CAAC,CAAC,CAAC;IAE5C,MAAM,IAAI,GAAG,WAAW,CAAC,CAAC,CAAC,CAAC;IAE5B,OAAO;QACL,GAAG,EAAE,OAAO,CAAC,GAAG;QAChB,UAAU;QACV,IAAI,EAAE,UAAU,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QACxC,OAAO,EAAE,aAAa,CAAC,CAAC,CAAC;QACzB,QAAQ,EAAE,UAAU,CAAC,CAAC,CAAC,yBAAyB,CAAC,CAAC,KAAK,EAAE,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QAC1E,MAAM;QACN,KAAK,EAAE,aAAa,CAAC,MAAM,CAAC;QAC5B,gBAAgB;QAChB,eAAe,EAAE,aAAa,CAAC,gBAAgB,CAAC;QAChD,UAAU;QACV,UAAU;QACV,SAAS,EAAE,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,UAAU,CAAC,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI;QACjE,QAAQ;QACR,GAAG,EAAE,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;QAC7D,EAAE;QACF,OAAO;QACP,MAAM;QACN,KAAK;QACL,QAAQ;QACR,cAAc,EAAE,UAAU,CAAC,KAAK;QAChC,oBAAoB,EAAE,UAAU,CAAC,MAAM;QACvC,IAAI;QACJ,SAAS,EAAE,SAAS,CAAC,IAAI,CAAC;QAC1B,SAAS,EAAE,MAAM,CAAC,UAAU,CAAC,IAAI,EAAE,MAAM,CAAC;QAC1C,WAAW,EAAE,OAAO,CAAC,QAAQ,CAAC;KAC/B,CAAC;AACJ,CAAC;AAED,SAAS,OAAO,CAAC,GAA8B;IAC7C,OAAO,CAAC,GAAG,IAAI,EAAE,CAAC;SACf,WAAW,EAAE;SACb,KAAK,CAAC,KAAK,CAAC;SACZ,MAAM,CAAC,OAAO,CAAC,CAAC;AACrB,CAAC;AAED,SAAS,UAAU,CAAC,KAAyB;IAC3C,MAAM,CAAC,GAAG,KAAK,EAAE,IAAI,EAAE,CAAC;IACxB,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;AACtB,CAAC;AAED,SAAS,aAAa,CAAC,MAAgB;IACrC,KAAK,MAAM,CAAC,IAAI,MAAM;QAAE,IAAI,CAAC;YAAE,OAAO,CAAC,CAAC;IACxC,OAAO,IAAI,CAAC;AACd,CAAC;AAED,SAAS,aAAa,CAAC,CAAa;IAClC,MAAM,MAAM,GAAG,CAAC,CAAC,eAAe,CAAC,CAAC,KAAK,EAAE,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;IAC1D,IAAI,MAAM;QAAE,OAAO,MAAM,CAAC,IAAI,EAAE,CAAC;IACjC,MAAM,SAAS,GAAG,CAAC,CAAC,kBAAkB,CAAC;SACpC,OAAO,EAAE;SACT,IAAI,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,YAAY,CAAC,IAAI,EAAE,CAAC,CAAC,WAAW,EAAE,KAAK,cAAc,CAAC,CAAC;IACnF,MAAM,OAAO,GAAG,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;IACrE,MAAM,KAAK,GAAG,OAAO,EAAE,KAAK,CAAC,mBAAmB,CAAC,CAAC;IAClD,OAAO,KAAK,EAAE,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC;AAC5B,CAAC;AAED,SAAS,WAAW,CAAC,CAAa;IAChC,MAAM,IAAI,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,CAAC;IAC/B,IAAI,CAAC,IAAI,CAAC,sDAAsD,CAAC,CAAC,MAAM,EAAE,CAAC;IAC3E,yDAAyD;IACzD,IAAI,CAAC,IAAI,CAAC,gIAAgI,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,EAAE,EAAE;QACzJ,CAAC,CAAC,EAAE,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;IACjC,CAAC,CAAC,CAAC;IACH,OAAO,kBAAkB,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC;AACzC,CAAC"}

package/dist/core/html/structured-data.d.ts

@@ -0,0 +1,11 @@
+import type { CheerioAPI } from 'cheerio';
+import type { StructuredDataError, StructuredDataItem } from '../types.js';
+export interface StructuredDataExtraction {
+    items: StructuredDataItem[];
+    errors: StructuredDataError[];
+}
+export declare function extractStructuredData($: CheerioAPI): StructuredDataExtraction;
+export declare function contextString(value: unknown): string | null;
+export declare function typesOf(value: unknown): string[];
+/** `https://schema.org/Article` -> `Article`, `schema:Article` -> `Article`. */
+export declare function shortType(type: string): string;

package/dist/core/html/structured-data.js

@@ -0,0 +1,185 @@
+import { collapseWhitespace } from './text.js';
+export function extractStructuredData($) {
+    const items = [];
+    const errors = [];
+    $('script[type]').each((_, el) => {
+        const type = ($(el).attr('type') ?? '').trim().toLowerCase();
+        if (type !== 'application/ld+json')
+            return;
+        const raw = $(el).text().trim();
+        if (!raw) {
+            errors.push({ source: 'json-ld', message: 'Empty JSON-LD script block', snippet: '' });
+            return;
+        }
+        let parsed;
+        try {
+            parsed = JSON.parse(stripCdata(raw));
+        }
+        catch (err) {
+            errors.push({
+                source: 'json-ld',
+                message: `Invalid JSON: ${err.message}`,
+                snippet: raw.slice(0, 200),
+            });
+            return;
+        }
+        for (const node of flattenJsonLd(parsed, null))
+            items.push(node);
+    });
+    for (const item of extractMicrodata($))
+        items.push(item);
+    return { items, errors };
+}
+function stripCdata(text) {
+    return text
+        .replace(/^\s*<!--/, '')
+        .replace(/-->\s*$/, '')
+        .replace(/^\s*\/\*\s*<!\[CDATA\[\s*\*\//, '')
+        .replace(/\/\*\s*\]\]>\s*\*\/\s*$/, '')
+        .replace(/^\s*<!\[CDATA\[/, '')
+        .replace(/\]\]>\s*$/, '');
+}
+function flattenJsonLd(node, inheritedContext) {
+    if (Array.isArray(node))
+        return node.flatMap((n) => flattenJsonLd(n, inheritedContext));
+    if (!node || typeof node !== 'object')
+        return [];
+    const obj = node;
+    const context = contextString(obj['@context']) ?? inheritedContext;
+    if (Array.isArray(obj['@graph'])) {
+        return obj['@graph'].flatMap((n) => flattenJsonLd(n, context));
+    }
+    return [{ source: 'json-ld', types: typesOf(obj['@type']), context, data: obj }];
+}
+export function contextString(value) {
+    if (typeof value === 'string')
+        return value;
+    if (Array.isArray(value)) {
+        for (const v of value) {
+            const s = contextString(v);
+            if (s)
+                return s;
+        }
+        return null;
+    }
+    if (value && typeof value === 'object') {
+        const vocab = value['@vocab'];
+        if (typeof vocab === 'string')
+            return vocab;
+    }
+    return null;
+}
+export function typesOf(value) {
+    if (typeof value === 'string')
+        return [shortType(value)];
+    if (Array.isArray(value))
+        return value.filter((v) => typeof v === 'string').map(shortType);
+    return [];
+}
+/** `https://schema.org/Article` -> `Article`, `schema:Article` -> `Article`. */
+export function shortType(type) {
+    const t = type.trim();
+    const slash = t.lastIndexOf('/');
+    const colon = t.lastIndexOf(':');
+    const cut = Math.max(slash, t.startsWith('http') ? -1 : colon);
+    return cut >= 0 ? t.slice(cut + 1) : t;
+}
+/* ---------------------------------------------------------------------------
+ * Microdata
+ * ------------------------------------------------------------------------- */
+function extractMicrodata($) {
+    const items = [];
+    $('[itemscope]').each((_, el) => {
+        // Top-level scopes only: nested scopes become property values.
+        if ($(el).parents('[itemscope]').length > 0)
+            return;
+        const data = microdataObject($, el);
+        items.push({
+            source: 'microdata',
+            types: typesOf(data['@type']),
+            context: contextString(data['@context']),
+            data,
+        });
+    });
+    return items;
+}
+function microdataObject($, scope) {
+    const obj = {};
+    const itemtype = $(scope).attr('itemtype');
+    if (itemtype) {
+        const types = itemtype.split(/\s+/).filter(Boolean);
+        obj['@type'] = types.length === 1 ? types[0] : types;
+        const first = types[0];
+        if (first) {
+            try {
+                const u = new URL(first);
+                obj['@context'] = `${u.protocol}//${u.host}`;
+            }
+            catch {
+                /* not a URL */
+            }
+        }
+    }
+    const itemid = $(scope).attr('itemid');
+    if (itemid)
+        obj['@id'] = itemid;
+    const props = [];
+    collectProps($, scope, props);
+    for (const el of props) {
+        const names = ($(el).attr('itemprop') ?? '').split(/\s+/).filter(Boolean);
+        const value = $(el).is('[itemscope]') ? microdataObject($, el) : microdataValue($, el);
+        for (const name of names) {
+            const existing = obj[name];
+            if (existing === undefined)
+                obj[name] = value;
+            else if (Array.isArray(existing))
+                existing.push(value);
+            else
+                obj[name] = [existing, value];
+        }
+    }
+    return obj;
+}
+/** Walk descendants, stopping at nested itemscopes (they are added as a single property). */
+function collectProps($, root, out) {
+    const children = $(root).children().toArray();
+    for (const child of children) {
+        const hasProp = $(child).is('[itemprop]');
+        if (hasProp)
+            out.push(child);
+        if (!$(child).is('[itemscope]'))
+            collectProps($, child, out);
+    }
+}
+function microdataValue($, el) {
+    const $el = $(el);
+    const tag = (el.tagName ?? '').toLowerCase();
+    switch (tag) {
+        case 'meta':
+            return $el.attr('content') ?? '';
+        case 'img':
+        case 'audio':
+        case 'embed':
+        case 'iframe':
+        case 'source':
+        case 'track':
+        case 'video':
+            return $el.attr('src') ?? '';
+        case 'a':
+        case 'area':
+        case 'link':
+            return $el.attr('href') ?? '';
+        case 'object':
+            return $el.attr('data') ?? '';
+        case 'data':
+        case 'meter':
+            return $el.attr('value') ?? collapseWhitespace($el.text());
+        case 'time':
+            return $el.attr('datetime') ?? collapseWhitespace($el.text());
+        default: {
+            const content = $el.attr('content');
+            return content !== undefined ? content : collapseWhitespace($el.text());
+        }
+    }
+}
+//# sourceMappingURL=structured-data.js.map

package/dist/core/html/structured-data.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"structured-data.js","sourceRoot":"","sources":["../../../src/core/html/structured-data.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,kBAAkB,EAAE,MAAM,WAAW,CAAC;AAO/C,MAAM,UAAU,qBAAqB,CAAC,CAAa;IACjD,MAAM,KAAK,GAAyB,EAAE,CAAC;IACvC,MAAM,MAAM,GAA0B,EAAE,CAAC;IAEzC,CAAC,CAAC,cAAc,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,EAAE,EAAE;QAC/B,MAAM,IAAI,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;QAC7D,IAAI,IAAI,KAAK,qBAAqB;YAAE,OAAO;QAC3C,MAAM,GAAG,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC;QAChC,IAAI,CAAC,GAAG,EAAE,CAAC;YACT,MAAM,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,SAAS,EAAE,OAAO,EAAE,4BAA4B,EAAE,OAAO,EAAE,EAAE,EAAE,CAAC,CAAC;YACvF,OAAO;QACT,CAAC;QACD,IAAI,MAAe,CAAC;QACpB,IAAI,CAAC;YACH,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC;QACvC,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,CAAC,IAAI,CAAC;gBACV,MAAM,EAAE,SAAS;gBACjB,OAAO,EAAE,iBAAkB,GAAa,CAAC,OAAO,EAAE;gBAClD,OAAO,EAAE,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC;aAC3B,CAAC,CAAC;YACH,OAAO;QACT,CAAC;QACD,KAAK,MAAM,IAAI,IAAI,aAAa,CAAC,MAAM,EAAE,IAAI,CAAC;YAAE,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACnE,CAAC,CAAC,CAAC;IAEH,KAAK,MAAM,IAAI,IAAI,gBAAgB,CAAC,CAAC,CAAC;QAAE,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAEzD,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC;AAC3B,CAAC;AAED,SAAS,UAAU,CAAC,IAAY;IAC9B,OAAO,IAAI;SACR,OAAO,CAAC,UAAU,EAAE,EAAE,CAAC;SACvB,OAAO,CAAC,SAAS,EAAE,EAAE,CAAC;SACtB,OAAO,CAAC,+BAA+B,EAAE,EAAE,CAAC;SAC5C,OAAO,CAAC,yBAAyB,EAAE,EAAE,CAAC;SACtC,OAAO,CAAC,iBAAiB,EAAE,EAAE,CAAC;SAC9B,OAAO,CAAC,WAAW,EAAE,EAAE,CAAC,CAAC;AAC9B,CAAC;AAED,SAAS,aAAa,CAAC,IAAa,EAAE,gBAA+B;IACnE,IAAI,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC;QAAE,OAAO,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,aAAa,CAAC,CAAC,EAAE,gBAAgB,CAAC,CAAC,CAAC;IACxF,IAAI,CAAC,IAAI,IAAI,OAAO,IAAI,KAAK,QAAQ;QAAE,OAAO,EAAE,CAAC;IACjD,MAAM,GAAG,GAAG,IAA+B,CAAC;IAC5C,MAAM,OAAO,GAAG,aAAa,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC,IAAI,gBAAgB,CAAC;IACnE,IAAI,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,EAAE,CAAC;QACjC,OAAQ,GAAG,CAAC,QAAQ,CAAe,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,aAAa,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC,CAAC;IAChF,CAAC;IACD,OAAO,CAAC,EAAE,MAAM,EAAE,SAAS,EAAE,KAAK,EAAE,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,EAAE,OAAO,EAAE,IAAI,EAAE,GAAG,EAAE,CAAC,CAAC;AACnF,CAAC;AAED,MAAM,UAAU,aAAa,CAAC,KAAc;IAC1C,IAAI,OAAO,KAAK,KAAK,QAAQ;QAAE,OAAO,KAAK,CAAC;IAC5C,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;QACzB,KAAK,MAAM,CAAC,IAAI,KAAK,EAAE,CAAC;YACtB,MAAM,CAAC,GAAG,aAAa,CAAC,CAAC,CAAC,CAAC;YAC3B,IAAI,CAAC;gBAAE,OAAO,CAAC,CAAC;QAClB,CAAC;QACD,OAAO,IAAI,CAAC;IACd,CAAC;IACD,IAAI,KAAK,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;QACvC,MAAM,KAAK,GAAI,KAAiC,CAAC,QAAQ,CAAC,CAAC;QAC3D,IAAI,OAAO,KAAK,KAAK,QAAQ;YAAE,OAAO,KAAK,CAAC;IAC9C,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,MAAM,UAAU,OAAO,CAAC,KAAc;IACpC,IAAI,OAAO,KAAK,KAAK,QAAQ;QAAE,OAAO,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,CAAC;IACzD,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC;QAAE,OAAO,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAe,EAAE,CAAC,OAAO,CAAC,KAAK,QAAQ,CAAC,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;IACxG,OAAO,EAAE,CAAC;AACZ,CAAC;AAED,gFAAgF;AAChF,MAAM,UAAU,SAAS,CAAC,IAAY;IACpC,MAAM,CAAC,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC;IACtB,MAAM,KAAK,GAAG,CAAC,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC;IACjC,MAAM,KAAK,GAAG,CAAC,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC;IACjC,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,CAAC,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;IAC/D,OAAO,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;AACzC,CAAC;AAED;;+EAE+E;AAE/E,SAAS,gBAAgB,CAAC,CAAa;IACrC,MAAM,KAAK,GAAyB,EAAE,CAAC;IACvC,CAAC,CAAC,aAAa,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,EAAE,EAAE;QAC9B,+DAA+D;QAC/D,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC,OAAO,CAAC,aAAa,CAAC,CAAC,MAAM,GAAG,CAAC;YAAE,OAAO;QACpD,MAAM,IAAI,GAAG,eAAe,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;QACpC,KAAK,CAAC,IAAI,CAAC;YACT,MAAM,EAAE,WAAW;YACnB,KAAK,EAAE,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YAC7B,OAAO,EAAE,aAAa,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;YACxC,IAAI;SACL,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IACH,OAAO,KAAK,CAAC;AACf,CAAC;AAED,SAAS,eAAe,CAAC,CAAa,EAAE,KAAc;IACpD,MAAM,GAAG,GAA4B,EAAE,CAAC;IACxC,MAAM,QAAQ,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;IAC3C,IAAI,QAAQ,EAAE,CAAC;QACb,MAAM,KAAK,GAAG,QAAQ,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;QACpD,GAAG,CAAC,OAAO,CAAC,GAAG,KAAK,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC;QACrD,MAAM,KAAK,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;QACvB,IAAI,KAAK,EAAE,CAAC;YACV,IAAI,CAAC;gBACH,MAAM,CAAC,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,CAAC;gBACzB,GAAG,CAAC,UAAU,CAAC,GAAG,GAAG,CAAC,CAAC,QAAQ,KAAK,CAAC,CAAC,IAAI,EAAE,CAAC;YAC/C,CAAC;YAAC,MAAM,CAAC;gBACP,eAAe;YACjB,CAAC;QACH,CAAC;IACH,CAAC;IACD,MAAM,MAAM,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;IACvC,IAAI,MAAM;QAAE,GAAG,CAAC,KAAK,CAAC,GAAG,MAAM,CAAC;IAEhC,MAAM,KAAK,GAAc,EAAE,CAAC;IAC5B,YAAY,CAAC,CAAC,EAAE,KAAK,EAAE,KAAK,CAAC,CAAC;IAC9B,KAAK,MAAM,EAAE,IAAI,KAAK,EAAE,CAAC;QACvB,MAAM,KAAK,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,UAAU,CAAC,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;QAC1E,MAAM,KAAK,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,eAAe,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;QACvF,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,MAAM,QAAQ,GAAG,GAAG,CAAC,IAAI,CAAC,CAAC;YAC3B,IAAI,QAAQ,KAAK,SAAS;gBAAE,GAAG,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC;iBACzC,IAAI,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC;gBAAE,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;;gBAClD,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAC;QACrC,CAAC;IACH,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED,6FAA6F;AAC7F,SAAS,YAAY,CAAC,CAAa,EAAE,IAAa,EAAE,GAAc;IAChE,MAAM,QAAQ,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,QAAQ,EAAE,CAAC,OAAO,EAAE,CAAC;IAC9C,KAAK,MAAM,KAAK,IAAI,QAAQ,EAAE,CAAC;QAC7B,MAAM,OAAO,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,YAAY,CAAC,CAAC;QAC1C,IAAI,OAAO;YAAE,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAC7B,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,aAAa,CAAC;YAAE,YAAY,CAAC,CAAC,EAAE,KAAK,EAAE,GAAG,CAAC,CAAC;IAC/D,CAAC;AACH,CAAC;AAED,SAAS,cAAc,CAAC,CAAa,EAAE,EAAW;IAChD,MAAM,GAAG,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC;IAClB,MAAM,GAAG,GAAG,CAAE,EAAc,CAAC,OAAO,IAAI,EAAE,CAAC,CAAC,WAAW,EAAE,CAAC;IAC1D,QAAQ,GAAG,EAAE,CAAC;QACZ,KAAK,MAAM;YACT,OAAO,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,CAAC;QACnC,KAAK,KAAK,CAAC;QACX,KAAK,OAAO,CAAC;QACb,KAAK,OAAO,CAAC;QACb,KAAK,QAAQ,CAAC;QACd,KAAK,QAAQ,CAAC;QACd,KAAK,OAAO,CAAC;QACb,KAAK,OAAO;YACV,OAAO,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC;QAC/B,KAAK,GAAG,CAAC;QACT,KAAK,MAAM,CAAC;QACZ,KAAK,MAAM;YACT,OAAO,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;QAChC,KAAK,QAAQ;YACX,OAAO,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;QAChC,KAAK,MAAM,CAAC;QACZ,KAAK,OAAO;YACV,OAAO,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,kBAAkB,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,CAAC;QAC7D,KAAK,MAAM;YACT,OAAO,GAAG,CAAC,IAAI,CAAC,UAAU,CAAC,IAAI,kBAAkB,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,CAAC;QAChE,OAAO,CAAC,CAAC,CAAC;YACR,MAAM,OAAO,GAAG,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;YACpC,OAAO,OAAO,KAAK,SAAS,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,kBAAkB,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,CAAC;QAC1E,CAAC;IACH,CAAC;AACH,CAAC"}

package/dist/core/html/text.d.ts

@@ -0,0 +1,23 @@
+/** Text analysis helpers: word counts, readability, placeholder detection. */
+export declare function words(text: string): string[];
+export declare function wordCount(text: string): number;
+export declare function collapseWhitespace(text: string): string;
+export declare function sentenceCount(text: string): number;
+/** English syllable heuristic: vowel groups minus silent trailing e, min 1. */
+export declare function syllables(word: string): number;
+/**
+ * Flesch Reading Ease (English formula). Returns null when there is not enough text.
+ * 90-100 very easy, 60-70 plain English, 30-50 difficult, <30 very difficult.
+ */
+export declare function fleschReadingEase(text: string): number | null;
+export declare function hasPlaceholderText(text: string): boolean;
+export declare const GENERIC_TITLES: Set<string>;
+export declare function isGenericTitle(title: string): boolean;
+export declare const GENERIC_ANCHORS: Set<string>;
+export declare function isGenericAnchor(text: string): boolean;
+export declare function normalizeForComparison(text: string): string;
+export declare function includesKeyword(haystack: string | null | undefined, keyword: string): boolean;
+export declare function keywordOccurrences(text: string, keyword: string): number;
+/** 5-word shingles for near-duplicate detection. */
+export declare function shingles(text: string, size?: number): Set<string>;
+export declare function jaccard(a: Set<string>, b: Set<string>): number;