@rivi-software/agentic-seo-check 0.1.1 → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (60) hide show
  1. package/README.md +61 -5
  2. package/dist/cli.js +69 -26
  3. package/dist/cli.js.map +1 -1
  4. package/dist/core/aeo/audit.d.ts +44 -0
  5. package/dist/core/aeo/audit.js +310 -0
  6. package/dist/core/aeo/audit.js.map +1 -0
  7. package/dist/core/aeo/bots.d.ts +30 -0
  8. package/dist/core/aeo/bots.js +299 -0
  9. package/dist/core/aeo/bots.js.map +1 -0
  10. package/dist/core/aeo/llms-txt.d.ts +8 -0
  11. package/dist/core/aeo/llms-txt.js +88 -0
  12. package/dist/core/aeo/llms-txt.js.map +1 -0
  13. package/dist/core/aeo/page.d.ts +53 -0
  14. package/dist/core/aeo/page.js +312 -0
  15. package/dist/core/aeo/page.js.map +1 -0
  16. package/dist/core/aeo/probe.d.ts +10 -0
  17. package/dist/core/aeo/probe.js +53 -0
  18. package/dist/core/aeo/probe.js.map +1 -0
  19. package/dist/core/aeo/site-rules.d.ts +13 -0
  20. package/dist/core/aeo/site-rules.js +178 -0
  21. package/dist/core/aeo/site-rules.js.map +1 -0
  22. package/dist/core/audit.d.ts +23 -2
  23. package/dist/core/audit.js +47 -22
  24. package/dist/core/audit.js.map +1 -1
  25. package/dist/core/crawl.d.ts +5 -1
  26. package/dist/core/crawl.js +26 -7
  27. package/dist/core/crawl.js.map +1 -1
  28. package/dist/core/html/answer.d.ts +4 -0
  29. package/dist/core/html/answer.js +198 -0
  30. package/dist/core/html/answer.js.map +1 -0
  31. package/dist/core/html/dom.d.ts +4 -0
  32. package/dist/core/html/dom.js +67 -1
  33. package/dist/core/html/dom.js.map +1 -1
  34. package/dist/core/html/extract.js +5 -71
  35. package/dist/core/html/extract.js.map +1 -1
  36. package/dist/core/report/markdown.d.ts +3 -1
  37. package/dist/core/report/markdown.js +102 -0
  38. package/dist/core/report/markdown.js.map +1 -1
  39. package/dist/core/robots.d.ts +7 -0
  40. package/dist/core/robots.js +12 -1
  41. package/dist/core/robots.js.map +1 -1
  42. package/dist/core/rules/aeo.d.ts +11 -0
  43. package/dist/core/rules/aeo.js +423 -0
  44. package/dist/core/rules/aeo.js.map +1 -0
  45. package/dist/core/rules/index.d.ts +4 -0
  46. package/dist/core/rules/index.js +7 -1
  47. package/dist/core/rules/index.js.map +1 -1
  48. package/dist/core/score.d.ts +4 -2
  49. package/dist/core/score.js.map +1 -1
  50. package/dist/core/static.js +1 -0
  51. package/dist/core/static.js.map +1 -1
  52. package/dist/core/types.d.ts +177 -2
  53. package/dist/core/types.js.map +1 -1
  54. package/dist/index.d.ts +6 -2
  55. package/dist/index.js +4 -2
  56. package/dist/index.js.map +1 -1
  57. package/dist/mcp.js +129 -3
  58. package/dist/mcp.js.map +1 -1
  59. package/package.json +6 -2
  60. package/skills/seo-audit/SKILL.md +8 -2
@@ -0,0 +1 @@
1
+ {"version":3,"file":"bots.js","sourceRoot":"","sources":["../../../src/core/aeo/bots.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAuB5C,MAAM,WAAW,GAAG,6CAA6C,CAAC;AAElE,MAAM,cAAc,GAClB,iIAAiI,CAAC;AAEpI,MAAM,eAAe,GAAG,wCAAwC,CAAC;AAEjE,MAAM,oBAAoB,GACxB,sFAAsF,CAAC;AAEzF;;;GAGG;AACH,MAAM,CAAC,MAAM,OAAO,GAAY;IAC9B;QACE,IAAI,EAAE,WAAW;QACjB,QAAQ,EAAE,QAAQ;QAClB,OAAO,EAAE,QAAQ;QACjB,OAAO,EAAE,yCAAyC;QAClD,IAAI,EAAE,kEAAkE;KACzE;IACD;QACE,IAAI,EAAE,SAAS;QACf,QAAQ,EAAE,WAAW;QACrB,OAAO,EAAE,QAAQ;QACjB,OAAO,EAAE,mCAAmC;QAC5C,IAAI,EAAE,oEAAoE;KAC3E;IACD;QACE,IAAI,EAAE,UAAU;QAChB,QAAQ,EAAE,OAAO;QACjB,OAAO,EAAE,QAAQ;QACjB,OAAO,EAAE,mCAAmC;QAC5C,IAAI,EAAE,wCAAwC;KAC/C;IACD;QACE,IAAI,EAAE,eAAe;QACrB,QAAQ,EAAE,QAAQ;QAClB,OAAO,EAAE,QAAQ;QACjB,OAAO,EAAE,wBAAwB;QACjC,IAAI,EAAE,WAAW;QACjB,SAAS,EACP,kHAAkH;KACrH;IACD;QACE,IAAI,EAAE,cAAc;QACpB,QAAQ,EAAE,QAAQ;QAClB,OAAO,EAAE,QAAQ;QACjB,OAAO,EAAE,4DAA4D;QACrE,IAAI,EAAE,WAAW;QACjB,SAAS,EACP,2GAA2G;KAC9G;IACD;QACE,IAAI,EAAE,kBAAkB;QACxB,QAAQ,EAAE,WAAW;QACrB,OAAO,EAAE,QAAQ;QACjB,OAAO,EAAE,yBAAyB;QAClC,IAAI,EAAE,cAAc;QACpB,SAAS,EACP,kHAAkH;KACrH;IACD;QACE,IAAI,EAAE,aAAa;QACnB,QAAQ,EAAE,WAAW;QACrB,OAAO,EAAE,QAAQ;QACjB,OAAO,EAAE,4CAA4C;QACrD,IAAI,EAAE,cAAc;QACpB,SAAS,EACP,6GAA6G;KAChH;IACD;QACE,IAAI,EAAE,eAAe;QACrB,QAAQ,EAAE,YAAY;QACtB,OAAO,EAAE,QAAQ;QACjB,OAAO,EAAE,2BAA2B;QACpC,IAAI,EAAE,eAAe;QACrB,SAAS,EACP,yHAAyH;KAC5H;IACD;QACE,IAAI,EAAE,iBAAiB;QACvB,QAAQ,EAAE,YAAY;QACtB,OAAO,EAAE,QAAQ;QACjB,OAAO,EAAE,qCAAqC;QAC9C,IAAI,EAAE,eAAe;QACrB,SAAS,EACP,6HAA6H;KAChI;IACD;QACE,IAAI,EAAE,iBAAiB;QACvB,QAAQ,EAAE,QAAQ;QAClB,OAAO,EAAE,QAAQ;QACjB,OAAO,EACL,oFAAoF;QACtF,IAAI,EAAE,oBAAoB;KAC3B;IACD;QACE,IAAI,EAAE,eAAe;QACrB,QAAQ,EAAE,YAAY;QACtB,OAAO,EAAE,QAAQ;QACjB,OAAO,EAAE,kCAAkC;QAC3C,IAAI,EAAE,oEAAoE;QAC1E,SAAS,EAAE,gEAAgE;KAC5E;IACD;QACE,IAAI,EAAE,gBAAgB;QACtB,QAAQ,EAAE,YAAY;QACtB,OAAO,EAAE,QAAQ;QACjB,OAAO,EAAE,kCAAkC;QAC3C,IAAI,EAAE,gCAAgC;QACtC,SAAS,EACP,qHAAqH;KACxH;IACD;QACE,IAAI,EAAE,QAAQ;QACd,QAAQ,EAAE,QAAQ;QAClB,OAAO,EAAE,UAAU;QACnB,OAAO,EAAE,0BAA0B;QACnC,IAAI,EAAE,WAAW;QACjB,SAAS,EACP,wGAAwG;KAC3G;IACD;QACE,IAAI,EAAE,WAAW;QACjB,QAAQ,EAAE,WAAW;QACrB,OAAO,EAAE,UAAU;QACnB,OAAO,EAAE,6BAA6B;QACtC,IAAI,EAAE,cAAc;QACpB,SAAS,EACP,yGAAyG;KAC5G;IACD;QACE,IAAI,EAAE,mBAAmB;QACzB,QAAQ,EAAE,OAAO;QACjB,OAAO,EAAE,UAAU;QACnB,OAAO,EAAE,gDAAgD;QACzD,IAAI,EAAE,wCAAwC;KAC/C;IACD;QACE,IAAI,EAAE,OAAO;QACb,QAAQ,EAAE,cAAc;QACxB,OAAO,EAAE,UAAU;QACnB,OAAO,EAAE,oDAAoD;QAC7D,IAAI,EAAE,+BAA+B;QACrC,SAAS,EAAE,0CAA0C;KACtD;IACD;QACE,IAAI,EAAE,oBAAoB;QAC1B,QAAQ,EAAE,MAAM;QAChB,OAAO,EAAE,UAAU;QACnB,OAAO,EAAE,0DAA0D;QACnE,IAAI,EAAE,sEAAsE;QAC5E,SAAS,EACP,2FAA2F;KAC9F;CACF,CAAC;AAEF,8GAA8G;AAC9G,MAAM,CAAC,MAAM,yBAAyB,GAAG,qBAAqB,CAAC;AAE/D,wFAAwF;AACxF,MAAM,aAAa,GAAG,IAAI,GAAG,CAAC,CAAC,QAAQ,EAAE,UAAU,EAAE,UAAU,EAAE,KAAK,CAAC,CAAC,CAAC;AAEzE,kFAAkF;AAClF,MAAM,YAAY,GAAG,CAAC,KAAa,EAAU,EAAE,CAC7C,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,WAAW,EAAE,IAAI,EAAE,CAAC;AASlD,qGAAqG;AACrG,MAAM,UAAU,GAAG,CAAC,IAAY,EAAE,EAAE;IAClC,MAAM,KAAK,GAAG,IAAI,GAAG,EAAU,CAAC;IAChC,MAAM,KAAK,GAAiB,EAAE,CAAC;IAC/B,IAAI,MAAM,GAAa,EAAE,CAAC;IAC1B,IAAI,UAAU,GAAG,KAAK,CAAC;IAEvB,KAAK,MAAM,GAAG,IAAI,IAAI,CAAC,KAAK,CAAC,aAAa,CAAC,EAAE,CAAC;QAC5C,MAAM,IAAI,GAAG,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;QAC9B,MAAM,IAAI,GAAG,CAAC,IAAI,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;QAC7D,MAAM,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;QAEhC,IAAI,KAAK,KAAK,CAAC,CAAC,EAAE,CAAC;YACjB,SAAS;QACX,CAAC;QAED,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;QACtD,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;QAE3C,IAAI,GAAG,KAAK,YAAY,EAAE,CAAC;YACzB,yFAAyF;YACzF,IAAI,CAAC,UAAU,EAAE,CAAC;gBAChB,MAAM,GAAG,EAAE,CAAC;YACd,CAAC;YAED,IAAI,KAAK,EAAE,CAAC;gBACV,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;gBACnB,KAAK,CAAC,GAAG,CAAC,YAAY,CAAC,KAAK,CAAC,CAAC,CAAC;YACjC,CAAC;YAED,UAAU,GAAG,IAAI,CAAC;YAClB,SAAS;QACX,CAAC;QAED,UAAU,GAAG,KAAK,CAAC;QACnB,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,GAAG,EAAE,KAAK,EAAE,CAAC,CAAC;IACrC,CAAC;IAED,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC;AAC1B,CAAC,CAAC;AAEF;;;GAGG;AACH,MAAM,CAAC,MAAM,cAAc,GAAG,CAAC,IAAY,EAAmB,EAAE,CAC9D,UAAU,CAAC,IAAI,CAAC;KACb,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,KAAK,gBAAgB,IAAI,CAAC,CAAC,KAAK,CAAC;KAC1D,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC;KACZ,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;IACT,IAAI,IAAI,GAAG,CAAC,CAAC,KAAK,CAAC;IACnB,IAAI,IAAI,GAAkB,IAAI,CAAC;IAE/B,IAAI,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;QACzB,MAAM,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QACjC,IAAI,GAAG,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC;QAClD,IAAI,GAAG,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC;IACnD,CAAC;IAED,MAAM,OAAO,GAA2B,EAAE,CAAC;IAE3C,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE,CAAC;QACnC,MAAM,EAAE,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;QAE7B,MAAM,IAAI,GAAG,CAAC,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;aAChD,IAAI,EAAE;aACN,WAAW,EAAE,CAAC;QAEjB,IAAI,IAAI,EAAE,CAAC;YACT,OAAO,CAAC,IAAI,CAAC;gBACX,EAAE,KAAK,CAAC,CAAC;oBACP,CAAC,CAAC,EAAE;oBACJ,CAAC,CAAC,IAAI;yBACD,KAAK,CAAC,EAAE,GAAG,CAAC,CAAC;yBACb,IAAI,EAAE;yBACN,WAAW,EAAE,CAAC;QACzB,CAAC;IACH,CAAC;IAED,OAAO,EAAE,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE,IAAI,EAAE,OAAO,EAAE,CAAC;AACjD,CAAC,CAAC,CAAC;AAEP,yGAAyG;AACzG,MAAM,CAAC,MAAM,cAAc,GAAG,CAAC,MAAqB,EAAY,EAAE,CAChE,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,EAAE,KAAK,CAAC,EAAE,EAAE;IACvD,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;QAC7B,OAAO,CAAC,GAAG,IAAI,IAAI,KAAK,EAAE,CAAC,CAAC;IAC9B,CAAC;IAED,OAAO,IAAI,KAAK,KAAK,IAAI,KAAK,KAAK,KAAK,IAAI,KAAK,KAAK,IAAI;QACxD,CAAC,CAAC,CAAC,GAAG,IAAI,IAAI,KAAK,EAAE,CAAC;QACtB,CAAC,CAAC,EAAE,CAAC;AACT,CAAC,CAAC,CAAC;AAEL;;;GAGG;AACH,MAAM,YAAY,GAAG,CAAC,MAAqB,EAA6B,EAAE;IACxE,IAAI,MAAM,CAAC,KAAK,EAAE,CAAC;QACjB,OAAO,OAAO,CAAC;IACjB,CAAC;IAED,IAAI,MAAM,CAAC,MAAM,KAAK,IAAI,IAAI,MAAM,CAAC,MAAM,IAAI,GAAG,EAAE,CAAC;QACnD,OAAO,aAAa,CAAC;IACvB,CAAC;IAED,OAAO,SAAS,CAAC;AACnB,CAAC,CAAC;AAEF,8GAA8G;AAC9G,MAAM,CAAC,MAAM,aAAa,GAAG,CAC3B,MAAqB,EACrB,IAAmB,EACnB,QAAgB,EAChB,QAAkB,EACD,EAAE;IACnB,MAAM,MAAM,GAAG,YAAY,CAAC,MAAM,CAAC,CAAC;IACpC,MAAM,KAAK,GAAG,MAAM,KAAK,OAAO,IAAI,IAAI,KAAK,IAAI,CAAC;IAClD,MAAM,MAAM,GAAG,KAAK,CAAC,CAAC,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;IAC/C,MAAM,MAAM,GAAG,KAAK,CAAC,CAAC,CAAC,YAAY,CAAC,MAAM,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;IAC7D,MAAM,IAAI,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC;IAErD,MAAM,IAAI,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,GAAG,EAAe,EAAE;QAC5C,MAAM,KAAK,GAAG,YAAY,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QACrC,IAAI,WAAW,GAA+B,MAAM,CAAC;QACrD,IAAI,KAAK,GAAG,GAAG,CAAC,IAAI,CAAC;QAErB,IAAI,MAAM,EAAE,KAAK,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC;YAC7B,WAAW,GAAG,OAAO,CAAC;QACxB,CAAC;aAAM,IAAI,KAAK,KAAK,UAAU,IAAI,MAAM,EAAE,KAAK,CAAC,GAAG,CAAC,WAAW,CAAC,EAAE,CAAC;YAClE,0EAA0E;YAC1E,WAAW,GAAG,WAAW,CAAC;YAC1B,KAAK,GAAG,WAAW,CAAC;QACtB,CAAC;aAAM,IAAI,MAAM,EAAE,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC;YAClC,WAAW,GAAG,UAAU,CAAC;QAC3B,CAAC;QAED,IAAI,MAAM,KAAK,aAAa,EAAE,CAAC;YAC7B,OAAO;gBACL,IAAI,EAAE,GAAG,CAAC,IAAI;gBACd,QAAQ,EAAE,GAAG,CAAC,QAAQ;gBACtB,OAAO,EAAE,GAAG,CAAC,OAAO;gBACpB,OAAO,EAAE,GAAG,CAAC,OAAO;gBACpB,WAAW;gBACX,OAAO,EAAE,KAAK;gBACd,YAAY,EAAE,IAAI,CAAC,MAAM;aAC1B,CAAC;QACJ,CAAC;QAED,OAAO;YACL,IAAI,EAAE,GAAG,CAAC,IAAI;YACd,QAAQ,EAAE,GAAG,CAAC,QAAQ;YACtB,OAAO,EAAE,GAAG,CAAC,OAAO;YACpB,OAAO,EAAE,GAAG,CAAC,OAAO;YACpB,WAAW;YACX,OAAO,EAAE,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI;YAChD,uFAAuF;YACvF,0FAA0F;YAC1F,YAAY,EAAE,MAAM;gBAClB,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC,MAAM;gBAChE,CAAC,CAAC,CAAC;SACN,CAAC;IACJ,CAAC,CAAC,CAAC;IAEH,IAAI,kBAAkB,GAA0C,MAAM,CAAC;IAEvE,2FAA2F;IAC3F,IAAI,MAAM,IAAI,MAAM,EAAE,KAAK,CAAC,GAAG,CAAC,yBAAyB,CAAC,EAAE,CAAC;QAC3D,kBAAkB,GAAG,MAAM,CAAC,QAAQ,EAAE,yBAAyB,CAAC;YAC9D,CAAC,CAAC,SAAS;YACX,CAAC,CAAC,YAAY,CAAC;IACnB,CAAC;IAED,OAAO;QACL,SAAS,EAAE,MAAM,CAAC,GAAG;QACrB,MAAM;QACN,UAAU,EAAE,MAAM,CAAC,MAAM;QACzB,kBAAkB;QAClB,cAAc,EAAE,KAAK,CAAC,CAAC,CAAC,cAAc,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,EAAE;QACjD,IAAI;KACL,CAAC;AACJ,CAAC,CAAC"}
@@ -0,0 +1,8 @@
1
+ import type { FetchOptions } from "../fetch.js";
2
+ import type { LlmsTxtInfo } from "../types.js";
3
+ /** Reads an llms.txt file against the format at llmstxt.org: an H1 title is the one required part. */
4
+ export declare const parseLlmsTxt: (url: string, httpStatus: number | null, raw: string) => LlmsTxtInfo;
5
+ /** Fetches /llms.txt from an origin. An HTML answer is a single-page app's fallback route: no file. */
6
+ export declare const loadLlmsTxt: (origin: string, options?: FetchOptions) => Promise<LlmsTxtInfo>;
7
+ /** Reads llms.txt from the root of a build output directory. */
8
+ export declare const readLlmsTxt: (dir: string, baseUrl: string) => Promise<LlmsTxtInfo>;
@@ -0,0 +1,88 @@
1
+ import { promises as fs } from "node:fs";
2
+ import path from "node:path";
3
+ import { fetchPage } from "../fetch.js";
4
+ /** An llms.txt is an index of links; a file this large is not one, and is read no further. */
5
+ const MAX_LLMS_TXT_BYTES = 1_000_000;
6
+ /**
7
+ * `[name](url)`. Each negated class also stops at the bracket that would start the next attempt,
8
+ * so a line of "[[[[" or "](](](" costs one pass. A URL with parentheses in it is not counted.
9
+ */
10
+ const MARKDOWN_LINK = /\[[^[\]\n]*\]\([^()\s]+\)/gu;
11
+ const missing = (url, httpStatus) => ({
12
+ url,
13
+ status: "missing",
14
+ httpStatus,
15
+ title: null,
16
+ summary: null,
17
+ sections: [],
18
+ links: 0,
19
+ bytes: 0,
20
+ problems: [],
21
+ });
22
+ /** Reads an llms.txt file against the format at llmstxt.org: an H1 title is the one required part. */
23
+ export const parseLlmsTxt = (url, httpStatus, raw) => {
24
+ const text = raw.replace(/^\uFEFF/u, "").slice(0, MAX_LLMS_TXT_BYTES);
25
+ const lines = text.split(/\r\n|\r|\n/u).map((l) => l.trim());
26
+ const firstIndex = lines.findIndex(Boolean);
27
+ const first = firstIndex === -1 ? "" : (lines[firstIndex] ?? "");
28
+ const title = /^#\s+\S/u.test(first) ? first.replace(/^#\s+/u, "") : null;
29
+ const afterTitle = lines.slice(firstIndex + 1).find(Boolean) ?? "";
30
+ const problems = [];
31
+ if (!first) {
32
+ problems.push("the file is empty");
33
+ }
34
+ else if (!title) {
35
+ problems.push(`it does not start with an H1 title ("# Site name"); the first line is "${first.slice(0, 80)}"`);
36
+ }
37
+ return {
38
+ url,
39
+ status: problems.length ? "invalid" : "found",
40
+ httpStatus,
41
+ title: title ? title.slice(0, 200) : null,
42
+ summary: title && afterTitle.startsWith(">")
43
+ ? afterTitle.replace(/^>\s*/u, "").slice(0, 300)
44
+ : null,
45
+ sections: lines
46
+ .filter((l) => /^##\s+\S/u.test(l))
47
+ .slice(0, 20)
48
+ .map((l) => l.replace(/^##\s+/u, "").slice(0, 120)),
49
+ links: [...text.matchAll(MARKDOWN_LINK)].length,
50
+ bytes: Buffer.byteLength(text, "utf-8"),
51
+ problems,
52
+ };
53
+ };
54
+ /** Fetches /llms.txt from an origin. An HTML answer is a single-page app's fallback route: no file. */
55
+ export const loadLlmsTxt = async (origin, options = {}) => {
56
+ const url = new URL("/llms.txt", origin).href;
57
+ const res = await fetchPage(url, {
58
+ ...options,
59
+ timeoutMs: Math.min(options.timeoutMs ?? 10_000, 10_000),
60
+ maxBodyBytes: MAX_LLMS_TXT_BYTES,
61
+ });
62
+ const contentType = res.contentType?.toLowerCase() ?? "";
63
+ const body = res.body ?? "";
64
+ if (res.status !== 200 ||
65
+ res.error ||
66
+ contentType.includes("text/html") ||
67
+ /^\s*</u.test(body.slice(0, 200))) {
68
+ return missing(url, res.status);
69
+ }
70
+ return parseLlmsTxt(url, res.status, body);
71
+ };
72
+ /** Reads llms.txt from the root of a build output directory. */
73
+ export const readLlmsTxt = async (dir, baseUrl) => {
74
+ const url = new URL("/llms.txt", baseUrl).href;
75
+ const file = await fs.open(path.join(dir, "llms.txt")).catch(() => null);
76
+ if (!file) {
77
+ return missing(url, 404);
78
+ }
79
+ try {
80
+ const buffer = Buffer.alloc(MAX_LLMS_TXT_BYTES);
81
+ const { bytesRead } = await file.read(buffer, 0, MAX_LLMS_TXT_BYTES, 0);
82
+ return parseLlmsTxt(url, 200, buffer.subarray(0, bytesRead).toString("utf-8"));
83
+ }
84
+ finally {
85
+ await file.close();
86
+ }
87
+ };
88
+ //# sourceMappingURL=llms-txt.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"llms-txt.js","sourceRoot":"","sources":["../../../src/core/aeo/llms-txt.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,IAAI,EAAE,EAAE,MAAM,SAAS,CAAC;AACzC,OAAO,IAAI,MAAM,WAAW,CAAC;AAE7B,OAAO,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AAIxC,8FAA8F;AAC9F,MAAM,kBAAkB,GAAG,SAAS,CAAC;AAErC;;;GAGG;AACH,MAAM,aAAa,GAAG,6BAA6B,CAAC;AAEpD,MAAM,OAAO,GAAG,CAAC,GAAW,EAAE,UAAyB,EAAe,EAAE,CAAC,CAAC;IACxE,GAAG;IACH,MAAM,EAAE,SAAS;IACjB,UAAU;IACV,KAAK,EAAE,IAAI;IACX,OAAO,EAAE,IAAI;IACb,QAAQ,EAAE,EAAE;IACZ,KAAK,EAAE,CAAC;IACR,KAAK,EAAE,CAAC;IACR,QAAQ,EAAE,EAAE;CACb,CAAC,CAAC;AAEH,sGAAsG;AACtG,MAAM,CAAC,MAAM,YAAY,GAAG,CAC1B,GAAW,EACX,UAAyB,EACzB,GAAW,EACE,EAAE;IACf,MAAM,IAAI,GAAG,GAAG,CAAC,OAAO,CAAC,UAAU,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,kBAAkB,CAAC,CAAC;IACtE,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;IAC7D,MAAM,UAAU,GAAG,KAAK,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC;IAC5C,MAAM,KAAK,GAAG,UAAU,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,UAAU,CAAC,IAAI,EAAE,CAAC,CAAC;IACjE,MAAM,KAAK,GAAG,UAAU,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;IAC1E,MAAM,UAAU,GAAG,KAAK,CAAC,KAAK,CAAC,UAAU,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC;IACnE,MAAM,QAAQ,GAAa,EAAE,CAAC;IAE9B,IAAI,CAAC,KAAK,EAAE,CAAC;QACX,QAAQ,CAAC,IAAI,CAAC,mBAAmB,CAAC,CAAC;IACrC,CAAC;SAAM,IAAI,CAAC,KAAK,EAAE,CAAC;QAClB,QAAQ,CAAC,IAAI,CACX,0EAA0E,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAChG,CAAC;IACJ,CAAC;IAED,OAAO;QACL,GAAG;QACH,MAAM,EAAE,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,OAAO;QAC7C,UAAU;QACV,KAAK,EAAE,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI;QACzC,OAAO,EACL,KAAK,IAAI,UAAU,CAAC,UAAU,CAAC,GAAG,CAAC;YACjC,CAAC,CAAC,UAAU,CAAC,OAAO,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC;YAChD,CAAC,CAAC,IAAI;QACV,QAAQ,EAAE,KAAK;aACZ,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;aAClC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC;aACZ,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;QACrD,KAAK,EAAE,CAAC,GAAG,IAAI,CAAC,QAAQ,CAAC,aAAa,CAAC,CAAC,CAAC,MAAM;QAC/C,KAAK,EAAE,MAAM,CAAC,UAAU,CAAC,IAAI,EAAE,OAAO,CAAC;QACvC,QAAQ;KACT,CAAC;AACJ,CAAC,CAAC;AAEF,uGAAuG;AACvG,MAAM,CAAC,MAAM,WAAW,GAAG,KAAK,EAC9B,MAAc,EACd,OAAO,GAAiB,EAAE,EACJ,EAAE;IACxB,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,WAAW,EAAE,MAAM,CAAC,CAAC,IAAI,CAAC;IAE9C,MAAM,GAAG,GAAG,MAAM,SAAS,CAAC,GAAG,EAAE;QAC/B,GAAG,OAAO;QACV,SAAS,EAAE,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,SAAS,IAAI,MAAM,EAAE,MAAM,CAAC;QACxD,YAAY,EAAE,kBAAkB;KACjC,CAAC,CAAC;IAEH,MAAM,WAAW,GAAG,GAAG,CAAC,WAAW,EAAE,WAAW,EAAE,IAAI,EAAE,CAAC;IACzD,MAAM,IAAI,GAAG,GAAG,CAAC,IAAI,IAAI,EAAE,CAAC;IAE5B,IACE,GAAG,CAAC,MAAM,KAAK,GAAG;QAClB,GAAG,CAAC,KAAK;QACT,WAAW,CAAC,QAAQ,CAAC,WAAW,CAAC;QACjC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,EACjC,CAAC;QACD,OAAO,OAAO,CAAC,GAAG,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC;IAClC,CAAC;IAED,OAAO,YAAY,CAAC,GAAG,EAAE,GAAG,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;AAC7C,CAAC,CAAC;AAEF,gEAAgE;AAChE,MAAM,CAAC,MAAM,WAAW,GAAG,KAAK,EAC9B,GAAW,EACX,OAAe,EACO,EAAE;IACxB,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,WAAW,EAAE,OAAO,CAAC,CAAC,IAAI,CAAC;IAC/C,MAAM,IAAI,GAAG,MAAM,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,UAAU,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,CAAC;IAEzE,IAAI,CAAC,IAAI,EAAE,CAAC;QACV,OAAO,OAAO,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC;IAC3B,CAAC;IAED,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,kBAAkB,CAAC,CAAC;QAChD,MAAM,EAAE,SAAS,EAAE,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE,kBAAkB,EAAE,CAAC,CAAC,CAAC;QAExE,OAAO,YAAY,CACjB,GAAG,EACH,GAAG,EACH,MAAM,CAAC,QAAQ,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,CAChD,CAAC;IACJ,CAAC;YAAS,CAAC;QACT,MAAM,IAAI,CAAC,KAAK,EAAE,CAAC;IACrB,CAAC;AACH,CAAC,CAAC"}
@@ -0,0 +1,53 @@
1
+ import type { JsonObject, JsonValue } from "../html/structured-data.js";
2
+ import type { AeoPageSummary, ExtractedPage, ResponseInfo, StructuredDataItem } from "../types.js";
3
+ export type Engine = "google" | "bing";
4
+ export interface ScopedDirective {
5
+ /** The user agent the directive is for, or null when it is for every crawler. */
6
+ agent: string | null;
7
+ directive: string;
8
+ }
9
+ /**
10
+ * X-Robots-Tag values. "googlebot: nosnippet, noarchive" scopes every directive after the prefix
11
+ * to googlebot, up to the next prefix.
12
+ */
13
+ export declare const parseRobotsHeader: (value: string) => ScopedDirective[];
14
+ /** The robots directives one engine applies to the page, from its meta tags and X-Robots-Tag. */
15
+ export declare const directivesFor: (engine: Engine, page: ExtractedPage, response?: ResponseInfo) => string[];
16
+ /** The `max-snippet` limit, or null when none is set (-1 means no limit). */
17
+ export declare const maxSnippet: (directives: string[]) => number | null;
18
+ export declare const hasNoindex: (directives: string[]) => boolean;
19
+ export declare const hasNosnippet: (directives: string[]) => boolean;
20
+ export interface TypedNode {
21
+ types: string[];
22
+ node: JsonObject;
23
+ /** True for the item itself, false for an object nested in it (publisher, author, mainEntity). */
24
+ topLevel: boolean;
25
+ }
26
+ /** Every typed object in the page's structured data, nested ones included. */
27
+ export declare const typedNodes: (items: StructuredDataItem[]) => TypedNode[];
28
+ export declare const isArticleType: (type: string) => boolean;
29
+ /**
30
+ * Whether a typed object describes an organization or a person. LocalBusiness alone has over a
31
+ * hundred subtypes (Dentist, Bakery, ...), so an object with a name and a logo, address or phone
32
+ * number counts too.
33
+ */
34
+ export declare const isEntity: ({ types, node }: TypedNode) => boolean;
35
+ /** The text of a name-like value: a string, or the `name` of an object, or of the first in a list. */
36
+ export declare const nameOf: (value: JsonValue | undefined) => string | null;
37
+ export declare const articles: (page: ExtractedPage) => TypedNode[];
38
+ /** Lower-cased letters and digits only, so punctuation and spacing differences do not count. */
39
+ export declare const comparable: (value: string) => string;
40
+ /**
41
+ * Structured-data text may hold HTML: tags, and entities such as &amp;. `[^<>]` stops each attempt
42
+ * at the next bracket, so a run of "<" without ">" costs one pass, not one pass per bracket.
43
+ */
44
+ export declare const stripTags: (value: string) => string;
45
+ export interface MarkedUpText {
46
+ type: "Question" | "Answer" | "HowToStep";
47
+ text: string;
48
+ }
49
+ /** The questions, answers and steps FAQPage, QAPage and HowTo markup claims the page shows. */
50
+ export declare const markedUpTexts: (page: ExtractedPage) => MarkedUpText[];
51
+ /** Whether the URL is a site's home page (also /index.html in build output). */
52
+ export declare const isHomePage: (url: string) => boolean;
53
+ export declare const aeoPageSummary: (page: ExtractedPage, response?: ResponseInfo) => AeoPageSummary;
@@ -0,0 +1,312 @@
1
+ import { isJsonObject, isString, typesOf } from "../html/structured-data.js";
2
+ import { robotsDirectives } from "../rules/indexing.js";
3
+ import { normalizePath } from "../url.js";
4
+ /** Meta tags each engine reads: the generic one and its own. */
5
+ const META_NAMES = {
6
+ google: ["robots", "googlebot"],
7
+ bing: ["robots", "bingbot"],
8
+ };
9
+ const HEADER_AGENT = {
10
+ google: "googlebot",
11
+ bing: "bingbot",
12
+ };
13
+ /** Directives that carry a value after a colon, which is not a user agent prefix. */
14
+ const VALUE_DIRECTIVES = new Set([
15
+ "max-snippet",
16
+ "max-image-preview",
17
+ "max-video-preview",
18
+ "unavailable_after",
19
+ ]);
20
+ /**
21
+ * X-Robots-Tag values. "googlebot: nosnippet, noarchive" scopes every directive after the prefix
22
+ * to googlebot, up to the next prefix.
23
+ */
24
+ export const parseRobotsHeader = (value) => {
25
+ const out = [];
26
+ let agent = null;
27
+ for (const piece of value.toLowerCase().split(",")) {
28
+ let directive = piece.trim();
29
+ const match = /^(?<name>[a-z0-9_-]+)\s*:\s*(?<rest>.*)$/u.exec(directive);
30
+ const name = match?.groups?.name;
31
+ if (name && !VALUE_DIRECTIVES.has(name)) {
32
+ agent = name;
33
+ directive = match.groups?.rest?.trim() ?? "";
34
+ }
35
+ if (directive) {
36
+ out.push({ agent, directive: directive.replaceAll(/\s+/gu, "") });
37
+ }
38
+ }
39
+ return out;
40
+ };
41
+ /** The robots directives one engine applies to the page, from its meta tags and X-Robots-Tag. */
42
+ export const directivesFor = (engine, page, response) => {
43
+ const out = [];
44
+ for (const meta of page.answer.robotsMeta) {
45
+ if (META_NAMES[engine].includes(meta.name)) {
46
+ for (const d of meta.content.split(",")) {
47
+ const directive = d.replaceAll(/\s+/gu, "");
48
+ if (directive) {
49
+ out.push(directive);
50
+ }
51
+ }
52
+ }
53
+ }
54
+ const header = response?.headers["x-robots-tag"];
55
+ if (header) {
56
+ for (const d of parseRobotsHeader(header)) {
57
+ if (d.agent === null || d.agent === HEADER_AGENT[engine]) {
58
+ out.push(d.directive);
59
+ }
60
+ }
61
+ }
62
+ return [...new Set(out)];
63
+ };
64
+ /** The `max-snippet` limit, or null when none is set (-1 means no limit). */
65
+ export const maxSnippet = (directives) => {
66
+ for (const d of directives) {
67
+ if (d.startsWith("max-snippet:")) {
68
+ const n = Number(d.slice("max-snippet:".length));
69
+ if (Number.isInteger(n)) {
70
+ return n;
71
+ }
72
+ }
73
+ }
74
+ return null;
75
+ };
76
+ export const hasNoindex = (directives) => directives.includes("noindex") || directives.includes("none");
77
+ export const hasNosnippet = (directives) => directives.includes("nosnippet") || maxSnippet(directives) === 0;
78
+ /** Every typed object in the page's structured data, nested ones included. */
79
+ export const typedNodes = (items) => {
80
+ const out = [];
81
+ const seen = new Set();
82
+ const walk = (value, depth, top) => {
83
+ if (depth > 6 || out.length >= 500 || value === undefined) {
84
+ return;
85
+ }
86
+ if (Array.isArray(value)) {
87
+ for (const v of value) {
88
+ walk(v, depth, top);
89
+ }
90
+ return;
91
+ }
92
+ if (!isJsonObject(value) || seen.has(value)) {
93
+ return;
94
+ }
95
+ seen.add(value);
96
+ const types = typesOf(value["@type"]);
97
+ if (types.length) {
98
+ out.push({ types, node: value, topLevel: top });
99
+ }
100
+ for (const [key, v] of Object.entries(value)) {
101
+ if (!key.startsWith("@")) {
102
+ walk(v, depth + 1, false);
103
+ }
104
+ }
105
+ };
106
+ for (const item of items) {
107
+ // SAFETY: extractStructuredData builds every item's data from JSON.parse or from microdata, and both hold JSON values alone.
108
+ walk(item.data, 0, true);
109
+ }
110
+ return out;
111
+ };
112
+ export const isArticleType = (type) => type.endsWith("Article") ||
113
+ type === "BlogPosting" ||
114
+ type === "LiveBlogPosting" ||
115
+ type === "Report";
116
+ /** schema.org types for an organization or a person, beyond the many that end in Organization or Business. */
117
+ const ENTITY_TYPES = new Set([
118
+ "Organization",
119
+ "Corporation",
120
+ "NGO",
121
+ "Airline",
122
+ "Consortium",
123
+ "Project",
124
+ "WorkersUnion",
125
+ "LibrarySystem",
126
+ "FundingScheme",
127
+ "LocalBusiness",
128
+ "Store",
129
+ "OnlineStore",
130
+ "Restaurant",
131
+ "Hotel",
132
+ "PerformingGroup",
133
+ "Person",
134
+ ]);
135
+ /**
136
+ * Whether a typed object describes an organization or a person. LocalBusiness alone has over a
137
+ * hundred subtypes (Dentist, Bakery, ...), so an object with a name and a logo, address or phone
138
+ * number counts too.
139
+ */
140
+ export const isEntity = ({ types, node }) => types.some((t) => ENTITY_TYPES.has(t) || /(?:Organization|Business|Store)$/u.test(t)) ||
141
+ (isString(node["name"]) &&
142
+ (node["logo"] !== undefined ||
143
+ node["address"] !== undefined ||
144
+ node["telephone"] !== undefined));
145
+ /** The text of a name-like value: a string, or the `name` of an object, or of the first in a list. */
146
+ export const nameOf = (value) => {
147
+ if (isString(value)) {
148
+ return value.trim() || null;
149
+ }
150
+ if (Array.isArray(value)) {
151
+ for (const v of value) {
152
+ const name = nameOf(v);
153
+ if (name) {
154
+ return name;
155
+ }
156
+ }
157
+ return null;
158
+ }
159
+ if (isJsonObject(value)) {
160
+ const { name } = value;
161
+ return isString(name) ? name.trim() || null : null;
162
+ }
163
+ return null;
164
+ };
165
+ const stringOf = (value) => isString(value) && value.trim() ? value.trim() : null;
166
+ export const articles = (page) => typedNodes(page.structuredData).filter((n) => n.types.some(isArticleType));
167
+ /** Lower-cased letters and digits only, so punctuation and spacing differences do not count. */
168
+ export const comparable = (value) => value
169
+ .toLowerCase()
170
+ .replaceAll(/[^\p{L}\p{N}]+/gu, " ")
171
+ .trim();
172
+ const NAMED_ENTITIES = new Map([
173
+ ["amp", "&"],
174
+ ["lt", "<"],
175
+ ["gt", ">"],
176
+ ["quot", '"'],
177
+ ["apos", "'"],
178
+ ["nbsp", " "],
179
+ ["rsquo", "’"],
180
+ ["lsquo", "‘"],
181
+ ["rdquo", "”"],
182
+ ["ldquo", "“"],
183
+ ["ndash", "–"],
184
+ ["mdash", "—"],
185
+ ]);
186
+ const decodeEntity = (entity, body) => {
187
+ if (body.startsWith("#")) {
188
+ const code = body[1] === "x" || body[1] === "X"
189
+ ? Number.parseInt(body.slice(2), 16)
190
+ : Number(body.slice(1));
191
+ return Number.isInteger(code) && code > 0 && code <= 0x10_ff_ff
192
+ ? String.fromCodePoint(code)
193
+ : entity;
194
+ }
195
+ return NAMED_ENTITIES.get(body.toLowerCase()) ?? entity;
196
+ };
197
+ /**
198
+ * Structured-data text may hold HTML: tags, and entities such as &amp;. `[^<>]` stops each attempt
199
+ * at the next bracket, so a run of "<" without ">" costs one pass, not one pass per bracket.
200
+ */
201
+ export const stripTags = (value) => value
202
+ .replaceAll(/<[^<>]*>/gu, " ")
203
+ .replaceAll(/&(?<body>#\d{1,7}|#[xX][\da-fA-F]{1,6}|[a-zA-Z]{2,8});/gu, (entity, body) => decodeEntity(entity, body));
204
+ const textsOf = (value) => {
205
+ if (value === undefined) {
206
+ return [];
207
+ }
208
+ return Array.isArray(value) ? value : [value];
209
+ };
210
+ const pushStep = (out, step) => {
211
+ if (isString(step)) {
212
+ out.push({ type: "HowToStep", text: step });
213
+ return;
214
+ }
215
+ if (!isJsonObject(step)) {
216
+ return;
217
+ }
218
+ // A HowToSection holds its steps in itemListElement.
219
+ const nested = step["itemListElement"];
220
+ if (nested !== undefined) {
221
+ for (const s of textsOf(nested)) {
222
+ pushStep(out, s);
223
+ }
224
+ return;
225
+ }
226
+ const text = stringOf(step["text"]) ?? stringOf(step["name"]);
227
+ if (text) {
228
+ out.push({ type: "HowToStep", text });
229
+ }
230
+ };
231
+ /** The questions, answers and steps FAQPage, QAPage and HowTo markup claims the page shows. */
232
+ export const markedUpTexts = (page) => {
233
+ const out = [];
234
+ for (const { types, node } of typedNodes(page.structuredData)) {
235
+ if (types.includes("FAQPage") || types.includes("QAPage")) {
236
+ for (const q of textsOf(node["mainEntity"])) {
237
+ if (!isJsonObject(q)) {
238
+ continue;
239
+ }
240
+ const question = stringOf(q["name"]) ?? stringOf(q["text"]);
241
+ if (question) {
242
+ out.push({ type: "Question", text: question });
243
+ }
244
+ for (const a of [
245
+ ...textsOf(q["acceptedAnswer"]),
246
+ ...textsOf(q["suggestedAnswer"]),
247
+ ]) {
248
+ const answer = isJsonObject(a) ? stringOf(a["text"]) : null;
249
+ if (answer) {
250
+ out.push({ type: "Answer", text: answer });
251
+ }
252
+ }
253
+ }
254
+ }
255
+ if (types.includes("HowTo")) {
256
+ for (const step of textsOf(node["step"])) {
257
+ pushStep(out, step);
258
+ }
259
+ }
260
+ }
261
+ return out.slice(0, 100);
262
+ };
263
+ /** Whether the URL is a site's home page (also /index.html in build output). */
264
+ export const isHomePage = (url) => {
265
+ try {
266
+ return normalizePath(new URL(url).pathname) === "/";
267
+ }
268
+ catch {
269
+ return false;
270
+ }
271
+ };
272
+ /* ---------------------------------------------------------------------------
273
+ * Summary
274
+ * ------------------------------------------------------------------------- */
275
+ const googleSnippets = (page, directives) => {
276
+ if (hasNosnippet(directives)) {
277
+ return "blocked";
278
+ }
279
+ const limit = maxSnippet(directives);
280
+ if ((limit !== null && limit > 0 && limit < 100) ||
281
+ page.answer.nosnippetWords > 0) {
282
+ return "limited";
283
+ }
284
+ return "allowed";
285
+ };
286
+ const copilotAnswers = (directives) => {
287
+ // Bing: NOCACHE wins when a page has both tags.
288
+ if (directives.includes("nocache")) {
289
+ return hasNoindex(directives) ? "excluded" : "snippet-only";
290
+ }
291
+ return hasNoindex(directives) || directives.includes("noarchive")
292
+ ? "excluded"
293
+ : "allowed";
294
+ };
295
+ export const aeoPageSummary = (page, response) => {
296
+ const article = articles(page)[0]?.node;
297
+ return {
298
+ indexable: !robotsDirectives(page, response).noindex &&
299
+ (response?.status ?? 200) < 400,
300
+ googleSnippets: googleSnippets(page, directivesFor("google", page, response)),
301
+ copilotAnswers: copilotAnswers(directivesFor("bing", page, response)),
302
+ wordCount: page.wordCount,
303
+ questions: page.answer.questions.length,
304
+ structuredDataTypes: [
305
+ ...new Set(page.structuredData.flatMap((i) => i.types)),
306
+ ],
307
+ author: article ? nameOf(article["author"]) : null,
308
+ datePublished: article ? stringOf(article["datePublished"]) : null,
309
+ dateModified: article ? stringOf(article["dateModified"]) : null,
310
+ };
311
+ };
312
+ //# sourceMappingURL=page.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"page.js","sourceRoot":"","sources":["../../../src/core/aeo/page.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,4BAA4B,CAAC;AAE7E,OAAO,EAAE,gBAAgB,EAAE,MAAM,sBAAsB,CAAC;AAOxD,OAAO,EAAE,aAAa,EAAE,MAAM,WAAW,CAAC;AAQ1C,gEAAgE;AAChE,MAAM,UAAU,GAA6B;IAC3C,MAAM,EAAE,CAAC,QAAQ,EAAE,WAAW,CAAC;IAC/B,IAAI,EAAE,CAAC,QAAQ,EAAE,SAAS,CAAC;CAC5B,CAAC;AAEF,MAAM,YAAY,GAA2B;IAC3C,MAAM,EAAE,WAAW;IACnB,IAAI,EAAE,SAAS;CAChB,CAAC;AAEF,qFAAqF;AACrF,MAAM,gBAAgB,GAAG,IAAI,GAAG,CAAC;IAC/B,aAAa;IACb,mBAAmB;IACnB,mBAAmB;IACnB,mBAAmB;CACpB,CAAC,CAAC;AAQH;;;GAGG;AACH,MAAM,CAAC,MAAM,iBAAiB,GAAG,CAAC,KAAa,EAAqB,EAAE;IACpE,MAAM,GAAG,GAAsB,EAAE,CAAC;IAClC,IAAI,KAAK,GAAkB,IAAI,CAAC;IAEhC,KAAK,MAAM,KAAK,IAAI,KAAK,CAAC,WAAW,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE,CAAC;QACnD,IAAI,SAAS,GAAG,KAAK,CAAC,IAAI,EAAE,CAAC;QAC7B,MAAM,KAAK,GAAG,2CAA2C,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QAC1E,MAAM,IAAI,GAAG,KAAK,EAAE,MAAM,EAAE,IAAI,CAAC;QAEjC,IAAI,IAAI,IAAI,CAAC,gBAAgB,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;YACxC,KAAK,GAAG,IAAI,CAAC;YACb,SAAS,GAAG,KAAK,CAAC,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;QAC/C,CAAC;QAED,IAAI,SAAS,EAAE,CAAC;YACd,GAAG,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,SAAS,EAAE,SAAS,CAAC,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC,EAAE,CAAC,CAAC;QACpE,CAAC;IACH,CAAC;IAED,OAAO,GAAG,CAAC;AACb,CAAC,CAAC;AAEF,iGAAiG;AACjG,MAAM,CAAC,MAAM,aAAa,GAAG,CAC3B,MAAc,EACd,IAAmB,EACnB,QAAuB,EACb,EAAE;IACZ,MAAM,GAAG,GAAa,EAAE,CAAC;IAEzB,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,MAAM,CAAC,UAAU,EAAE,CAAC;QAC1C,IAAI,UAAU,CAAC,MAAM,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;YAC3C,KAAK,MAAM,CAAC,IAAI,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE,CAAC;gBACxC,MAAM,SAAS,GAAG,CAAC,CAAC,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC;gBAE5C,IAAI,SAAS,EAAE,CAAC;oBACd,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;gBACtB,CAAC;YACH,CAAC;QACH,CAAC;IACH,CAAC;IAED,MAAM,MAAM,GAAG,QAAQ,EAAE,OAAO,CAAC,cAAc,CAAC,CAAC;IAEjD,IAAI,MAAM,EAAE,CAAC;QACX,KAAK,MAAM,CAAC,IAAI,iBAAiB,CAAC,MAAM,CAAC,EAAE,CAAC;YAC1C,IAAI,CAAC,CAAC,KAAK,KAAK,IAAI,IAAI,CAAC,CAAC,KAAK,KAAK,YAAY,CAAC,MAAM,CAAC,EAAE,CAAC;gBACzD,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;YACxB,CAAC;QACH,CAAC;IACH,CAAC;IAED,OAAO,CAAC,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC;AAC3B,CAAC,CAAC;AAEF,6EAA6E;AAC7E,MAAM,CAAC,MAAM,UAAU,GAAG,CAAC,UAAoB,EAAiB,EAAE;IAChE,KAAK,MAAM,CAAC,IAAI,UAAU,EAAE,CAAC;QAC3B,IAAI,CAAC,CAAC,UAAU,CAAC,cAAc,CAAC,EAAE,CAAC;YACjC,MAAM,CAAC,GAAG,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,cAAc,CAAC,MAAM,CAAC,CAAC,CAAC;YAEjD,IAAI,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC;gBACxB,OAAO,CAAC,CAAC;YACX,CAAC;QACH,CAAC;IACH,CAAC;IAED,OAAO,IAAI,CAAC;AACd,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,UAAU,GAAG,CAAC,UAAoB,EAAW,EAAE,CAC1D,UAAU,CAAC,QAAQ,CAAC,SAAS,CAAC,IAAI,UAAU,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;AAEhE,MAAM,CAAC,MAAM,YAAY,GAAG,CAAC,UAAoB,EAAW,EAAE,CAC5D,UAAU,CAAC,QAAQ,CAAC,WAAW,CAAC,IAAI,UAAU,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC;AAanE,8EAA8E;AAC9E,MAAM,CAAC,MAAM,UAAU,GAAG,CAAC,KAA2B,EAAe,EAAE;IACrE,MAAM,GAAG,GAAgB,EAAE,CAAC;IAC5B,MAAM,IAAI,GAAG,IAAI,GAAG,EAAa,CAAC;IAElC,MAAM,IAAI,GAAG,CAAC,KAA4B,EAAE,KAAa,EAAE,GAAY,EAAE,EAAE;QACzE,IAAI,KAAK,GAAG,CAAC,IAAI,GAAG,CAAC,MAAM,IAAI,GAAG,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;YAC1D,OAAO;QACT,CAAC;QAED,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;YACzB,KAAK,MAAM,CAAC,IAAI,KAAK,EAAE,CAAC;gBACtB,IAAI,CAAC,CAAC,EAAE,KAAK,EAAE,GAAG,CAAC,CAAC;YACtB,CAAC;YAED,OAAO;QACT,CAAC;QAED,IAAI,CAAC,YAAY,CAAC,KAAK,CAAC,IAAI,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC;YAC5C,OAAO;QACT,CAAC;QAED,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;QAChB,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC;QAEtC,IAAI,KAAK,CAAC,MAAM,EAAE,CAAC;YACjB,GAAG,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,IAAI,EAAE,KAAK,EAAE,QAAQ,EAAE,GAAG,EAAE,CAAC,CAAC;QAClD,CAAC;QAED,KAAK,MAAM,CAAC,GAAG,EAAE,CAAC,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;YAC7C,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;gBACzB,IAAI,CAAC,CAAC,EAAE,KAAK,GAAG,CAAC,EAAE,KAAK,CAAC,CAAC;YAC5B,CAAC;QACH,CAAC;IACH,CAAC,CAAC;IAEF,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,6HAA6H;QAC7H,IAAI,CAAC,IAAI,CAAC,IAAkB,EAAE,CAAC,EAAE,IAAI,CAAC,CAAC;IACzC,CAAC;IAED,OAAO,GAAG,CAAC;AACb,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,aAAa,GAAG,CAAC,IAAY,EAAW,EAAE,CACrD,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC;IACxB,IAAI,KAAK,aAAa;IACtB,IAAI,KAAK,iBAAiB;IAC1B,IAAI,KAAK,QAAQ,CAAC;AAEpB,8GAA8G;AAC9G,MAAM,YAAY,GAAG,IAAI,GAAG,CAAC;IAC3B,cAAc;IACd,aAAa;IACb,KAAK;IACL,SAAS;IACT,YAAY;IACZ,SAAS;IACT,cAAc;IACd,eAAe;IACf,eAAe;IACf,eAAe;IACf,OAAO;IACP,aAAa;IACb,YAAY;IACZ,OAAO;IACP,iBAAiB;IACjB,QAAQ;CACT,CAAC,CAAC;AAEH;;;;GAIG;AACH,MAAM,CAAC,MAAM,QAAQ,GAAG,CAAC,EAAE,KAAK,EAAE,IAAI,EAAa,EAAW,EAAE,CAC9D,KAAK,CAAC,IAAI,CACR,CAAC,CAAC,EAAE,EAAE,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,mCAAmC,CAAC,IAAI,CAAC,CAAC,CAAC,CAC1E;IACD,CAAC,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QACrB,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,SAAS;YACzB,IAAI,CAAC,SAAS,CAAC,KAAK,SAAS;YAC7B,IAAI,CAAC,WAAW,CAAC,KAAK,SAAS,CAAC,CAAC,CAAC;AAExC,sGAAsG;AACtG,MAAM,CAAC,MAAM,MAAM,GAAG,CAAC,KAA4B,EAAiB,EAAE;IACpE,IAAI,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;QACpB,OAAO,KAAK,CAAC,IAAI,EAAE,IAAI,IAAI,CAAC;IAC9B,CAAC;IAED,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;QACzB,KAAK,MAAM,CAAC,IAAI,KAAK,EAAE,CAAC;YACtB,MAAM,IAAI,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;YAEvB,IAAI,IAAI,EAAE,CAAC;gBACT,OAAO,IAAI,CAAC;YACd,CAAC;QACH,CAAC;QAED,OAAO,IAAI,CAAC;IACd,CAAC;IAED,IAAI,YAAY,CAAC,KAAK,CAAC,EAAE,CAAC;QACxB,MAAM,EAAE,IAAI,EAAE,GAAG,KAAK,CAAC;QAEvB,OAAO,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,EAAE,IAAI,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC;IACrD,CAAC;IAED,OAAO,IAAI,CAAC;AACd,CAAC,CAAC;AAEF,MAAM,QAAQ,GAAG,CAAC,KAA4B,EAAiB,EAAE,CAC/D,QAAQ,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;AAExD,MAAM,CAAC,MAAM,QAAQ,GAAG,CAAC,IAAmB,EAAe,EAAE,CAC3D,UAAU,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC,CAAC;AAE7E,gGAAgG;AAChG,MAAM,CAAC,MAAM,UAAU,GAAG,CAAC,KAAa,EAAU,EAAE,CAClD,KAAK;KACF,WAAW,EAAE;KACb,UAAU,CAAC,kBAAkB,EAAE,GAAG,CAAC;KACnC,IAAI,EAAE,CAAC;AAEZ,MAAM,cAAc,GAAgC,IAAI,GAAG,CAAC;IAC1D,CAAC,KAAK,EAAE,GAAG,CAAC;IACZ,CAAC,IAAI,EAAE,GAAG,CAAC;IACX,CAAC,IAAI,EAAE,GAAG,CAAC;IACX,CAAC,MAAM,EAAE,GAAG,CAAC;IACb,CAAC,MAAM,EAAE,GAAG,CAAC;IACb,CAAC,MAAM,EAAE,GAAG,CAAC;IACb,CAAC,OAAO,EAAE,GAAG,CAAC;IACd,CAAC,OAAO,EAAE,GAAG,CAAC;IACd,CAAC,OAAO,EAAE,GAAG,CAAC;IACd,CAAC,OAAO,EAAE,GAAG,CAAC;IACd,CAAC,OAAO,EAAE,GAAG,CAAC;IACd,CAAC,OAAO,EAAE,GAAG,CAAC;CACf,CAAC,CAAC;AAEH,MAAM,YAAY,GAAG,CAAC,MAAc,EAAE,IAAY,EAAU,EAAE;IAC5D,IAAI,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;QACzB,MAAM,IAAI,GACR,IAAI,CAAC,CAAC,CAAC,KAAK,GAAG,IAAI,IAAI,CAAC,CAAC,CAAC,KAAK,GAAG;YAChC,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC;YACpC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;QAE5B,OAAO,MAAM,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,IAAI,GAAG,CAAC,IAAI,IAAI,IAAI,UAAU;YAC7D,CAAC,CAAC,MAAM,CAAC,aAAa,CAAC,IAAI,CAAC;YAC5B,CAAC,CAAC,MAAM,CAAC;IACb,CAAC;IAED,OAAO,cAAc,CAAC,GAAG,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC,IAAI,MAAM,CAAC;AAC1D,CAAC,CAAC;AAEF;;;GAGG;AACH,MAAM,CAAC,MAAM,SAAS,GAAG,CAAC,KAAa,EAAU,EAAE,CACjD,KAAK;KACF,UAAU,CAAC,YAAY,EAAE,GAAG,CAAC;KAC7B,UAAU,CACT,0DAA0D,EAC1D,CAAC,MAAM,EAAE,IAAY,EAAE,EAAE,CAAC,YAAY,CAAC,MAAM,EAAE,IAAI,CAAC,CACrD,CAAC;AAON,MAAM,OAAO,GAAG,CAAC,KAA4B,EAAe,EAAE;IAC5D,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;QACxB,OAAO,EAAE,CAAC;IACZ,CAAC;IAED,OAAO,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;AAChD,CAAC,CAAC;AAEF,MAAM,QAAQ,GAAG,CAAC,GAAmB,EAAE,IAAe,EAAQ,EAAE;IAC9D,IAAI,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;QACnB,GAAG,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,WAAW,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC;QAE5C,OAAO;IACT,CAAC;IAED,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,EAAE,CAAC;QACxB,OAAO;IACT,CAAC;IAED,qDAAqD;IACrD,MAAM,MAAM,GAAG,IAAI,CAAC,iBAAiB,CAAC,CAAC;IAEvC,IAAI,MAAM,KAAK,SAAS,EAAE,CAAC;QACzB,KAAK,MAAM,CAAC,IAAI,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;YAChC,QAAQ,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC;QACnB,CAAC;QAED,OAAO;IACT,CAAC;IAED,MAAM,IAAI,GAAG,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,IAAI,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC;IAE9D,IAAI,IAAI,EAAE,CAAC;QACT,GAAG,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,WAAW,EAAE,IAAI,EAAE,CAAC,CAAC;IACxC,CAAC;AACH,CAAC,CAAC;AAEF,+FAA+F;AAC/F,MAAM,CAAC,MAAM,aAAa,GAAG,CAAC,IAAmB,EAAkB,EAAE;IACnE,MAAM,GAAG,GAAmB,EAAE,CAAC;IAE/B,KAAK,MAAM,EAAE,KAAK,EAAE,IAAI,EAAE,IAAI,UAAU,CAAC,IAAI,CAAC,cAAc,CAAC,EAAE,CAAC;QAC9D,IAAI,KAAK,CAAC,QAAQ,CAAC,SAAS,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE,CAAC;YAC1D,KAAK,MAAM,CAAC,IAAI,OAAO,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC,EAAE,CAAC;gBAC5C,IAAI,CAAC,YAAY,CAAC,CAAC,CAAC,EAAE,CAAC;oBACrB,SAAS;gBACX,CAAC;gBAED,MAAM,QAAQ,GAAG,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC;gBAE5D,IAAI,QAAQ,EAAE,CAAC;oBACb,GAAG,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,UAAU,EAAE,IAAI,EAAE,QAAQ,EAAE,CAAC,CAAC;gBACjD,CAAC;gBAED,KAAK,MAAM,CAAC,IAAI;oBACd,GAAG,OAAO,CAAC,CAAC,CAAC,gBAAgB,CAAC,CAAC;oBAC/B,GAAG,OAAO,CAAC,CAAC,CAAC,iBAAiB,CAAC,CAAC;iBACjC,EAAE,CAAC;oBACF,MAAM,MAAM,GAAG,YAAY,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;oBAE5D,IAAI,MAAM,EAAE,CAAC;wBACX,GAAG,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC,CAAC;oBAC7C,CAAC;gBACH,CAAC;YACH,CAAC;QACH,CAAC;QAED,IAAI,KAAK,CAAC,QAAQ,CAAC,OAAO,CAAC,EAAE,CAAC;YAC5B,KAAK,MAAM,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,EAAE,CAAC;gBACzC,QAAQ,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC;YACtB,CAAC;QACH,CAAC;IACH,CAAC;IAED,OAAO,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;AAC3B,CAAC,CAAC;AAEF,gFAAgF;AAChF,MAAM,CAAC,MAAM,UAAU,GAAG,CAAC,GAAW,EAAW,EAAE;IACjD,IAAI,CAAC;QACH,OAAO,aAAa,CAAC,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,QAAQ,CAAC,KAAK,GAAG,CAAC;IACtD,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,KAAK,CAAC;IACf,CAAC;AACH,CAAC,CAAC;AAEF;;+EAE+E;AAE/E,MAAM,cAAc,GAAG,CACrB,IAAmB,EACnB,UAAoB,EACc,EAAE;IACpC,IAAI,YAAY,CAAC,UAAU,CAAC,EAAE,CAAC;QAC7B,OAAO,SAAS,CAAC;IACnB,CAAC;IAED,MAAM,KAAK,GAAG,UAAU,CAAC,UAAU,CAAC,CAAC;IAErC,IACE,CAAC,KAAK,KAAK,IAAI,IAAI,KAAK,GAAG,CAAC,IAAI,KAAK,GAAG,GAAG,CAAC;QAC5C,IAAI,CAAC,MAAM,CAAC,cAAc,GAAG,CAAC,EAC9B,CAAC;QACD,OAAO,SAAS,CAAC;IACnB,CAAC;IAED,OAAO,SAAS,CAAC;AACnB,CAAC,CAAC;AAEF,MAAM,cAAc,GAAG,CACrB,UAAoB,EACc,EAAE;IACpC,gDAAgD;IAChD,IAAI,UAAU,CAAC,QAAQ,CAAC,SAAS,CAAC,EAAE,CAAC;QACnC,OAAO,UAAU,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,cAAc,CAAC;IAC9D,CAAC;IAED,OAAO,UAAU,CAAC,UAAU,CAAC,IAAI,UAAU,CAAC,QAAQ,CAAC,WAAW,CAAC;QAC/D,CAAC,CAAC,UAAU;QACZ,CAAC,CAAC,SAAS,CAAC;AAChB,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,cAAc,GAAG,CAC5B,IAAmB,EACnB,QAAuB,EACP,EAAE;IAClB,MAAM,OAAO,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,EAAE,IAAI,CAAC;IAExC,OAAO;QACL,SAAS,EACP,CAAC,gBAAgB,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC,OAAO;YACzC,CAAC,QAAQ,EAAE,MAAM,IAAI,GAAG,CAAC,GAAG,GAAG;QACjC,cAAc,EAAE,cAAc,CAC5B,IAAI,EACJ,aAAa,CAAC,QAAQ,EAAE,IAAI,EAAE,QAAQ,CAAC,CACxC;QACD,cAAc,EAAE,cAAc,CAAC,aAAa,CAAC,MAAM,EAAE,IAAI,EAAE,QAAQ,CAAC,CAAC;QACrE,SAAS,EAAE,IAAI,CAAC,SAAS;QACzB,SAAS,EAAE,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,MAAM;QACvC,mBAAmB,EAAE;YACnB,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,cAAc,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;SACxD;QACD,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI;QAClD,aAAa,EAAE,OAAO,CAAC,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI;QAClE,YAAY,EAAE,OAAO,CAAC,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI;KACjE,CAAC;AACJ,CAAC,CAAC"}
@@ -0,0 +1,10 @@
1
+ import type { FetchOptions, FetchResult } from "../fetch.js";
2
+ import type { BotProbe } from "../types.js";
3
+ import type { AiBot } from "./bots.js";
4
+ /** What one bot's request says about access, given that the audit's own request got the page. */
5
+ export declare const classifyProbe: (res: FetchResult) => BotProbe;
6
+ /**
7
+ * Requests `url` once per bot with the bot's User-Agent, to catch a CDN or firewall that refuses
8
+ * AI bots while robots.txt allows them. The requests come from this machine, not the bots' IPs.
9
+ */
10
+ export declare const probeBots: (url: string, bots: AiBot[], options?: FetchOptions) => Promise<Map<string, BotProbe>>;
@@ -0,0 +1,53 @@
1
+ import { mapConcurrent } from "../concurrency.js";
2
+ import { fetchPage } from "../fetch.js";
3
+ /** Titles of the interstitials CDNs and bot protection show instead of the page. */
4
+ const CHALLENGE_TITLE = /^(?:just a moment|attention required|access denied|request blocked|verifying you are human|pardon our interruption)/iu;
5
+ /**
6
+ * The page title, enough to recognise a challenge page. `[^<>]` stops each attempt at the next
7
+ * bracket, which keeps the scan linear on a body full of "<title" without ">".
8
+ */
9
+ const TITLE = /<title[^<>]*>(?<title>[^<]{0,200})/iu;
10
+ /** What one bot's request says about access, given that the audit's own request got the page. */
11
+ export const classifyProbe = (res) => {
12
+ if (res.error) {
13
+ return {
14
+ status: res.status,
15
+ blocked: true,
16
+ reason: `connection failed: ${res.error}`,
17
+ };
18
+ }
19
+ if (res.status !== null && res.status >= 400) {
20
+ return { status: res.status, blocked: true, reason: `HTTP ${res.status}` };
21
+ }
22
+ const title = TITLE.exec(res.body ?? "")?.groups?.title?.trim() ?? "";
23
+ if (CHALLENGE_TITLE.test(title)) {
24
+ return {
25
+ status: res.status,
26
+ blocked: true,
27
+ reason: `challenge page ("${title}")`,
28
+ };
29
+ }
30
+ return { status: res.status, blocked: false, reason: null };
31
+ };
32
+ /**
33
+ * Requests `url` once per bot with the bot's User-Agent, to catch a CDN or firewall that refuses
34
+ * AI bots while robots.txt allows them. The requests come from this machine, not the bots' IPs.
35
+ */
36
+ export const probeBots = async (url, bots, options = {}) => {
37
+ const probes = new Map();
38
+ await mapConcurrent(bots, 3, async (bot) => {
39
+ if (!bot.userAgent) {
40
+ return;
41
+ }
42
+ const res = await fetchPage(url, {
43
+ ...options,
44
+ userAgent: bot.userAgent,
45
+ timeoutMs: Math.min(options.timeoutMs ?? 15_000, 15_000),
46
+ // Enough for the <title> of a challenge page.
47
+ maxBodyBytes: 100_000,
48
+ });
49
+ probes.set(bot.name, classifyProbe(res));
50
+ });
51
+ return probes;
52
+ };
53
+ //# sourceMappingURL=probe.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"probe.js","sourceRoot":"","sources":["../../../src/core/aeo/probe.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,MAAM,mBAAmB,CAAC;AAClD,OAAO,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AAKxC,oFAAoF;AACpF,MAAM,eAAe,GACnB,uHAAuH,CAAC;AAE1H;;;GAGG;AACH,MAAM,KAAK,GAAG,sCAAsC,CAAC;AAErD,iGAAiG;AACjG,MAAM,CAAC,MAAM,aAAa,GAAG,CAAC,GAAgB,EAAY,EAAE;IAC1D,IAAI,GAAG,CAAC,KAAK,EAAE,CAAC;QACd,OAAO;YACL,MAAM,EAAE,GAAG,CAAC,MAAM;YAClB,OAAO,EAAE,IAAI;YACb,MAAM,EAAE,sBAAsB,GAAG,CAAC,KAAK,EAAE;SAC1C,CAAC;IACJ,CAAC;IAED,IAAI,GAAG,CAAC,MAAM,KAAK,IAAI,IAAI,GAAG,CAAC,MAAM,IAAI,GAAG,EAAE,CAAC;QAC7C,OAAO,EAAE,MAAM,EAAE,GAAG,CAAC,MAAM,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,QAAQ,GAAG,CAAC,MAAM,EAAE,EAAE,CAAC;IAC7E,CAAC;IAED,MAAM,KAAK,GAAG,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,IAAI,EAAE,CAAC,EAAE,MAAM,EAAE,KAAK,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;IAEtE,IAAI,eAAe,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;QAChC,OAAO;YACL,MAAM,EAAE,GAAG,CAAC,MAAM;YAClB,OAAO,EAAE,IAAI;YACb,MAAM,EAAE,oBAAoB,KAAK,IAAI;SACtC,CAAC;IACJ,CAAC;IAED,OAAO,EAAE,MAAM,EAAE,GAAG,CAAC,MAAM,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC;AAC9D,CAAC,CAAC;AAEF;;;GAGG;AACH,MAAM,CAAC,MAAM,SAAS,GAAG,KAAK,EAC5B,GAAW,EACX,IAAa,EACb,OAAO,GAAiB,EAAE,EACM,EAAE;IAClC,MAAM,MAAM,GAAG,IAAI,GAAG,EAAoB,CAAC;IAE3C,MAAM,aAAa,CAAC,IAAI,EAAE,CAAC,EAAE,KAAK,EAAE,GAAG,EAAE,EAAE;QACzC,IAAI,CAAC,GAAG,CAAC,SAAS,EAAE,CAAC;YACnB,OAAO;QACT,CAAC;QAED,MAAM,GAAG,GAAG,MAAM,SAAS,CAAC,GAAG,EAAE;YAC/B,GAAG,OAAO;YACV,SAAS,EAAE,GAAG,CAAC,SAAS;YACxB,SAAS,EAAE,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,SAAS,IAAI,MAAM,EAAE,MAAM,CAAC;YACxD,8CAA8C;YAC9C,YAAY,EAAE,OAAO;SACtB,CAAC,CAAC;QAEH,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,EAAE,aAAa,CAAC,GAAG,CAAC,CAAC,CAAC;IAC3C,CAAC,CAAC,CAAC;IAEH,OAAO,MAAM,CAAC;AAChB,CAAC,CAAC"}
@@ -0,0 +1,13 @@
1
+ import type { AiCrawlerReport, Finding, LlmsTxtInfo, RuleMeta } from "../types.js";
2
+ /** Rules about the site as a whole, which the AEO audit checks once per origin. */
3
+ export declare const AEO_SITE_RULES: RuleMeta[];
4
+ export interface AeoSiteContext {
5
+ crawlers: AiCrawlerReport | null;
6
+ llmsTxt: LlmsTxtInfo | null;
7
+ /** Pages audited, for "blocked from N of M pages". */
8
+ pageCount: number;
9
+ /** Set when the caller asked for the bot probe but it could not run. */
10
+ probeSkipped?: string;
11
+ }
12
+ /** Findings about robots.txt, llms.txt and the bot probe, for the site the audited pages belong to. */
13
+ export declare const aeoSiteFindings: (ctx: AeoSiteContext) => Finding[];