@rivi-software/agentic-seo-check 0.1.0 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +1 -1
- package/README.md +124 -18
- package/dist/brand.d.ts +11 -0
- package/dist/brand.js +22 -0
- package/dist/brand.js.map +1 -0
- package/dist/cli.js +174 -86
- package/dist/cli.js.map +1 -1
- package/dist/core/aeo/audit.d.ts +44 -0
- package/dist/core/aeo/audit.js +310 -0
- package/dist/core/aeo/audit.js.map +1 -0
- package/dist/core/aeo/bots.d.ts +30 -0
- package/dist/core/aeo/bots.js +299 -0
- package/dist/core/aeo/bots.js.map +1 -0
- package/dist/core/aeo/llms-txt.d.ts +8 -0
- package/dist/core/aeo/llms-txt.js +88 -0
- package/dist/core/aeo/llms-txt.js.map +1 -0
- package/dist/core/aeo/page.d.ts +53 -0
- package/dist/core/aeo/page.js +312 -0
- package/dist/core/aeo/page.js.map +1 -0
- package/dist/core/aeo/probe.d.ts +10 -0
- package/dist/core/aeo/probe.js +53 -0
- package/dist/core/aeo/probe.js.map +1 -0
- package/dist/core/aeo/site-rules.d.ts +13 -0
- package/dist/core/aeo/site-rules.js +178 -0
- package/dist/core/aeo/site-rules.js.map +1 -0
- package/dist/core/audit.d.ts +37 -16
- package/dist/core/audit.js +389 -249
- package/dist/core/audit.js.map +1 -1
- package/dist/core/concurrency.d.ts +4 -4
- package/dist/core/concurrency.js +33 -23
- package/dist/core/concurrency.js.map +1 -1
- package/dist/core/crawl.d.ts +9 -5
- package/dist/core/crawl.js +271 -123
- package/dist/core/crawl.js.map +1 -1
- package/dist/core/fetch.d.ts +5 -5
- package/dist/core/fetch.js +151 -100
- package/dist/core/fetch.js.map +1 -1
- package/dist/core/html/answer.d.ts +4 -0
- package/dist/core/html/answer.js +198 -0
- package/dist/core/html/answer.js.map +1 -0
- package/dist/core/html/dom.d.ts +17 -0
- package/dist/core/html/dom.js +83 -0
- package/dist/core/html/dom.js.map +1 -0
- package/dist/core/html/extract.d.ts +2 -2
- package/dist/core/html/extract.js +102 -111
- package/dist/core/html/extract.js.map +1 -1
- package/dist/core/html/structured-data.d.ts +16 -6
- package/dist/core/html/structured-data.js +180 -145
- package/dist/core/html/structured-data.js.map +1 -1
- package/dist/core/html/text.d.ts +14 -14
- package/dist/core/html/text.js +110 -107
- package/dist/core/html/text.js.map +1 -1
- package/dist/core/lighthouse.d.ts +26 -25
- package/dist/core/lighthouse.js +181 -105
- package/dist/core/lighthouse.js.map +1 -1
- package/dist/core/performance-rules.d.ts +2 -2
- package/dist/core/performance-rules.js +105 -33
- package/dist/core/performance-rules.js.map +1 -1
- package/dist/core/render-rules.d.ts +5 -4
- package/dist/core/render-rules.js +129 -62
- package/dist/core/render-rules.js.map +1 -1
- package/dist/core/render.d.ts +2 -2
- package/dist/core/render.js +29 -23
- package/dist/core/render.js.map +1 -1
- package/dist/core/report/markdown.d.ts +4 -2
- package/dist/core/report/markdown.js +210 -129
- package/dist/core/report/markdown.js.map +1 -1
- package/dist/core/robots.d.ts +12 -5
- package/dist/core/robots.js +43 -17
- package/dist/core/robots.js.map +1 -1
- package/dist/core/rules/aeo.d.ts +11 -0
- package/dist/core/rules/aeo.js +423 -0
- package/dist/core/rules/aeo.js.map +1 -0
- package/dist/core/rules/canonical.d.ts +1 -1
- package/dist/core/rules/canonical.js +93 -57
- package/dist/core/rules/canonical.js.map +1 -1
- package/dist/core/rules/content.js +40 -26
- package/dist/core/rules/content.js.map +1 -1
- package/dist/core/rules/document.js +64 -41
- package/dist/core/rules/document.js.map +1 -1
- package/dist/core/rules/headings.js +58 -38
- package/dist/core/rules/headings.js.map +1 -1
- package/dist/core/rules/hreflang.js +70 -38
- package/dist/core/rules/hreflang.js.map +1 -1
- package/dist/core/rules/images.js +44 -26
- package/dist/core/rules/images.js.map +1 -1
- package/dist/core/rules/index.d.ts +10 -6
- package/dist/core/rules/index.js +43 -36
- package/dist/core/rules/index.js.map +1 -1
- package/dist/core/rules/indexing.d.ts +2 -2
- package/dist/core/rules/indexing.js +48 -26
- package/dist/core/rules/indexing.js.map +1 -1
- package/dist/core/rules/keyword.js +84 -47
- package/dist/core/rules/keyword.js.map +1 -1
- package/dist/core/rules/links.js +72 -37
- package/dist/core/rules/links.js.map +1 -1
- package/dist/core/rules/meta.js +55 -34
- package/dist/core/rules/meta.js.map +1 -1
- package/dist/core/rules/response.js +75 -41
- package/dist/core/rules/response.js.map +1 -1
- package/dist/core/rules/rule.d.ts +6 -6
- package/dist/core/rules/rule.js +36 -36
- package/dist/core/rules/rule.js.map +1 -1
- package/dist/core/rules/social.js +86 -45
- package/dist/core/rules/social.js.map +1 -1
- package/dist/core/rules/structured-data.d.ts +6 -4
- package/dist/core/rules/structured-data.js +314 -102
- package/dist/core/rules/structured-data.js.map +1 -1
- package/dist/core/rules/title.js +48 -33
- package/dist/core/rules/title.js.map +1 -1
- package/dist/core/score.d.ts +5 -3
- package/dist/core/score.js +11 -9
- package/dist/core/score.js.map +1 -1
- package/dist/core/site-rules.d.ts +3 -3
- package/dist/core/site-rules.js +409 -198
- package/dist/core/site-rules.js.map +1 -1
- package/dist/core/sitemap.d.ts +5 -5
- package/dist/core/sitemap.js +109 -64
- package/dist/core/sitemap.js.map +1 -1
- package/dist/core/static.d.ts +2 -2
- package/dist/core/static.js +210 -112
- package/dist/core/static.js.map +1 -1
- package/dist/core/summary.d.ts +2 -2
- package/dist/core/summary.js +11 -6
- package/dist/core/summary.js.map +1 -1
- package/dist/core/types.d.ts +189 -14
- package/dist/core/types.js +15 -8
- package/dist/core/types.js.map +1 -1
- package/dist/core/url.d.ts +11 -11
- package/dist/core/url.js +122 -70
- package/dist/core/url.js.map +1 -1
- package/dist/index.d.ts +15 -11
- package/dist/index.js +11 -9
- package/dist/index.js.map +1 -1
- package/dist/mcp-bin.js +7 -4
- package/dist/mcp-bin.js.map +1 -1
- package/dist/mcp.d.ts +3 -3
- package/dist/mcp.js +437 -121
- package/dist/mcp.js.map +1 -1
- package/dist/version.d.ts +0 -1
- package/dist/version.js +4 -2
- package/dist/version.js.map +1 -1
- package/package.json +32 -10
- package/skills/seo-audit/SKILL.md +8 -2
package/dist/cli.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"cli.js","sourceRoot":"","sources":["../src/cli.ts"],"names":[],"mappings":";AACA,OAAO,EAAE,QAAQ,IAAI,EAAE,EAAE,MAAM,SAAS,CAAC;
|
|
1
|
+
{"version":3,"file":"cli.js","sourceRoot":"","sources":["../src/cli.ts"],"names":[],"mappings":";AACA,OAAO,EAAE,QAAQ,IAAI,EAAE,EAAE,MAAM,SAAS,CAAC;AAEzC,OAAO,EAAE,OAAO,EAAE,oBAAoB,EAAE,MAAM,EAAE,MAAM,WAAW,CAAC;AAElE,OAAO,EAAE,KAAK,EAAE,SAAS,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC;AAC9D,OAAO,EAAE,cAAc,EAAE,eAAe,EAAE,MAAM,qBAAqB,CAAC;AACtE,OAAO,EAAE,WAAW,EAAE,YAAY,EAAE,MAAM,iBAAiB,CAAC;AAG5D,OAAO,EAAE,iBAAiB,EAAE,cAAc,EAAE,MAAM,2BAA2B,CAAC;AAC9E,OAAO,EAAE,SAAS,EAAE,MAAM,uBAAuB,CAAC;AAClD,OAAO,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAC;AAEhD,OAAO,EAAE,cAAc,EAAE,MAAM,UAAU,CAAC;AAC1C,OAAO,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AAEvC,MAAM,OAAO,GAAG,IAAI,OAAO,EAAE,CAAC;AAE9B,OAAO;KACJ,IAAI,CAAC,mBAAmB,CAAC;KACzB,WAAW,CACV,2LAA2L,CAC5L;KACA,OAAO,CAAC,OAAO,CAAC;KAChB,WAAW,CACV,UAAU,EACV,aAAa,KAAK,KAAK,SAAS,uBAAuB,cAAc,EAAE,CACxE;KACA,kBAAkB,EAAE,CAAC;AAExB,MAAM,MAAM,GAAG,CAAC,KAAa,EAAU,EAAE;IACvC,uKAAuK;IACvK,MAAM,CAAC,GAAG,MAAM,CAAC,QAAQ,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;IAErC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;QACjC,MAAM,IAAI,oBAAoB,CAAC,kCAAkC,CAAC,CAAC;IACrE,CAAC;IAED,OAAO,CAAC,CAAC;AACX,CAAC,CAAC;AAKF,MAAM,eAAe,GAAG,CAAC,GAAY,EAAE,IAAU,EAAW,EAAE;IAC5D,IAAI,IAAI,KAAK,OAAO,EAAE,CAAC;QACrB,GAAG,CAAC,MAAM,CACR,SAAS,EACT,uEAAuE,CACxE,CAAC;IACJ,CAAC;IAED,GAAG;SACA,MAAM,CAAC,iBAAiB,EAAE,wBAAwB,EAAE,MAAM,EAAE,EAAE,CAAC;SAC/D,MAAM,CACL,iBAAiB,EACjB,uCAAuC,EACvC,MAAM,EACN,CAAC,CACF;SACA,MAAM,CAAC,mBAAmB,EAAE,kCAAkC,EAAE,MAAM,EAAE,CAAC,CAAC;SAC1E,MAAM,CAAC,cAAc,EAAE,mCAAmC,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC;IAE1E,IAAI,IAAI,KAAK,KAAK,EAAE,CAAC;QACnB,GAAG,CAAC,MAAM,CACR,WAAW,EACX,qFAAqF,CACtF,CAAC;IACJ,CAAC;SAAM,CAAC;QACN,GAAG;aACA,MAAM,CAAC,yBAAyB,EAAE,qCAAqC,CAAC;aACxE,MAAM,CACL,UAAU,EACV,6EAA6E,CAC9E;aACA,MAAM,CACL,cAAc,EACd,qEAAqE,CACtE;aACA,SAAS,CACR,IAAI,MAAM,CAAC,sBAAsB,EAAE,6BAA6B,CAAC;aAC9D,OAAO,CAAC,CAAC,QAAQ,EAAE,SAAS,CAAC,CAAC;aAC9B,OAAO,CAAC,QAAQ,CAAC,CACrB,CAAC;IACN,CAAC;IAED,GAAG;SACA,MAAM,CACL,kBAAkB,EAClB,+EAA+E,CAChF;SACA,MAAM,CAAC,iBAAiB,EAAE,uCAAuC,CAAC;SAClE,MAAM,CACL,sBAAsB,EACtB,4CAA4C,CAC7C,CAAC;IAEJ,IAAI,IAAI,KAAK,KAAK,EAAE,CAAC;QACnB,GAAG,CAAC,MAAM,CACR,kBAAkB,EAClB,iDAAiD,CAClD,CAAC;IACJ,CAAC;IAED,OAAO,GAAG;SACP,MAAM,CAAC,sBAAsB,EAAE,yCAAyC,CAAC;SACzE,MAAM,CAAC,sBAAsB,EAAE,mCAAmC,CAAC;SACnE,SAAS,CACR,IAAI,MAAM,CACR,uBAAuB,EACvB,4DAA4D,CAC7D,CAAC,OAAO,CAAC,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC,CAC1B;SACA,MAAM,CAAC,kBAAkB,EAAE,8CAA8C,CAAC;SAC1E,SAAS,CACR,IAAI,MAAM,CACR,sBAAsB,EACtB,gEAAgE,CACjE;SACE,OAAO,CAAC,CAAC,OAAO,EAAE,SAAS,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC;SAC7C,OAAO,CAAC,MAAM,CAAC,CACnB;SACA,MAAM,CAAC,WAAW,EAAE,0BAA0B,CAAC;SAC/C,MAAM,CACL,WAAW,EACX,2EAA2E,CAC5E;SACA,SAAS,CACR,IAAI,MAAM,CAAC,uBAAuB,EAAE,iCAAiC,CAAC;SACnE,OAAO,CAAC,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;SAC5B,OAAO,CAAC,MAAM,CAAC,CACnB;SACA,MAAM,CAAC,qBAAqB,EAAE,kBAAkB,CAAC;SACjD,MAAM,CAAC,gBAAgB,EAAE,qBAAqB,EAAE,MAAM,EAAE,MAAM,CAAC;SAC/D,MAAM,CAAC,mBAAmB,EAAE,0BAA0B,CAAC;SACvD,MAAM,CACL,0BAA0B,EAC1B,iDAAiD,CAClD;SACA,MAAM,CAAC,aAAa,EAAE,oCAAoC,CAAC,CAAC;AACjE,CAAC,CAAC;AAgCF,qGAAqG;AACrG,MAAM,QAAQ,GAAG,CAAC,IAAgB,EAAiB,EAAE;IACnD,IAAI,IAAI,CAAC,MAAM,KAAK,SAAS,EAAE,CAAC;QAC9B,OAAO,IAAI,CAAC,MAAM,CAAC;IACrB,CAAC;IAED,IAAI,IAAI,CAAC,GAAG,EAAE,CAAC;QACb,OAAO,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC;IAClD,CAAC;IAED,OAAO,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC;AAC9C,CAAC,CAAC;AAEF,mDAAmD;AACnD,MAAM,YAAY,GAAG,CAAC,MAAM,GAAa,EAAE,EAAE,EAAE;IAC7C,MAAM,OAAO,GAAyC,EAAE,CAAC;IAEzD,KAAK,MAAM,CAAC,IAAI,MAAM,EAAE,CAAC;QACvB,MAAM,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;QAE3B,IAAI,GAAG,KAAK,CAAC,CAAC,EAAE,CAAC;YACf,MAAM,IAAI,oBAAoB,CAAC,WAAW,CAAC,0BAA0B,CAAC,CAAC;QACzE,CAAC;QAED,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;IAC5D,CAAC;IAED,OAAO,OAAO,CAAC;AACjB,CAAC,CAAC;AAEF,MAAM,eAAe,GAAG,CAAC,IAAgB,EAAE,EAAE;IAC3C,MAAM,YAAY,GAAG,CAAC,IAAI,CAAC,KAAK,IAAI,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC;IAEzD,OAAO,CAAC,CAAgB,EAAQ,EAAE;QAChC,IAAI,CAAC,YAAY,EAAE,CAAC;YAClB,OAAO;QACT,CAAC;QAED,MAAM,OAAO,GACX,CAAC,CAAC,OAAO,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,OAAO,IAAI,CAAC,CAAC,UAAU,GAAG,CAAC;QAEnE,OAAO,CAAC,MAAM,CAAC,KAAK,CAClB,YAAY,CAAC,CAAC,KAAK,GAAG,OAAO,KAAK,CAAC,CAAC,OAAO,aAAa,CACzD,CAAC;IACJ,CAAC,CAAC;AACJ,CAAC,CAAC;AAEF,mFAAmF;AACnF,MAAM,QAAQ,GAAG,KAAK,EACpB,MAAc,EACd,IAAgB,EAChB,IAAU,EAC4C,EAAE;IACxD,MAAM,OAAO,GAAG,YAAY,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IAE1C,MAAM,OAAO,GAAuC;QAClD,KAAK,EAAE,IAAI,KAAK,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC;QAC9C,QAAQ,EAAE,IAAI,CAAC,QAAQ;QACvB,QAAQ,EAAE,IAAI,CAAC,QAAQ;QACvB,WAAW,EAAE,IAAI,CAAC,WAAW;QAC7B,OAAO,EAAE,IAAI,CAAC,KAAK;QACnB,OAAO,EAAE,IAAI,CAAC,OAAO;QACrB,MAAM,EAAE,IAAI,CAAC,MAAM;QACnB,UAAU,EAAE,IAAI,CAAC,UAAU;QAC3B,UAAU,EAAE,IAAI,CAAC,UAAU;QAC3B,OAAO,EAAE,IAAI,CAAC,OAAO;QACrB,aAAa,EAAE,CAAC,IAAI,CAAC,YAAY;QACjC,iBAAiB,EAAE,IAAI,CAAC,iBAAiB;QACzC,kBAAkB,EAAE,IAAI,CAAC,aAAa;QACtC,eAAe,EAAE,IAAI,CAAC,OAAO;QAC7B,eAAe,EAAE,IAAI,CAAC,OAAO;QAC7B,WAAW,EAAE,IAAI,CAAC,IAAI;QACtB,YAAY,EAAE,IAAI,CAAC,OAAO;QAC1B,UAAU,EAAE,IAAI,CAAC,UAAU;QAC3B,KAAK,EAAE;YACL,SAAS,EAAE,IAAI,CAAC,OAAO;YACvB,SAAS,EAAE,IAAI,CAAC,SAAS;YACzB,OAAO,EAAE,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,SAAS;SAC3D;QACD,UAAU,EAAE,eAAe,CAAC,IAAI,CAAC;KAClC,CAAC;IAEF,MAAM,MAAM,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC;IAE9B,IAAI,IAAI,KAAK,KAAK,EAAE,CAAC;QACnB,IAAI,GAAG,GAAa,MAAM,cAAc,CAAC,MAAM,EAAE;YAC/C,GAAG,OAAO;YACV,MAAM,EAAE,IAAI,CAAC,MAAM;SACpB,CAAC,CAAC;QAEH,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;YACjB,GAAG,GAAG,eAAe,CAAC,GAAG,CAAC,CAAC;QAC7B,CAAC;QAED,OAAO;YACL,MAAM,EAAE,GAAG,CAAC,MAAM;YAClB,MAAM,EACJ,MAAM,KAAK,IAAI,CAAC,CAAC,CAAC,iBAAiB,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,GAAG,EAAE,IAAI,EAAE,CAAC,CAAC;SAC1E,CAAC;IACJ,CAAC;IAED,IAAI,KAAK,GAAU,MAAM,WAAW,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAEtD,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;QACjB,KAAK,GAAG,YAAY,CAAC,KAAK,CAAC,CAAC;IAC9B,CAAC;IAED,OAAO;QACL,MAAM,EAAE,KAAK,CAAC,MAAM;QACpB,MAAM,EACJ,MAAM,KAAK,IAAI,CAAC,CAAC,CAAC,cAAc,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;KAC3E,CAAC;AACJ,CAAC,CAAC;AAEF,MAAM,QAAQ,GAAG,KAAK,EACpB,MAAc,EACd,IAAgB,EAChB,IAAU,EACK,EAAE;IACjB,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,GAAG,MAAM,QAAQ,CAAC,MAAM,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC;IAE9D,IAAI,IAAI,CAAC,GAAG,EAAE,CAAC;QACb,MAAM,EAAE,CAAC,SAAS,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,MAAM,IAAI,EAAE,OAAO,CAAC,CAAC;QAErD,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC;YAChB,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,qBAAqB,IAAI,CAAC,GAAG,IAAI,CAAC,CAAC;QAC1D,CAAC;IACH,CAAC;SAAM,CAAC;QACN,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,MAAM,IAAI,CAAC,CAAC;IACtC,CAAC;IAED,IAAI,IAAI,CAAC,MAAM,KAAK,MAAM,EAAE,CAAC;QAC3B,MAAM,SAAS,GAAG,aAAa,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAE7C,MAAM,IAAI,GAAI,CAAC,OAAO,EAAE,SAAS,EAAE,MAAM,CAAW;aACjD,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,aAAa,CAAC,CAAC,CAAC,IAAI,SAAS,CAAC;aAC5C,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;QAEtC,IAAI,IAAI,GAAG,CAAC,EAAE,CAAC;YACb,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC;gBAChB,OAAO,CAAC,MAAM,CAAC,KAAK,CAClB,GAAG,IAAI,4BAA4B,IAAI,CAAC,MAAM,eAAe,CAC9D,CAAC;YACJ,CAAC;YAED,OAAO,CAAC,QAAQ,GAAG,CAAC,CAAC;QACvB,CAAC;IACH,CAAC;AACH,CAAC,CAAC;AAEF,eAAe,CACb,OAAO;KACJ,OAAO,CAAC,OAAO,CAAC;KAChB,QAAQ,CACP,UAAU,EACV,oEAAoE,CACrE;KACA,WAAW,CACV,+DAA+D,CAChE,EACH,OAAO,CACR,CAAC,MAAM,CAAC,KAAK,EAAE,MAAc,EAAE,IAAgB,EAAE,EAAE;IAClD,MAAM,QAAQ,CAAC,MAAM,EAAE,IAAI,EAAE,OAAO,CAAC,CAAC;AACxC,CAAC,CAAC,CAAC;AAEH,eAAe,CACb,OAAO;KACJ,OAAO,CAAC,OAAO,CAAC;KAChB,QAAQ,CAAC,OAAO,EAAE,WAAW,CAAC;KAC9B,WAAW,CAAC,6DAA6D,CAAC,EAC7E,OAAO,CACR,CAAC,MAAM,CAAC,KAAK,EAAE,GAAW,EAAE,IAAgB,EAAE,EAAE;IAC/C,MAAM,QAAQ,CAAC,GAAG,EAAE,IAAI,EAAE,OAAO,CAAC,CAAC;AACrC,CAAC,CAAC,CAAC;AAEH,eAAe,CACb,OAAO;KACJ,OAAO,CAAC,KAAK,CAAC;KACd,QAAQ,CACP,UAAU,EACV,oEAAoE,CACrE;KACA,WAAW,CACV,uKAAuK,CACxK,EACH,KAAK,CACN,CAAC,MAAM,CAAC,KAAK,EAAE,MAAc,EAAE,IAAgB,EAAE,EAAE;IAClD,MAAM,QAAQ,CAAC,MAAM,EAAE,IAAI,EAAE,KAAK,CAAC,CAAC;AACtC,CAAC,CAAC,CAAC;AAEH,OAAO;KACJ,OAAO,CAAC,OAAO,CAAC;KAChB,WAAW,CAAC,qCAAqC,CAAC;KAClD,MAAM,CAAC,uBAAuB,EAAE,oBAAoB,CAAC;KACrD,MAAM,CAAC,aAAa,EAAE,eAAe,CAAC;KACtC,SAAS,CACR,IAAI,MAAM,CAAC,uBAAuB,EAAE,eAAe,CAAC;KACjD,OAAO,CAAC,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;KACvB,OAAO,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CACjD;KACA,MAAM,CACL,CAAC,IAAiE,EAAE,EAAE;IACpE,MAAM,KAAK,GAAG,SAAS,CAAC,MAAM,CAC5B,CAAC,CAAC,EAAE,EAAE,CACJ,CAAC,CAAC,IAAI,CAAC,QAAQ,IAAI,CAAC,CAAC,QAAQ,KAAK,IAAI,CAAC,QAAQ,CAAC;QAChD,CAAC,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,CAAC,EAAE,KAAK,IAAI,CAAC,IAAI,CAAC,CACrC,CAAC;IAEF,IAAI,IAAI,CAAC,MAAM,KAAK,MAAM,EAAE,CAAC;QAC3B,OAAO,CAAC,MAAM,CAAC,KAAK,CAClB,GAAG,IAAI,CAAC,SAAS,CAAC,EAAE,KAAK,EAAE,KAAK,CAAC,MAAM,EAAE,KAAK,EAAE,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,CAC/D,CAAC;QAEF,OAAO;IACT,CAAC;IAED,MAAM,KAAK,GAAG;QACZ,yCAAyC;QACzC,2BAA2B;KAC5B,CAAC;IAEF,KAAK,MAAM,CAAC,IAAI,KAAK,EAAE,CAAC;QACtB,KAAK,CAAC,IAAI,CACR,OAAO,CAAC,CAAC,EAAE,QAAQ,CAAC,CAAC,QAAQ,MAAM,CAAC,CAAC,QAAQ,MAAM,CAAC,CAAC,WAAW,CAAC,UAAU,CAAC,GAAG,EAAE,KAAK,CAAC,GAAG,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,eAAe,CAAC,CAAC,QAAQ,IAAI,CAAC,CAAC,CAAC,EAAE,IAAI,CAC9I,CAAC;IACJ,CAAC;IAED,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAChD,CAAC,CACF,CAAC;AAEJ,OAAO;KACJ,OAAO,CAAC,KAAK,CAAC;KACd,WAAW,CACV,sEAAsE,CACvE;KACA,MAAM,CAAC,KAAK,IAAI,EAAE;IACjB,MAAM,cAAc,EAAE,CAAC;AACzB,CAAC,CAAC,CAAC;AAEL,IAAI,CAAC;IACH,MAAM,OAAO,CAAC,UAAU,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;AACzC,CAAC;AAAC,OAAO,KAAK,EAAE,CAAC;IACf,OAAO,CAAC,MAAM,CAAC,KAAK,CAClB,sBAAsB,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CACjF,CAAC;IACF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AAClB,CAAC"}
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
import type { ProgressEvent } from "../audit.js";
|
|
2
|
+
import type { FetchOptions } from "../fetch.js";
|
|
3
|
+
import type { AeoAudit } from "../types.js";
|
|
4
|
+
export interface AeoOptions {
|
|
5
|
+
/** Crawl the site from a URL target instead of auditing one page. */
|
|
6
|
+
crawl?: boolean;
|
|
7
|
+
maxPages?: number;
|
|
8
|
+
maxDepth?: number;
|
|
9
|
+
concurrency?: number;
|
|
10
|
+
delayMs?: number;
|
|
11
|
+
respectRobots?: boolean;
|
|
12
|
+
includeSubdomains?: boolean;
|
|
13
|
+
includePatterns?: string[];
|
|
14
|
+
excludePatterns?: string[];
|
|
15
|
+
/** Base URL for inline HTML, files and directories (default http://localhost/). */
|
|
16
|
+
baseUrl?: string;
|
|
17
|
+
/** Also request the page with each AI bot's User-Agent, to catch CDN or firewall blocks. */
|
|
18
|
+
aiBots?: boolean;
|
|
19
|
+
/** Include info-level findings (default true). */
|
|
20
|
+
includeInfo?: boolean;
|
|
21
|
+
/** Rule ids to skip. */
|
|
22
|
+
disableRules?: string[];
|
|
23
|
+
/** Site audits: "summary" keeps page findings out of `findings`; they stay rolled up in `issues`. */
|
|
24
|
+
pageDetail?: "full" | "summary";
|
|
25
|
+
fetch?: FetchOptions;
|
|
26
|
+
onProgress?: (event: ProgressEvent) => void;
|
|
27
|
+
}
|
|
28
|
+
/** AEO audit of an HTML string. `url` is where the HTML would be served, for links and display. */
|
|
29
|
+
export declare const auditAeoHtml: (html: string, input?: {
|
|
30
|
+
url?: string;
|
|
31
|
+
target?: string;
|
|
32
|
+
}, options?: AeoOptions) => Promise<AeoAudit>;
|
|
33
|
+
/** AEO audit of one live page, with its site's robots.txt and llms.txt. */
|
|
34
|
+
export declare const auditAeoUrl: (url: string, options?: AeoOptions) => Promise<AeoAudit>;
|
|
35
|
+
/** AEO audit of a site, crawled from `startUrl`. */
|
|
36
|
+
export declare const auditAeoSite: (startUrl: string, options?: AeoOptions) => Promise<AeoAudit>;
|
|
37
|
+
/** AEO audit of a directory of built HTML, with the robots.txt and llms.txt in it. */
|
|
38
|
+
export declare const auditAeoDirectory: (dir: string, options?: AeoOptions) => Promise<AeoAudit>;
|
|
39
|
+
/** AEO audit of a URL (a crawl with `crawl`), a directory, an .html file, or "-" for stdin. */
|
|
40
|
+
export declare const auditAeoTarget: (target: string, options?: AeoOptions & {
|
|
41
|
+
stdinHtml?: string;
|
|
42
|
+
}) => Promise<AeoAudit>;
|
|
43
|
+
/** Drops bulky detail to save tokens when an LLM reads the result. */
|
|
44
|
+
export declare const compactAeoAudit: (audit: AeoAudit) => AeoAudit;
|
|
@@ -0,0 +1,310 @@
|
|
|
1
|
+
import { promises as fs } from "node:fs";
|
|
2
|
+
import path from "node:path";
|
|
3
|
+
import { DEFAULT_CRAWL, readStdin, resolveTarget, rollupIssues, trimFinding, } from "../audit.js";
|
|
4
|
+
import { crawlSite } from "../crawl.js";
|
|
5
|
+
import { DEFAULT_USER_AGENT, fetchPage, toResponseInfo } from "../fetch.js";
|
|
6
|
+
import { extractPage } from "../html/extract.js";
|
|
7
|
+
import { loadRobotsTxt } from "../robots.js";
|
|
8
|
+
import { AEO_PAGE_RULES, runPageRules, sortFindings } from "../rules/index.js";
|
|
9
|
+
import { pageScore, siteScore } from "../score.js";
|
|
10
|
+
import { scanDirectory } from "../static.js";
|
|
11
|
+
import { countFindings, DEFAULT_THRESHOLDS } from "../types.js";
|
|
12
|
+
import { AI_BOTS, crawlerAccess } from "./bots.js";
|
|
13
|
+
import { loadLlmsTxt, readLlmsTxt } from "./llms-txt.js";
|
|
14
|
+
import { aeoPageSummary } from "./page.js";
|
|
15
|
+
import { probeBots } from "./probe.js";
|
|
16
|
+
import { aeoSiteFindings } from "./site-rules.js";
|
|
17
|
+
const userAgentOf = (options) => options.fetch?.userAgent ?? DEFAULT_USER_AGENT;
|
|
18
|
+
/** robots.txt and llms.txt of the origin a page is served from. */
|
|
19
|
+
const loadOrigin = async (pageUrl, options) => {
|
|
20
|
+
const { origin } = new URL(pageUrl);
|
|
21
|
+
options.onProgress?.({
|
|
22
|
+
phase: "robots",
|
|
23
|
+
message: `Fetching ${origin}/robots.txt and ${origin}/llms.txt`,
|
|
24
|
+
});
|
|
25
|
+
const [robots, llmsTxt] = await Promise.all([
|
|
26
|
+
loadRobotsTxt(origin, userAgentOf(options), options.fetch),
|
|
27
|
+
loadLlmsTxt(origin, options.fetch),
|
|
28
|
+
]);
|
|
29
|
+
return { robots: robots.info, robotsText: robots.text, llmsTxt };
|
|
30
|
+
};
|
|
31
|
+
const collectUrl = async (url, options) => {
|
|
32
|
+
const res = await fetchPage(url, options.fetch ?? {});
|
|
33
|
+
const page = extractPage(res.body ?? "", {
|
|
34
|
+
url: res.finalUrl,
|
|
35
|
+
includeSubdomains: options.includeSubdomains,
|
|
36
|
+
});
|
|
37
|
+
// A page that never answered says nothing about the site; its robots.txt would fail the same way.
|
|
38
|
+
const site = res.status === null ? null : await loadOrigin(res.finalUrl, options);
|
|
39
|
+
return {
|
|
40
|
+
target: url,
|
|
41
|
+
url: res.finalUrl,
|
|
42
|
+
source: "url",
|
|
43
|
+
pageSource: "url",
|
|
44
|
+
records: [
|
|
45
|
+
{
|
|
46
|
+
url: res.finalUrl,
|
|
47
|
+
requestedUrl: url,
|
|
48
|
+
page,
|
|
49
|
+
response: toResponseInfo(res),
|
|
50
|
+
depth: 0,
|
|
51
|
+
},
|
|
52
|
+
],
|
|
53
|
+
robots: site?.robots ?? null,
|
|
54
|
+
robotsText: site?.robotsText ?? null,
|
|
55
|
+
llmsTxt: site?.llmsTxt ?? null,
|
|
56
|
+
live: true,
|
|
57
|
+
};
|
|
58
|
+
};
|
|
59
|
+
const collectSite = async (url, options) => {
|
|
60
|
+
const result = await crawlSite(url, {
|
|
61
|
+
maxPages: options.maxPages ?? DEFAULT_CRAWL.maxPages,
|
|
62
|
+
maxDepth: options.maxDepth ?? DEFAULT_CRAWL.maxDepth,
|
|
63
|
+
concurrency: options.concurrency ?? DEFAULT_CRAWL.concurrency,
|
|
64
|
+
delayMs: options.delayMs ?? DEFAULT_CRAWL.delayMs,
|
|
65
|
+
respectRobots: options.respectRobots ?? true,
|
|
66
|
+
includeSubdomains: options.includeSubdomains ?? false,
|
|
67
|
+
checkExternalLinks: false,
|
|
68
|
+
// The AEO audit reads pages; broken links, sitemaps and redirect probes are the SEO crawl's job.
|
|
69
|
+
maxLinkChecks: 0,
|
|
70
|
+
siteChecks: false,
|
|
71
|
+
// oxlint-disable-next-line eslint/require-unicode-regexp -- the user's own patterns, in JavaScript's default syntax; the u flag would reject some of them
|
|
72
|
+
includePatterns: (options.includePatterns ?? []).map((p) => new RegExp(p)),
|
|
73
|
+
// oxlint-disable-next-line eslint/require-unicode-regexp -- the user's own patterns, in JavaScript's default syntax; the u flag would reject some of them
|
|
74
|
+
excludePatterns: (options.excludePatterns ?? []).map((p) => new RegExp(p)),
|
|
75
|
+
fetch: { ...options.fetch, userAgent: userAgentOf(options) },
|
|
76
|
+
onProgress: options.onProgress,
|
|
77
|
+
});
|
|
78
|
+
options.onProgress?.({ phase: "llms-txt", message: "Fetching llms.txt" });
|
|
79
|
+
return {
|
|
80
|
+
target: url,
|
|
81
|
+
url: result.startUrl,
|
|
82
|
+
source: "crawl",
|
|
83
|
+
pageSource: "url",
|
|
84
|
+
records: result.records,
|
|
85
|
+
robots: result.robots,
|
|
86
|
+
robotsText: result.robotsText ?? null,
|
|
87
|
+
llmsTxt: await loadLlmsTxt(new URL(result.startUrl).origin, options.fetch),
|
|
88
|
+
live: true,
|
|
89
|
+
};
|
|
90
|
+
};
|
|
91
|
+
const collectDirectory = async (dir, options) => {
|
|
92
|
+
const baseUrl = options.baseUrl ?? "http://localhost/";
|
|
93
|
+
const result = await scanDirectory(dir, {
|
|
94
|
+
baseUrl,
|
|
95
|
+
includeSubdomains: options.includeSubdomains ?? false,
|
|
96
|
+
maxPages: options.maxPages ?? 500,
|
|
97
|
+
userAgent: options.fetch?.userAgent,
|
|
98
|
+
});
|
|
99
|
+
return {
|
|
100
|
+
target: dir,
|
|
101
|
+
url: dir,
|
|
102
|
+
source: "directory",
|
|
103
|
+
pageSource: "file",
|
|
104
|
+
records: result.records,
|
|
105
|
+
robots: result.robots,
|
|
106
|
+
robotsText: result.robotsText ?? null,
|
|
107
|
+
llmsTxt: await readLlmsTxt(dir, baseUrl),
|
|
108
|
+
live: false,
|
|
109
|
+
};
|
|
110
|
+
};
|
|
111
|
+
/** One page from HTML in hand: no site around it, so no robots.txt or llms.txt. */
|
|
112
|
+
const collectHtml = (html, url, target, source, options) => ({
|
|
113
|
+
target,
|
|
114
|
+
url,
|
|
115
|
+
source,
|
|
116
|
+
pageSource: source,
|
|
117
|
+
records: [
|
|
118
|
+
{
|
|
119
|
+
url,
|
|
120
|
+
requestedUrl: url,
|
|
121
|
+
page: extractPage(html, {
|
|
122
|
+
url,
|
|
123
|
+
includeSubdomains: options.includeSubdomains,
|
|
124
|
+
}),
|
|
125
|
+
depth: 0,
|
|
126
|
+
},
|
|
127
|
+
],
|
|
128
|
+
robots: null,
|
|
129
|
+
robotsText: null,
|
|
130
|
+
llmsTxt: null,
|
|
131
|
+
live: false,
|
|
132
|
+
});
|
|
133
|
+
const evaluatePage = (record, source, options) => {
|
|
134
|
+
let findings = runPageRules({
|
|
135
|
+
page: record.page,
|
|
136
|
+
source,
|
|
137
|
+
response: record.response,
|
|
138
|
+
thresholds: DEFAULT_THRESHOLDS,
|
|
139
|
+
depth: record.depth,
|
|
140
|
+
}, { disable: new Set(options.disableRules), rules: AEO_PAGE_RULES });
|
|
141
|
+
if (options.includeInfo === false) {
|
|
142
|
+
findings = findings.filter((f) => f.severity !== "info");
|
|
143
|
+
}
|
|
144
|
+
const counts = countFindings(findings);
|
|
145
|
+
return { record, findings, counts, score: pageScore(counts) };
|
|
146
|
+
};
|
|
147
|
+
/**
|
|
148
|
+
* Requests the start page as each AI bot and records the answers on `crawlers`. Returns why the
|
|
149
|
+
* probe could not run, when it could not.
|
|
150
|
+
*/
|
|
151
|
+
const runProbe = async (c, start, crawlers, options) => {
|
|
152
|
+
const response = start?.record.response;
|
|
153
|
+
if (!c.live || !response) {
|
|
154
|
+
return "it needs a live URL, and this target is HTML, a file or a directory";
|
|
155
|
+
}
|
|
156
|
+
if (response.error || response.status === null || response.status >= 400) {
|
|
157
|
+
return `the audit's own request got ${response.status === null ? "no answer" : `HTTP ${response.status}`}, so a refusal could not be told apart from a page that fails for everyone`;
|
|
158
|
+
}
|
|
159
|
+
if (!crawlers) {
|
|
160
|
+
return "robots.txt could not be read";
|
|
161
|
+
}
|
|
162
|
+
options.onProgress?.({
|
|
163
|
+
phase: "ai-bots",
|
|
164
|
+
message: `Requesting ${response.finalUrl} with AI bot User-Agents`,
|
|
165
|
+
});
|
|
166
|
+
const probes = await probeBots(response.finalUrl, AI_BOTS, options.fetch);
|
|
167
|
+
crawlers.probeBaseline = {
|
|
168
|
+
userAgent: userAgentOf(options),
|
|
169
|
+
status: response.status,
|
|
170
|
+
};
|
|
171
|
+
for (const bot of crawlers.bots) {
|
|
172
|
+
const probe = probes.get(bot.name);
|
|
173
|
+
if (probe) {
|
|
174
|
+
bot.probe = probe;
|
|
175
|
+
}
|
|
176
|
+
}
|
|
177
|
+
return undefined;
|
|
178
|
+
};
|
|
179
|
+
const blockedBots = (crawlers, purpose) => crawlers
|
|
180
|
+
? crawlers.bots
|
|
181
|
+
.filter((b) => b.purpose === purpose && (!b.allowed || b.blockedPages > 0))
|
|
182
|
+
.map((b) => b.name)
|
|
183
|
+
: [];
|
|
184
|
+
const pageEntry = (p) => {
|
|
185
|
+
// url and depth lead the entry, the way a reader scans a list of pages.
|
|
186
|
+
const head = {
|
|
187
|
+
url: p.record.file ?? p.record.url,
|
|
188
|
+
};
|
|
189
|
+
if (p.record.depth >= 0) {
|
|
190
|
+
head.depth = p.record.depth;
|
|
191
|
+
}
|
|
192
|
+
return {
|
|
193
|
+
...head,
|
|
194
|
+
score: p.score,
|
|
195
|
+
counts: p.counts,
|
|
196
|
+
summary: aeoPageSummary(p.record.page, p.record.response),
|
|
197
|
+
};
|
|
198
|
+
};
|
|
199
|
+
const buildAudit = async (c, options) => {
|
|
200
|
+
const isSite = c.source === "crawl" || c.source === "directory";
|
|
201
|
+
const pages = c.records.map((r) => evaluatePage(r, c.pageSource, options));
|
|
202
|
+
const start = pages.find((p) => p.record.depth === 0) ?? pages[0];
|
|
203
|
+
const crawlers = c.robots
|
|
204
|
+
? crawlerAccess(c.robots, c.robotsText, start?.record.url ?? c.url, c.records.map((r) => r.url))
|
|
205
|
+
: null;
|
|
206
|
+
const probeSkipped = options.aiBots
|
|
207
|
+
? await runProbe(c, start, crawlers, options)
|
|
208
|
+
: undefined;
|
|
209
|
+
const disabled = new Set(options.disableRules);
|
|
210
|
+
let siteFindings = aeoSiteFindings({
|
|
211
|
+
crawlers,
|
|
212
|
+
llmsTxt: c.llmsTxt,
|
|
213
|
+
pageCount: pages.length,
|
|
214
|
+
probeSkipped,
|
|
215
|
+
}).filter((f) => !disabled.has(f.rule));
|
|
216
|
+
if (options.includeInfo === false) {
|
|
217
|
+
siteFindings = siteFindings.filter((f) => f.severity !== "info");
|
|
218
|
+
}
|
|
219
|
+
const pageFindings = pages.flatMap((p) => p.findings);
|
|
220
|
+
const audit = {
|
|
221
|
+
schemaVersion: 1,
|
|
222
|
+
kind: "aeo",
|
|
223
|
+
target: c.target,
|
|
224
|
+
url: c.url,
|
|
225
|
+
source: c.source,
|
|
226
|
+
score: siteScore(pages, siteFindings),
|
|
227
|
+
counts: countFindings([...siteFindings, ...pageFindings]),
|
|
228
|
+
auditedAt: new Date().toISOString(),
|
|
229
|
+
summary: {
|
|
230
|
+
pages: pages.length,
|
|
231
|
+
robotsTxt: crawlers?.status ?? null,
|
|
232
|
+
searchCrawlersBlocked: blockedBots(crawlers, "search"),
|
|
233
|
+
answerBotsBlocked: blockedBots(crawlers, "answer"),
|
|
234
|
+
trainingBotsBlocked: blockedBots(crawlers, "training"),
|
|
235
|
+
trainingPreference: crawlers?.trainingPreference ?? null,
|
|
236
|
+
blockedByServer: crawlers?.probeBaseline
|
|
237
|
+
? crawlers.bots.filter((b) => b.probe?.blocked).map((b) => b.name)
|
|
238
|
+
: null,
|
|
239
|
+
llmsTxt: c.llmsTxt?.status ?? null,
|
|
240
|
+
page: start
|
|
241
|
+
? aeoPageSummary(start.record.page, start.record.response)
|
|
242
|
+
: null,
|
|
243
|
+
},
|
|
244
|
+
findings: sortFindings(isSite && options.pageDetail === "summary"
|
|
245
|
+
? siteFindings
|
|
246
|
+
: [...siteFindings, ...pageFindings]),
|
|
247
|
+
crawlers,
|
|
248
|
+
llmsTxt: c.llmsTxt,
|
|
249
|
+
};
|
|
250
|
+
if (isSite) {
|
|
251
|
+
audit.pages = pages
|
|
252
|
+
.toSorted((a, b) => a.record.depth - b.record.depth || a.score - b.score)
|
|
253
|
+
.map(pageEntry);
|
|
254
|
+
audit.issues = rollupIssues(pages.map((p) => ({
|
|
255
|
+
url: p.record.file ?? p.record.url,
|
|
256
|
+
findings: p.findings,
|
|
257
|
+
})));
|
|
258
|
+
}
|
|
259
|
+
return audit;
|
|
260
|
+
};
|
|
261
|
+
/** AEO audit of an HTML string. `url` is where the HTML would be served, for links and display. */
|
|
262
|
+
// oxlint-disable-next-line eslint/require-await -- public API: it returns a promise like the other audit functions, so a throw reaches callers as a rejection
|
|
263
|
+
export const auditAeoHtml = async (html, input = {}, options = {}) => {
|
|
264
|
+
const url = input.url ?? options.baseUrl ?? "http://localhost/";
|
|
265
|
+
return buildAudit(collectHtml(html, url, input.target ?? url, "html", options), options);
|
|
266
|
+
};
|
|
267
|
+
/** AEO audit of one live page, with its site's robots.txt and llms.txt. */
|
|
268
|
+
export const auditAeoUrl = async (url, options = {}) => buildAudit(await collectUrl(url, options), options);
|
|
269
|
+
/** AEO audit of a site, crawled from `startUrl`. */
|
|
270
|
+
export const auditAeoSite = async (startUrl, options = {}) => buildAudit(await collectSite(startUrl, options), options);
|
|
271
|
+
/** AEO audit of a directory of built HTML, with the robots.txt and llms.txt in it. */
|
|
272
|
+
export const auditAeoDirectory = async (dir, options = {}) => buildAudit(await collectDirectory(dir, options), options);
|
|
273
|
+
/** AEO audit of a URL (a crawl with `crawl`), a directory, an .html file, or "-" for stdin. */
|
|
274
|
+
export const auditAeoTarget = async (target, options = {}) => {
|
|
275
|
+
if (target === "-" || options.stdinHtml !== undefined) {
|
|
276
|
+
const html = options.stdinHtml ?? (await readStdin());
|
|
277
|
+
return auditAeoHtml(html, { url: options.baseUrl, target: "stdin" }, options);
|
|
278
|
+
}
|
|
279
|
+
const resolved = await resolveTarget(target);
|
|
280
|
+
switch (resolved.kind) {
|
|
281
|
+
case "url": {
|
|
282
|
+
return options.crawl
|
|
283
|
+
? auditAeoSite(resolved.url, options)
|
|
284
|
+
: auditAeoUrl(resolved.url, options);
|
|
285
|
+
}
|
|
286
|
+
case "directory": {
|
|
287
|
+
return auditAeoDirectory(resolved.path, options);
|
|
288
|
+
}
|
|
289
|
+
default: {
|
|
290
|
+
const base = options.baseUrl ?? "http://localhost/";
|
|
291
|
+
const url = new URL(`/${path.basename(target)}`, base.endsWith("/") ? base : `${base}/`).href;
|
|
292
|
+
return buildAudit(collectHtml(await fs.readFile(target, "utf-8"), url, target, "file", options), options);
|
|
293
|
+
}
|
|
294
|
+
}
|
|
295
|
+
};
|
|
296
|
+
/** Drops bulky detail to save tokens when an LLM reads the result. */
|
|
297
|
+
export const compactAeoAudit = (audit) => ({
|
|
298
|
+
...audit,
|
|
299
|
+
findings: audit.findings.map(trimFinding),
|
|
300
|
+
crawlers: audit.crawlers
|
|
301
|
+
? {
|
|
302
|
+
...audit.crawlers,
|
|
303
|
+
contentSignals: audit.crawlers.contentSignals.slice(0, 3),
|
|
304
|
+
}
|
|
305
|
+
: null,
|
|
306
|
+
llmsTxt: audit.llmsTxt
|
|
307
|
+
? { ...audit.llmsTxt, sections: audit.llmsTxt.sections.slice(0, 5) }
|
|
308
|
+
: null,
|
|
309
|
+
});
|
|
310
|
+
//# sourceMappingURL=audit.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"audit.js","sourceRoot":"","sources":["../../../src/core/aeo/audit.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,IAAI,EAAE,EAAE,MAAM,SAAS,CAAC;AACzC,OAAO,IAAI,MAAM,WAAW,CAAC;AAE7B,OAAO,EACL,aAAa,EACb,SAAS,EACT,aAAa,EACb,YAAY,EACZ,WAAW,GACZ,MAAM,aAAa,CAAC;AAErB,OAAO,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AACxC,OAAO,EAAE,kBAAkB,EAAE,SAAS,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAE5E,OAAO,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAC;AACjD,OAAO,EAAE,aAAa,EAAE,MAAM,cAAc,CAAC;AAC7C,OAAO,EAAE,cAAc,EAAE,YAAY,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAC;AAC/E,OAAO,EAAE,SAAS,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AAEnD,OAAO,EAAE,aAAa,EAAE,MAAM,cAAc,CAAC;AAC7C,OAAO,EAAE,aAAa,EAAE,kBAAkB,EAAE,MAAM,aAAa,CAAC;AAYhE,OAAO,EAAE,OAAO,EAAE,aAAa,EAAE,MAAM,WAAW,CAAC;AACnD,OAAO,EAAE,WAAW,EAAE,WAAW,EAAE,MAAM,eAAe,CAAC;AACzD,OAAO,EAAE,cAAc,EAAE,MAAM,WAAW,CAAC;AAC3C,OAAO,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AACvC,OAAO,EAAE,eAAe,EAAE,MAAM,iBAAiB,CAAC;AAgDlD,MAAM,WAAW,GAAG,CAAC,OAAmB,EAAU,EAAE,CAClD,OAAO,CAAC,KAAK,EAAE,SAAS,IAAI,kBAAkB,CAAC;AAEjD,mEAAmE;AACnE,MAAM,UAAU,GAAG,KAAK,EAAE,OAAe,EAAE,OAAmB,EAAE,EAAE;IAChE,MAAM,EAAE,MAAM,EAAE,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,CAAC;IAEpC,OAAO,CAAC,UAAU,EAAE,CAAC;QACnB,KAAK,EAAE,QAAQ;QACf,OAAO,EAAE,YAAY,MAAM,mBAAmB,MAAM,WAAW;KAChE,CAAC,CAAC;IAEH,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC;QAC1C,aAAa,CAAC,MAAM,EAAE,WAAW,CAAC,OAAO,CAAC,EAAE,OAAO,CAAC,KAAK,CAAC;QAC1D,WAAW,CAAC,MAAM,EAAE,OAAO,CAAC,KAAK,CAAC;KACnC,CAAC,CAAC;IAEH,OAAO,EAAE,MAAM,EAAE,MAAM,CAAC,IAAI,EAAE,UAAU,EAAE,MAAM,CAAC,IAAI,EAAE,OAAO,EAAE,CAAC;AACnE,CAAC,CAAC;AAEF,MAAM,UAAU,GAAG,KAAK,EACtB,GAAW,EACX,OAAmB,EACC,EAAE;IACtB,MAAM,GAAG,GAAG,MAAM,SAAS,CAAC,GAAG,EAAE,OAAO,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC;IAEtD,MAAM,IAAI,GAAG,WAAW,CAAC,GAAG,CAAC,IAAI,IAAI,EAAE,EAAE;QACvC,GAAG,EAAE,GAAG,CAAC,QAAQ;QACjB,iBAAiB,EAAE,OAAO,CAAC,iBAAiB;KAC7C,CAAC,CAAC;IAEH,kGAAkG;IAClG,MAAM,IAAI,GACR,GAAG,CAAC,MAAM,KAAK,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,UAAU,CAAC,GAAG,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;IAEvE,OAAO;QACL,MAAM,EAAE,GAAG;QACX,GAAG,EAAE,GAAG,CAAC,QAAQ;QACjB,MAAM,EAAE,KAAK;QACb,UAAU,EAAE,KAAK;QACjB,OAAO,EAAE;YACP;gBACE,GAAG,EAAE,GAAG,CAAC,QAAQ;gBACjB,YAAY,EAAE,GAAG;gBACjB,IAAI;gBACJ,QAAQ,EAAE,cAAc,CAAC,GAAG,CAAC;gBAC7B,KAAK,EAAE,CAAC;aACT;SACF;QACD,MAAM,EAAE,IAAI,EAAE,MAAM,IAAI,IAAI;QAC5B,UAAU,EAAE,IAAI,EAAE,UAAU,IAAI,IAAI;QACpC,OAAO,EAAE,IAAI,EAAE,OAAO,IAAI,IAAI;QAC9B,IAAI,EAAE,IAAI;KACX,CAAC;AACJ,CAAC,CAAC;AAEF,MAAM,WAAW,GAAG,KAAK,EACvB,GAAW,EACX,OAAmB,EACC,EAAE;IACtB,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,GAAG,EAAE;QAClC,QAAQ,EAAE,OAAO,CAAC,QAAQ,IAAI,aAAa,CAAC,QAAQ;QACpD,QAAQ,EAAE,OAAO,CAAC,QAAQ,IAAI,aAAa,CAAC,QAAQ;QACpD,WAAW,EAAE,OAAO,CAAC,WAAW,IAAI,aAAa,CAAC,WAAW;QAC7D,OAAO,EAAE,OAAO,CAAC,OAAO,IAAI,aAAa,CAAC,OAAO;QACjD,aAAa,EAAE,OAAO,CAAC,aAAa,IAAI,IAAI;QAC5C,iBAAiB,EAAE,OAAO,CAAC,iBAAiB,IAAI,KAAK;QACrD,kBAAkB,EAAE,KAAK;QACzB,iGAAiG;QACjG,aAAa,EAAE,CAAC;QAChB,UAAU,EAAE,KAAK;QACjB,0JAA0J;QAC1J,eAAe,EAAE,CAAC,OAAO,CAAC,eAAe,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,MAAM,CAAC,CAAC,CAAC,CAAC;QAC1E,0JAA0J;QAC1J,eAAe,EAAE,CAAC,OAAO,CAAC,eAAe,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,MAAM,CAAC,CAAC,CAAC,CAAC;QAC1E,KAAK,EAAE,EAAE,GAAG,OAAO,CAAC,KAAK,EAAE,SAAS,EAAE,WAAW,CAAC,OAAO,CAAC,EAAE;QAC5D,UAAU,EAAE,OAAO,CAAC,UAAU;KAC/B,CAAC,CAAC;IAEH,OAAO,CAAC,UAAU,EAAE,CAAC,EAAE,KAAK,EAAE,UAAU,EAAE,OAAO,EAAE,mBAAmB,EAAE,CAAC,CAAC;IAE1E,OAAO;QACL,MAAM,EAAE,GAAG;QACX,GAAG,EAAE,MAAM,CAAC,QAAQ;QACpB,MAAM,EAAE,OAAO;QACf,UAAU,EAAE,KAAK;QACjB,OAAO,EAAE,MAAM,CAAC,OAAO;QACvB,MAAM,EAAE,MAAM,CAAC,MAAM;QACrB,UAAU,EAAE,MAAM,CAAC,UAAU,IAAI,IAAI;QACrC,OAAO,EAAE,MAAM,WAAW,CAAC,IAAI,GAAG,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,MAAM,EAAE,OAAO,CAAC,KAAK,CAAC;QAC1E,IAAI,EAAE,IAAI;KACX,CAAC;AACJ,CAAC,CAAC;AAEF,MAAM,gBAAgB,GAAG,KAAK,EAC5B,GAAW,EACX,OAAmB,EACC,EAAE;IACtB,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,IAAI,mBAAmB,CAAC;IAEvD,MAAM,MAAM,GAAG,MAAM,aAAa,CAAC,GAAG,EAAE;QACtC,OAAO;QACP,iBAAiB,EAAE,OAAO,CAAC,iBAAiB,IAAI,KAAK;QACrD,QAAQ,EAAE,OAAO,CAAC,QAAQ,IAAI,GAAG;QACjC,SAAS,EAAE,OAAO,CAAC,KAAK,EAAE,SAAS;KACpC,CAAC,CAAC;IAEH,OAAO;QACL,MAAM,EAAE,GAAG;QACX,GAAG,EAAE,GAAG;QACR,MAAM,EAAE,WAAW;QACnB,UAAU,EAAE,MAAM;QAClB,OAAO,EAAE,MAAM,CAAC,OAAO;QACvB,MAAM,EAAE,MAAM,CAAC,MAAM;QACrB,UAAU,EAAE,MAAM,CAAC,UAAU,IAAI,IAAI;QACrC,OAAO,EAAE,MAAM,WAAW,CAAC,GAAG,EAAE,OAAO,CAAC;QACxC,IAAI,EAAE,KAAK;KACZ,CAAC;AACJ,CAAC,CAAC;AAEF,mFAAmF;AACnF,MAAM,WAAW,GAAG,CAClB,IAAY,EACZ,GAAW,EACX,MAAc,EACd,MAAuB,EACvB,OAAmB,EACR,EAAE,CAAC,CAAC;IACf,MAAM;IACN,GAAG;IACH,MAAM;IACN,UAAU,EAAE,MAAM;IAClB,OAAO,EAAE;QACP;YACE,GAAG;YACH,YAAY,EAAE,GAAG;YACjB,IAAI,EAAE,WAAW,CAAC,IAAI,EAAE;gBACtB,GAAG;gBACH,iBAAiB,EAAE,OAAO,CAAC,iBAAiB;aAC7C,CAAC;YACF,KAAK,EAAE,CAAC;SACT;KACF;IACD,MAAM,EAAE,IAAI;IACZ,UAAU,EAAE,IAAI;IAChB,OAAO,EAAE,IAAI;IACb,IAAI,EAAE,KAAK;CACZ,CAAC,CAAC;AAEH,MAAM,YAAY,GAAG,CACnB,MAAkB,EAClB,MAAkB,EAClB,OAAmB,EACJ,EAAE;IACjB,IAAI,QAAQ,GAAG,YAAY,CACzB;QACE,IAAI,EAAE,MAAM,CAAC,IAAI;QACjB,MAAM;QACN,QAAQ,EAAE,MAAM,CAAC,QAAQ;QACzB,UAAU,EAAE,kBAAkB;QAC9B,KAAK,EAAE,MAAM,CAAC,KAAK;KACpB,EACD,EAAE,OAAO,EAAE,IAAI,GAAG,CAAC,OAAO,CAAC,YAAY,CAAC,EAAE,KAAK,EAAE,cAAc,EAAE,CAClE,CAAC;IAEF,IAAI,OAAO,CAAC,WAAW,KAAK,KAAK,EAAE,CAAC;QAClC,QAAQ,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,KAAK,MAAM,CAAC,CAAC;IAC3D,CAAC;IAED,MAAM,MAAM,GAAG,aAAa,CAAC,QAAQ,CAAC,CAAC;IAEvC,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,KAAK,EAAE,SAAS,CAAC,MAAM,CAAC,EAAE,CAAC;AAChE,CAAC,CAAC;AAEF;;;GAGG;AACH,MAAM,QAAQ,GAAG,KAAK,EACpB,CAAY,EACZ,KAAgC,EAChC,QAAgC,EAChC,OAAmB,EACU,EAAE;IAC/B,MAAM,QAAQ,GAAG,KAAK,EAAE,MAAM,CAAC,QAAQ,CAAC;IAExC,IAAI,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;QACzB,OAAO,qEAAqE,CAAC;IAC/E,CAAC;IAED,IAAI,QAAQ,CAAC,KAAK,IAAI,QAAQ,CAAC,MAAM,KAAK,IAAI,IAAI,QAAQ,CAAC,MAAM,IAAI,GAAG,EAAE,CAAC;QACzE,OAAO,+BAA+B,QAAQ,CAAC,MAAM,KAAK,IAAI,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,QAAQ,QAAQ,CAAC,MAAM,EAAE,4EAA4E,CAAC;IACvL,CAAC;IAED,IAAI,CAAC,QAAQ,EAAE,CAAC;QACd,OAAO,8BAA8B,CAAC;IACxC,CAAC;IAED,OAAO,CAAC,UAAU,EAAE,CAAC;QACnB,KAAK,EAAE,SAAS;QAChB,OAAO,EAAE,cAAc,QAAQ,CAAC,QAAQ,0BAA0B;KACnE,CAAC,CAAC;IAEH,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,QAAQ,CAAC,QAAQ,EAAE,OAAO,EAAE,OAAO,CAAC,KAAK,CAAC,CAAC;IAC1E,QAAQ,CAAC,aAAa,GAAG;QACvB,SAAS,EAAE,WAAW,CAAC,OAAO,CAAC;QAC/B,MAAM,EAAE,QAAQ,CAAC,MAAM;KACxB,CAAC;IAEF,KAAK,MAAM,GAAG,IAAI,QAAQ,CAAC,IAAI,EAAE,CAAC;QAChC,MAAM,KAAK,GAAG,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QAEnC,IAAI,KAAK,EAAE,CAAC;YACV,GAAG,CAAC,KAAK,GAAG,KAAK,CAAC;QACpB,CAAC;IACH,CAAC;IAED,OAAO,SAAS,CAAC;AACnB,CAAC,CAAC;AAEF,MAAM,WAAW,GAAG,CAClB,QAAgC,EAChC,OAAqB,EACX,EAAE,CACZ,QAAQ;IACN,CAAC,CAAC,QAAQ,CAAC,IAAI;SACV,MAAM,CACL,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,KAAK,OAAO,IAAI,CAAC,CAAC,CAAC,CAAC,OAAO,IAAI,CAAC,CAAC,YAAY,GAAG,CAAC,CAAC,CACnE;SACA,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;IACvB,CAAC,CAAC,EAAE,CAAC;AAET,MAAM,SAAS,GAAG,CAAC,CAAgB,EAAW,EAAE;IAC9C,wEAAwE;IACxE,MAAM,IAAI,GAAmC;QAC3C,GAAG,EAAE,CAAC,CAAC,MAAM,CAAC,IAAI,IAAI,CAAC,CAAC,MAAM,CAAC,GAAG;KACnC,CAAC;IAEF,IAAI,CAAC,CAAC,MAAM,CAAC,KAAK,IAAI,CAAC,EAAE,CAAC;QACxB,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC;IAC9B,CAAC;IAED,OAAO;QACL,GAAG,IAAI;QACP,KAAK,EAAE,CAAC,CAAC,KAAK;QACd,MAAM,EAAE,CAAC,CAAC,MAAM;QAChB,OAAO,EAAE,cAAc,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC;KAC1D,CAAC;AACJ,CAAC,CAAC;AAEF,MAAM,UAAU,GAAG,KAAK,EACtB,CAAY,EACZ,OAAmB,EACA,EAAE;IACrB,MAAM,MAAM,GAAG,CAAC,CAAC,MAAM,KAAK,OAAO,IAAI,CAAC,CAAC,MAAM,KAAK,WAAW,CAAC;IAChE,MAAM,KAAK,GAAG,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,YAAY,CAAC,CAAC,EAAE,CAAC,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC,CAAC;IAC3E,MAAM,KAAK,GAAG,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,KAAK,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,CAAC,CAAC;IAElE,MAAM,QAAQ,GAAG,CAAC,CAAC,MAAM;QACvB,CAAC,CAAC,aAAa,CACX,CAAC,CAAC,MAAM,EACR,CAAC,CAAC,UAAU,EACZ,KAAK,EAAE,MAAM,CAAC,GAAG,IAAI,CAAC,CAAC,GAAG,EAC1B,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,CAC5B;QACH,CAAC,CAAC,IAAI,CAAC;IAET,MAAM,YAAY,GAAG,OAAO,CAAC,MAAM;QACjC,CAAC,CAAC,MAAM,QAAQ,CAAC,CAAC,EAAE,KAAK,EAAE,QAAQ,EAAE,OAAO,CAAC;QAC7C,CAAC,CAAC,SAAS,CAAC;IAEd,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC;IAE/C,IAAI,YAAY,GAAG,eAAe,CAAC;QACjC,QAAQ;QACR,OAAO,EAAE,CAAC,CAAC,OAAO;QAClB,SAAS,EAAE,KAAK,CAAC,MAAM;QACvB,YAAY;KACb,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;IAExC,IAAI,OAAO,CAAC,WAAW,KAAK,KAAK,EAAE,CAAC;QAClC,YAAY,GAAG,YAAY,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,KAAK,MAAM,CAAC,CAAC;IACnE,CAAC;IAED,MAAM,YAAY,GAAG,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC;IAEtD,MAAM,KAAK,GAAa;QACtB,aAAa,EAAE,CAAC;QAChB,IAAI,EAAE,KAAK;QACX,MAAM,EAAE,CAAC,CAAC,MAAM;QAChB,GAAG,EAAE,CAAC,CAAC,GAAG;QACV,MAAM,EAAE,CAAC,CAAC,MAAM;QAChB,KAAK,EAAE,SAAS,CAAC,KAAK,EAAE,YAAY,CAAC;QACrC,MAAM,EAAE,aAAa,CAAC,CAAC,GAAG,YAAY,EAAE,GAAG,YAAY,CAAC,CAAC;QACzD,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;QACnC,OAAO,EAAE;YACP,KAAK,EAAE,KAAK,CAAC,MAAM;YACnB,SAAS,EAAE,QAAQ,EAAE,MAAM,IAAI,IAAI;YACnC,qBAAqB,EAAE,WAAW,CAAC,QAAQ,EAAE,QAAQ,CAAC;YACtD,iBAAiB,EAAE,WAAW,CAAC,QAAQ,EAAE,QAAQ,CAAC;YAClD,mBAAmB,EAAE,WAAW,CAAC,QAAQ,EAAE,UAAU,CAAC;YACtD,kBAAkB,EAAE,QAAQ,EAAE,kBAAkB,IAAI,IAAI;YACxD,eAAe,EAAE,QAAQ,EAAE,aAAa;gBACtC,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;gBAClE,CAAC,CAAC,IAAI;YACR,OAAO,EAAE,CAAC,CAAC,OAAO,EAAE,MAAM,IAAI,IAAI;YAClC,IAAI,EAAE,KAAK;gBACT,CAAC,CAAC,cAAc,CAAC,KAAK,CAAC,MAAM,CAAC,IAAI,EAAE,KAAK,CAAC,MAAM,CAAC,QAAQ,CAAC;gBAC1D,CAAC,CAAC,IAAI;SACT;QACD,QAAQ,EAAE,YAAY,CACpB,MAAM,IAAI,OAAO,CAAC,UAAU,KAAK,SAAS;YACxC,CAAC,CAAC,YAAY;YACd,CAAC,CAAC,CAAC,GAAG,YAAY,EAAE,GAAG,YAAY,CAAC,CACvC;QACD,QAAQ;QACR,OAAO,EAAE,CAAC,CAAC,OAAO;KACnB,CAAC;IAEF,IAAI,MAAM,EAAE,CAAC;QACX,KAAK,CAAC,KAAK,GAAG,KAAK;aAChB,QAAQ,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,GAAG,CAAC,CAAC,MAAM,CAAC,KAAK,IAAI,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC;aACxE,GAAG,CAAC,SAAS,CAAC,CAAC;QAClB,KAAK,CAAC,MAAM,GAAG,YAAY,CACzB,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;YAChB,GAAG,EAAE,CAAC,CAAC,MAAM,CAAC,IAAI,IAAI,CAAC,CAAC,MAAM,CAAC,GAAG;YAClC,QAAQ,EAAE,CAAC,CAAC,QAAQ;SACrB,CAAC,CAAC,CACJ,CAAC;IACJ,CAAC;IAED,OAAO,KAAK,CAAC;AACf,CAAC,CAAC;AAEF,mGAAmG;AACnG,8JAA8J;AAC9J,MAAM,CAAC,MAAM,YAAY,GAAG,KAAK,EAC/B,IAAY,EACZ,KAAK,GAAsC,EAAE,EAC7C,OAAO,GAAe,EAAE,EACL,EAAE;IACrB,MAAM,GAAG,GAAG,KAAK,CAAC,GAAG,IAAI,OAAO,CAAC,OAAO,IAAI,mBAAmB,CAAC;IAEhE,OAAO,UAAU,CACf,WAAW,CAAC,IAAI,EAAE,GAAG,EAAE,KAAK,CAAC,MAAM,IAAI,GAAG,EAAE,MAAM,EAAE,OAAO,CAAC,EAC5D,OAAO,CACR,CAAC;AACJ,CAAC,CAAC;AAEF,2EAA2E;AAC3E,MAAM,CAAC,MAAM,WAAW,GAAG,KAAK,EAC9B,GAAW,EACX,OAAO,GAAe,EAAE,EACL,EAAE,CAAC,UAAU,CAAC,MAAM,UAAU,CAAC,GAAG,EAAE,OAAO,CAAC,EAAE,OAAO,CAAC,CAAC;AAE5E,oDAAoD;AACpD,MAAM,CAAC,MAAM,YAAY,GAAG,KAAK,EAC/B,QAAgB,EAChB,OAAO,GAAe,EAAE,EACL,EAAE,CACrB,UAAU,CAAC,MAAM,WAAW,CAAC,QAAQ,EAAE,OAAO,CAAC,EAAE,OAAO,CAAC,CAAC;AAE5D,sFAAsF;AACtF,MAAM,CAAC,MAAM,iBAAiB,GAAG,KAAK,EACpC,GAAW,EACX,OAAO,GAAe,EAAE,EACL,EAAE,CACrB,UAAU,CAAC,MAAM,gBAAgB,CAAC,GAAG,EAAE,OAAO,CAAC,EAAE,OAAO,CAAC,CAAC;AAE5D,+FAA+F;AAC/F,MAAM,CAAC,MAAM,cAAc,GAAG,KAAK,EACjC,MAAc,EACd,OAAO,GAAwC,EAAE,EAC9B,EAAE;IACrB,IAAI,MAAM,KAAK,GAAG,IAAI,OAAO,CAAC,SAAS,KAAK,SAAS,EAAE,CAAC;QACtD,MAAM,IAAI,GAAG,OAAO,CAAC,SAAS,IAAI,CAAC,MAAM,SAAS,EAAE,CAAC,CAAC;QAEtD,OAAO,YAAY,CACjB,IAAI,EACJ,EAAE,GAAG,EAAE,OAAO,CAAC,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,EACzC,OAAO,CACR,CAAC;IACJ,CAAC;IAED,MAAM,QAAQ,GAAG,MAAM,aAAa,CAAC,MAAM,CAAC,CAAC;IAE7C,QAAQ,QAAQ,CAAC,IAAI,EAAE,CAAC;QACtB,KAAK,KAAK,EAAE,CAAC;YACX,OAAO,OAAO,CAAC,KAAK;gBAClB,CAAC,CAAC,YAAY,CAAC,QAAQ,CAAC,GAAG,EAAE,OAAO,CAAC;gBACrC,CAAC,CAAC,WAAW,CAAC,QAAQ,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;QACzC,CAAC;QAED,KAAK,WAAW,EAAE,CAAC;YACjB,OAAO,iBAAiB,CAAC,QAAQ,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;QACnD,CAAC;QAED,SAAS,CAAC;YACR,MAAM,IAAI,GAAG,OAAO,CAAC,OAAO,IAAI,mBAAmB,CAAC;YAEpD,MAAM,GAAG,GAAG,IAAI,GAAG,CACjB,IAAI,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,EAC3B,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,IAAI,GAAG,CACvC,CAAC,IAAI,CAAC;YAEP,OAAO,UAAU,CACf,WAAW,CACT,MAAM,EAAE,CAAC,QAAQ,CAAC,MAAM,EAAE,OAAO,CAAC,EAClC,GAAG,EACH,MAAM,EACN,MAAM,EACN,OAAO,CACR,EACD,OAAO,CACR,CAAC;QACJ,CAAC;IACH,CAAC;AACH,CAAC,CAAC;AAEF,sEAAsE;AACtE,MAAM,CAAC,MAAM,eAAe,GAAG,CAAC,KAAe,EAAY,EAAE,CAAC,CAAC;IAC7D,GAAG,KAAK;IACR,QAAQ,EAAE,KAAK,CAAC,QAAQ,CAAC,GAAG,CAAC,WAAW,CAAC;IACzC,QAAQ,EAAE,KAAK,CAAC,QAAQ;QACtB,CAAC,CAAC;YACE,GAAG,KAAK,CAAC,QAAQ;YACjB,cAAc,EAAE,KAAK,CAAC,QAAQ,CAAC,cAAc,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC;SAC1D;QACH,CAAC,CAAC,IAAI;IACR,OAAO,EAAE,KAAK,CAAC,OAAO;QACpB,CAAC,CAAC,EAAE,GAAG,KAAK,CAAC,OAAO,EAAE,QAAQ,EAAE,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE;QACpE,CAAC,CAAC,IAAI;CACT,CAAC,CAAC"}
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
import type { AiBotPurpose, AiCrawlerReport, ContentSignal, RobotsTxtInfo } from "../types.js";
|
|
2
|
+
export interface AiBot {
|
|
3
|
+
/** The robots.txt product token. */
|
|
4
|
+
name: string;
|
|
5
|
+
operator: string;
|
|
6
|
+
purpose: AiBotPurpose;
|
|
7
|
+
usedFor: string;
|
|
8
|
+
docs: string;
|
|
9
|
+
/**
|
|
10
|
+
* The User-Agent the probe sends. Absent for robots.txt-only tokens (Google-Extended), and for
|
|
11
|
+
* search crawlers, which CDNs often block when the request does not come from the crawler's IPs.
|
|
12
|
+
*/
|
|
13
|
+
userAgent?: string;
|
|
14
|
+
}
|
|
15
|
+
/**
|
|
16
|
+
* The crawlers that decide whether a site can show up in AI answers, from the operators' own
|
|
17
|
+
* documentation. Order: search crawlers, answer bots, training bots.
|
|
18
|
+
*/
|
|
19
|
+
export declare const AI_BOTS: AiBot[];
|
|
20
|
+
/** The policy token of the shared AI training preference (draft-kelley-aipref-training-preference-signal). */
|
|
21
|
+
export declare const TRAINING_PREFERENCE_TOKEN = "__usage_ai-training";
|
|
22
|
+
/**
|
|
23
|
+
* Content-Signal lines: `Content-Signal: search=yes, ai-input=no` or, for one path,
|
|
24
|
+
* `Content-Signal: /blog/ ai-train=no`.
|
|
25
|
+
*/
|
|
26
|
+
export declare const contentSignals: (text: string) => ContentSignal[];
|
|
27
|
+
/** Signals in a Content-Signal line that the policy does not define, or values other than yes and no. */
|
|
28
|
+
export declare const invalidSignals: (signal: ContentSignal) => string[];
|
|
29
|
+
/** What robots.txt lets each AI bot do on the audited pages; `startUrl` is the page the summary describes. */
|
|
30
|
+
export declare const crawlerAccess: (robots: RobotsTxtInfo, text: string | null, startUrl: string, pageUrls: string[]) => AiCrawlerReport;
|