@agentsean/analyzers 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +661 -0
- package/dist/ai-crawlers.d.ts +19 -0
- package/dist/ai-crawlers.d.ts.map +1 -0
- package/dist/ai-crawlers.js +111 -0
- package/dist/ai-crawlers.js.map +1 -0
- package/dist/audit.d.ts +21 -0
- package/dist/audit.d.ts.map +1 -0
- package/dist/audit.js +87 -0
- package/dist/audit.js.map +1 -0
- package/dist/catalogue-data.d.ts +4 -0
- package/dist/catalogue-data.d.ts.map +1 -0
- package/dist/catalogue-data.js +5103 -0
- package/dist/catalogue-data.js.map +1 -0
- package/dist/catalogue.d.ts +6 -0
- package/dist/catalogue.d.ts.map +1 -0
- package/dist/catalogue.js +25 -0
- package/dist/catalogue.js.map +1 -0
- package/dist/data/checks.json +5078 -0
- package/dist/data/lighthouse-13-insights.json +40 -0
- package/dist/data/schemaorg-vocab.json +89 -0
- package/dist/data/supported_types.json +71 -0
- package/dist/detectors/all.d.ts +30 -0
- package/dist/detectors/all.d.ts.map +1 -0
- package/dist/detectors/all.js +1619 -0
- package/dist/detectors/all.js.map +1 -0
- package/dist/detectors/canon.d.ts +2 -0
- package/dist/detectors/canon.d.ts.map +1 -0
- package/dist/detectors/canon.js +2 -0
- package/dist/detectors/canon.js.map +1 -0
- package/dist/detectors/conf.d.ts +2 -0
- package/dist/detectors/conf.d.ts.map +1 -0
- package/dist/detectors/conf.js +2 -0
- package/dist/detectors/conf.js.map +1 -0
- package/dist/detectors/crawl.d.ts +2 -0
- package/dist/detectors/crawl.d.ts.map +1 -0
- package/dist/detectors/crawl.js +2 -0
- package/dist/detectors/crawl.js.map +1 -0
- package/dist/detectors/cwv.d.ts +2 -0
- package/dist/detectors/cwv.d.ts.map +1 -0
- package/dist/detectors/cwv.js +2 -0
- package/dist/detectors/cwv.js.map +1 -0
- package/dist/detectors/direct.d.ts +2 -0
- package/dist/detectors/direct.d.ts.map +1 -0
- package/dist/detectors/direct.js +2 -0
- package/dist/detectors/direct.js.map +1 -0
- package/dist/detectors/dup.d.ts +2 -0
- package/dist/detectors/dup.d.ts.map +1 -0
- package/dist/detectors/dup.js +2 -0
- package/dist/detectors/dup.js.map +1 -0
- package/dist/detectors/gap.d.ts +2 -0
- package/dist/detectors/gap.d.ts.map +1 -0
- package/dist/detectors/gap.js +2 -0
- package/dist/detectors/gap.js.map +1 -0
- package/dist/detectors/href.d.ts +2 -0
- package/dist/detectors/href.d.ts.map +1 -0
- package/dist/detectors/href.js +2 -0
- package/dist/detectors/href.js.map +1 -0
- package/dist/detectors/img.d.ts +2 -0
- package/dist/detectors/img.d.ts.map +1 -0
- package/dist/detectors/img.js +2 -0
- package/dist/detectors/img.js.map +1 -0
- package/dist/detectors/index.d.ts +27 -0
- package/dist/detectors/index.d.ts.map +1 -0
- package/dist/detectors/index.js +27 -0
- package/dist/detectors/index.js.map +1 -0
- package/dist/detectors/js.d.ts +2 -0
- package/dist/detectors/js.d.ts.map +1 -0
- package/dist/detectors/js.js +2 -0
- package/dist/detectors/js.js.map +1 -0
- package/dist/detectors/link.d.ts +2 -0
- package/dist/detectors/link.d.ts.map +1 -0
- package/dist/detectors/link.js +2 -0
- package/dist/detectors/link.js.map +1 -0
- package/dist/detectors/log.d.ts +2 -0
- package/dist/detectors/log.d.ts.map +1 -0
- package/dist/detectors/log.js +2 -0
- package/dist/detectors/log.js.map +1 -0
- package/dist/detectors/mig.d.ts +2 -0
- package/dist/detectors/mig.d.ts.map +1 -0
- package/dist/detectors/mig.js +2 -0
- package/dist/detectors/mig.js.map +1 -0
- package/dist/detectors/mob.d.ts +2 -0
- package/dist/detectors/mob.d.ts.map +1 -0
- package/dist/detectors/mob.js +2 -0
- package/dist/detectors/mob.js.map +1 -0
- package/dist/detectors/onp.d.ts +2 -0
- package/dist/detectors/onp.d.ts.map +1 -0
- package/dist/detectors/onp.js +2 -0
- package/dist/detectors/onp.js.map +1 -0
- package/dist/detectors/page.d.ts +2 -0
- package/dist/detectors/page.d.ts.map +1 -0
- package/dist/detectors/page.js +2 -0
- package/dist/detectors/page.js.map +1 -0
- package/dist/detectors/param.d.ts +2 -0
- package/dist/detectors/param.d.ts.map +1 -0
- package/dist/detectors/param.js +2 -0
- package/dist/detectors/param.js.map +1 -0
- package/dist/detectors/perf.d.ts +2 -0
- package/dist/detectors/perf.d.ts.map +1 -0
- package/dist/detectors/perf.js +2 -0
- package/dist/detectors/perf.js.map +1 -0
- package/dist/detectors/predicates.d.ts +22 -0
- package/dist/detectors/predicates.d.ts.map +1 -0
- package/dist/detectors/predicates.js +142 -0
- package/dist/detectors/predicates.js.map +1 -0
- package/dist/detectors/resp.d.ts +2 -0
- package/dist/detectors/resp.d.ts.map +1 -0
- package/dist/detectors/resp.js +2 -0
- package/dist/detectors/resp.js.map +1 -0
- package/dist/detectors/robots.d.ts +2 -0
- package/dist/detectors/robots.d.ts.map +1 -0
- package/dist/detectors/robots.js +2 -0
- package/dist/detectors/robots.js.map +1 -0
- package/dist/detectors/sd.d.ts +2 -0
- package/dist/detectors/sd.d.ts.map +1 -0
- package/dist/detectors/sd.js +2 -0
- package/dist/detectors/sd.js.map +1 -0
- package/dist/detectors/sec.d.ts +2 -0
- package/dist/detectors/sec.d.ts.map +1 -0
- package/dist/detectors/sec.js +2 -0
- package/dist/detectors/sec.js.map +1 -0
- package/dist/detectors/smap.d.ts +2 -0
- package/dist/detectors/smap.d.ts.map +1 -0
- package/dist/detectors/smap.js +2 -0
- package/dist/detectors/smap.js.map +1 -0
- package/dist/detectors/soft404.d.ts +2 -0
- package/dist/detectors/soft404.d.ts.map +1 -0
- package/dist/detectors/soft404.js +2 -0
- package/dist/detectors/soft404.js.map +1 -0
- package/dist/detectors/thin.d.ts +2 -0
- package/dist/detectors/thin.d.ts.map +1 -0
- package/dist/detectors/thin.js +2 -0
- package/dist/detectors/thin.js.map +1 -0
- package/dist/index.d.ts +10 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +9 -0
- package/dist/index.js.map +1 -0
- package/dist/openseo-seed.d.ts +16 -0
- package/dist/openseo-seed.d.ts.map +1 -0
- package/dist/openseo-seed.js +157 -0
- package/dist/openseo-seed.js.map +1 -0
- package/dist/priority.d.ts +8 -0
- package/dist/priority.d.ts.map +1 -0
- package/dist/priority.js +53 -0
- package/dist/priority.js.map +1 -0
- package/dist/schemaorg.d.ts +28 -0
- package/dist/schemaorg.d.ts.map +1 -0
- package/dist/schemaorg.js +198 -0
- package/dist/schemaorg.js.map +1 -0
- package/dist/score.d.ts +10 -0
- package/dist/score.d.ts.map +1 -0
- package/dist/score.js +177 -0
- package/dist/score.js.map +1 -0
- package/dist/types.d.ts +109 -0
- package/dist/types.d.ts.map +1 -0
- package/dist/types.js +2 -0
- package/dist/types.js.map +1 -0
- package/package.json +51 -0
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"ai-crawlers.d.ts","sourceRoot":"","sources":["../src/ai-crawlers.ts"],"names":[],"mappings":"AAAA,mFAAmF;AAEnF,MAAM,MAAM,YAAY,GAAG,UAAU,GAAG,UAAU,GAAG,MAAM,GAAG,cAAc,CAAC;AAE7E,MAAM,MAAM,SAAS,GAAG;IACtB,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,YAAY,CAAC;IACpB,aAAa,EAAE,MAAM,CAAC;CACvB,CAAC;AAEF,eAAO,MAAM,WAAW,EAAE,SAAS,SAAS,EAqF3C,CAAC;AAEF,eAAO,MAAM,eAAe,UAE3B,CAAC;AACF,eAAO,MAAM,eAAe,UAE3B,CAAC;AAEF,MAAM,MAAM,mBAAmB,GAAG;IAChC,eAAe,EAAE,MAAM,EAAE,CAAC;IAC1B,eAAe,EAAE,MAAM,EAAE,CAAC;IAC1B,4BAA4B,EAAE,OAAO,CAAC;CACvC,CAAC;AAEF,wBAAgB,iBAAiB,CAAC,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,OAAO,CASrE;AAED,wBAAgB,eAAe,CAAC,GAAG,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,GAAG,mBAAmB,CAUnF"}
|
|
@@ -0,0 +1,111 @@
|
|
|
1
|
+
/** Training crawlers ≠ search/citation crawlers. Conflating them is a landmine. */
|
|
2
|
+
export const AI_CRAWLERS = [
|
|
3
|
+
{
|
|
4
|
+
token: "GPTBot",
|
|
5
|
+
vendor: "OpenAI",
|
|
6
|
+
class: "training",
|
|
7
|
+
blockingCosts: "Foundation-model training only",
|
|
8
|
+
},
|
|
9
|
+
{
|
|
10
|
+
token: "ClaudeBot",
|
|
11
|
+
vendor: "Anthropic",
|
|
12
|
+
class: "training",
|
|
13
|
+
blockingCosts: "Model training",
|
|
14
|
+
},
|
|
15
|
+
{
|
|
16
|
+
token: "Google-Extended",
|
|
17
|
+
vendor: "Google",
|
|
18
|
+
class: "training",
|
|
19
|
+
blockingCosts: "Gemini training + grounding; not Search ranking",
|
|
20
|
+
},
|
|
21
|
+
{
|
|
22
|
+
token: "CCBot",
|
|
23
|
+
vendor: "Common Crawl",
|
|
24
|
+
class: "training",
|
|
25
|
+
blockingCosts: "Common Crawl corpus",
|
|
26
|
+
},
|
|
27
|
+
{
|
|
28
|
+
token: "Applebot-Extended",
|
|
29
|
+
vendor: "Apple",
|
|
30
|
+
class: "training",
|
|
31
|
+
blockingCosts: "Apple Intelligence training",
|
|
32
|
+
},
|
|
33
|
+
{
|
|
34
|
+
token: "Bytespider",
|
|
35
|
+
vendor: "ByteDance",
|
|
36
|
+
class: "training",
|
|
37
|
+
blockingCosts: "ByteDance training",
|
|
38
|
+
},
|
|
39
|
+
{
|
|
40
|
+
token: "OAI-SearchBot",
|
|
41
|
+
vendor: "OpenAI",
|
|
42
|
+
class: "citation",
|
|
43
|
+
blockingCosts: "ChatGPT search citations",
|
|
44
|
+
},
|
|
45
|
+
{
|
|
46
|
+
token: "Claude-SearchBot",
|
|
47
|
+
vendor: "Anthropic",
|
|
48
|
+
class: "citation",
|
|
49
|
+
blockingCosts: "Claude search citations",
|
|
50
|
+
},
|
|
51
|
+
{
|
|
52
|
+
token: "PerplexityBot",
|
|
53
|
+
vendor: "Perplexity",
|
|
54
|
+
class: "citation",
|
|
55
|
+
blockingCosts: "Perplexity citations",
|
|
56
|
+
},
|
|
57
|
+
{
|
|
58
|
+
token: "Bingbot",
|
|
59
|
+
vendor: "Microsoft",
|
|
60
|
+
class: "citation",
|
|
61
|
+
blockingCosts: "Bing + Copilot grounding",
|
|
62
|
+
},
|
|
63
|
+
{
|
|
64
|
+
token: "ChatGPT-User",
|
|
65
|
+
vendor: "OpenAI",
|
|
66
|
+
class: "user",
|
|
67
|
+
blockingCosts: "Live browsing on user request",
|
|
68
|
+
},
|
|
69
|
+
{
|
|
70
|
+
token: "Claude-User",
|
|
71
|
+
vendor: "Anthropic",
|
|
72
|
+
class: "user",
|
|
73
|
+
blockingCosts: "Claude live browsing",
|
|
74
|
+
},
|
|
75
|
+
{
|
|
76
|
+
token: "Perplexity-User",
|
|
77
|
+
vendor: "Perplexity",
|
|
78
|
+
class: "user",
|
|
79
|
+
blockingCosts: "User-initiated fetch (often ignores robots.txt)",
|
|
80
|
+
},
|
|
81
|
+
{
|
|
82
|
+
token: "Googlebot",
|
|
83
|
+
vendor: "Google",
|
|
84
|
+
class: "search_index",
|
|
85
|
+
blockingCosts: "Search, including AI Overviews / AI Mode eligibility",
|
|
86
|
+
},
|
|
87
|
+
];
|
|
88
|
+
export const TRAINING_TOKENS = AI_CRAWLERS.filter((c) => c.class === "training").map((c) => c.token.toLowerCase());
|
|
89
|
+
export const CITATION_TOKENS = AI_CRAWLERS.filter((c) => c.class === "citation").map((c) => c.token.toLowerCase());
|
|
90
|
+
export function groupDisallowsAll(raw, token) {
|
|
91
|
+
const lower = raw.toLowerCase();
|
|
92
|
+
const needle = `user-agent: ${token.toLowerCase()}`;
|
|
93
|
+
const idx = lower.indexOf(needle);
|
|
94
|
+
if (idx < 0)
|
|
95
|
+
return false;
|
|
96
|
+
const rest = lower.slice(idx);
|
|
97
|
+
const nextUa = rest.slice(needle.length).search(/\nuser-agent:/);
|
|
98
|
+
const block = nextUa >= 0 ? rest.slice(0, needle.length + nextUa) : rest;
|
|
99
|
+
return /disallow:\s*\/\s*$/m.test(block) || /disallow:\s*\/\s*\n/.test(block);
|
|
100
|
+
}
|
|
101
|
+
export function analyzeAiRobots(raw) {
|
|
102
|
+
const text = raw ?? "";
|
|
103
|
+
const blockedTraining = TRAINING_TOKENS.filter((t) => groupDisallowsAll(text, t));
|
|
104
|
+
const blockedCitation = CITATION_TOKENS.filter((t) => groupDisallowsAll(text, t));
|
|
105
|
+
return {
|
|
106
|
+
blockedTraining,
|
|
107
|
+
blockedCitation,
|
|
108
|
+
conflatesTrainingAndCitation: blockedTraining.length > 0 && blockedCitation.length > 0,
|
|
109
|
+
};
|
|
110
|
+
}
|
|
111
|
+
//# sourceMappingURL=ai-crawlers.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"ai-crawlers.js","sourceRoot":"","sources":["../src/ai-crawlers.ts"],"names":[],"mappings":"AAAA,mFAAmF;AAWnF,MAAM,CAAC,MAAM,WAAW,GAAyB;IAC/C;QACE,KAAK,EAAE,QAAQ;QACf,MAAM,EAAE,QAAQ;QAChB,KAAK,EAAE,UAAU;QACjB,aAAa,EAAE,gCAAgC;KAChD;IACD;QACE,KAAK,EAAE,WAAW;QAClB,MAAM,EAAE,WAAW;QACnB,KAAK,EAAE,UAAU;QACjB,aAAa,EAAE,gBAAgB;KAChC;IACD;QACE,KAAK,EAAE,iBAAiB;QACxB,MAAM,EAAE,QAAQ;QAChB,KAAK,EAAE,UAAU;QACjB,aAAa,EAAE,iDAAiD;KACjE;IACD;QACE,KAAK,EAAE,OAAO;QACd,MAAM,EAAE,cAAc;QACtB,KAAK,EAAE,UAAU;QACjB,aAAa,EAAE,qBAAqB;KACrC;IACD;QACE,KAAK,EAAE,mBAAmB;QAC1B,MAAM,EAAE,OAAO;QACf,KAAK,EAAE,UAAU;QACjB,aAAa,EAAE,6BAA6B;KAC7C;IACD;QACE,KAAK,EAAE,YAAY;QACnB,MAAM,EAAE,WAAW;QACnB,KAAK,EAAE,UAAU;QACjB,aAAa,EAAE,oBAAoB;KACpC;IACD;QACE,KAAK,EAAE,eAAe;QACtB,MAAM,EAAE,QAAQ;QAChB,KAAK,EAAE,UAAU;QACjB,aAAa,EAAE,0BAA0B;KAC1C;IACD;QACE,KAAK,EAAE,kBAAkB;QACzB,MAAM,EAAE,WAAW;QACnB,KAAK,EAAE,UAAU;QACjB,aAAa,EAAE,yBAAyB;KACzC;IACD;QACE,KAAK,EAAE,eAAe;QACtB,MAAM,EAAE,YAAY;QACpB,KAAK,EAAE,UAAU;QACjB,aAAa,EAAE,sBAAsB;KACtC;IACD;QACE,KAAK,EAAE,SAAS;QAChB,MAAM,EAAE,WAAW;QACnB,KAAK,EAAE,UAAU;QACjB,aAAa,EAAE,0BAA0B;KAC1C;IACD;QACE,KAAK,EAAE,cAAc;QACrB,MAAM,EAAE,QAAQ;QAChB,KAAK,EAAE,MAAM;QACb,aAAa,EAAE,+BAA+B;KAC/C;IACD;QACE,KAAK,EAAE,aAAa;QACpB,MAAM,EAAE,WAAW;QACnB,KAAK,EAAE,MAAM;QACb,aAAa,EAAE,sBAAsB;KACtC;IACD;QACE,KAAK,EAAE,iBAAiB;QACxB,MAAM,EAAE,YAAY;QACpB,KAAK,EAAE,MAAM;QACb,aAAa,EAAE,iDAAiD;KACjE;IACD;QACE,KAAK,EAAE,WAAW;QAClB,MAAM,EAAE,QAAQ;QAChB,KAAK,EAAE,cAAc;QACrB,aAAa,EAAE,sDAAsD;KACtE;CACF,CAAC;AAEF,MAAM,CAAC,MAAM,eAAe,GAAG,WAAW,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,KAAK,UAAU,CAAC,CAAC,GAAG,CAClF,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,WAAW,EAAE,CAC7B,CAAC;AACF,MAAM,CAAC,MAAM,eAAe,GAAG,WAAW,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,KAAK,UAAU,CAAC,CAAC,GAAG,CAClF,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,WAAW,EAAE,CAC7B,CAAC;AAQF,MAAM,UAAU,iBAAiB,CAAC,GAAW,EAAE,KAAa;IAC1D,MAAM,KAAK,GAAG,GAAG,CAAC,WAAW,EAAE,CAAC;IAChC,MAAM,MAAM,GAAG,eAAe,KAAK,CAAC,WAAW,EAAE,EAAE,CAAC;IACpD,MAAM,GAAG,GAAG,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;IAClC,IAAI,GAAG,GAAG,CAAC;QAAE,OAAO,KAAK,CAAC;IAC1B,MAAM,IAAI,GAAG,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IAC9B,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC,eAAe,CAAC,CAAC;IACjE,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,MAAM,CAAC,MAAM,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;IACzE,OAAO,qBAAqB,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,qBAAqB,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;AAChF,CAAC;AAED,MAAM,UAAU,eAAe,CAAC,GAA8B;IAC5D,MAAM,IAAI,GAAG,GAAG,IAAI,EAAE,CAAC;IACvB,MAAM,eAAe,GAAG,eAAe,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,iBAAiB,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;IAClF,MAAM,eAAe,GAAG,eAAe,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,iBAAiB,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;IAClF,OAAO;QACL,eAAe;QACf,eAAe;QACf,4BAA4B,EAC1B,eAAe,CAAC,MAAM,GAAG,CAAC,IAAI,eAAe,CAAC,MAAM,GAAG,CAAC;KAC3D,CAAC;AACJ,CAAC"}
|
package/dist/audit.d.ts
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
import type { CrawlResult } from "@agentsean/crawler";
|
|
2
|
+
import { computeSiteScore } from "./score.js";
|
|
3
|
+
import type { AuditContext, AuditReport, PrioritizedFinding } from "./types.js";
|
|
4
|
+
export declare function analyzeCrawl(crawl: CrawlResult, extra?: Omit<AuditContext, "crawl">): {
|
|
5
|
+
findings: PrioritizedFinding[];
|
|
6
|
+
score: ReturnType<typeof computeSiteScore>;
|
|
7
|
+
};
|
|
8
|
+
export declare function buildReport(crawl: CrawlResult, elapsedMs: number, extra?: Omit<AuditContext, "crawl">): AuditReport;
|
|
9
|
+
export declare function findingFingerprint(siteId: string, ruleId: string, url: string | null): string;
|
|
10
|
+
export declare function flattenForDb(siteId: string, findings: PrioritizedFinding[]): {
|
|
11
|
+
pageUrl: string | null;
|
|
12
|
+
ruleId: string;
|
|
13
|
+
severity: string;
|
|
14
|
+
autonomyTier: string;
|
|
15
|
+
title: string;
|
|
16
|
+
explanation: string | null;
|
|
17
|
+
evidence: unknown;
|
|
18
|
+
fingerprint: string;
|
|
19
|
+
}[];
|
|
20
|
+
export declare function catalogueSize(): number;
|
|
21
|
+
//# sourceMappingURL=audit.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"audit.d.ts","sourceRoot":"","sources":["../src/audit.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAC;AAItD,OAAO,EAAE,gBAAgB,EAA0C,MAAM,YAAY,CAAC;AAGtF,OAAO,KAAK,EAAE,YAAY,EAAE,WAAW,EAAE,kBAAkB,EAAE,MAAM,YAAY,CAAC;AAEhF,wBAAgB,YAAY,CAC1B,KAAK,EAAE,WAAW,EAClB,KAAK,CAAC,EAAE,IAAI,CAAC,YAAY,EAAE,OAAO,CAAC,GAClC;IACD,QAAQ,EAAE,kBAAkB,EAAE,CAAC;IAC/B,KAAK,EAAE,UAAU,CAAC,OAAO,gBAAgB,CAAC,CAAC;CAC5C,CAaA;AAED,wBAAgB,WAAW,CACzB,KAAK,EAAE,WAAW,EAClB,SAAS,EAAE,MAAM,EACjB,KAAK,CAAC,EAAE,IAAI,CAAC,YAAY,EAAE,OAAO,CAAC,GAClC,WAAW,CAcb;AAED,wBAAgB,kBAAkB,CAChC,MAAM,EAAE,MAAM,EACd,MAAM,EAAE,MAAM,EACd,GAAG,EAAE,MAAM,GAAG,IAAI,GACjB,MAAM,CAKR;AAED,wBAAgB,YAAY,CAC1B,MAAM,EAAE,MAAM,EACd,QAAQ,EAAE,kBAAkB,EAAE,GAC7B;IACD,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC;IACvB,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,MAAM,CAAC;IACjB,YAAY,EAAE,MAAM,CAAC;IACrB,KAAK,EAAE,MAAM,CAAC;IACd,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,QAAQ,EAAE,OAAO,CAAC;IAClB,WAAW,EAAE,MAAM,CAAC;CACrB,EAAE,CAkCF;AASD,wBAAgB,aAAa,IAAI,MAAM,CAEtC"}
|
package/dist/audit.js
ADDED
|
@@ -0,0 +1,87 @@
|
|
|
1
|
+
import { createHash } from "node:crypto";
|
|
2
|
+
import { CHECKS } from "./catalogue.js";
|
|
3
|
+
import { detectAll } from "./detectors/all.js";
|
|
4
|
+
import { prioritize } from "./priority.js";
|
|
5
|
+
import { computeSiteScore, SITE_SCORE_FORMULA, SITE_SCORE_VERSION } from "./score.js";
|
|
6
|
+
import { PRIORITY_FORMULA } from "./priority.js";
|
|
7
|
+
import { isIndexable } from "./detectors/predicates.js";
|
|
8
|
+
export function analyzeCrawl(crawl, extra) {
|
|
9
|
+
const ctx = { crawl, ...extra };
|
|
10
|
+
const drafts = detectAll(ctx);
|
|
11
|
+
const indexable = new Set(crawl.pages.filter(isIndexable).map((p) => p.url));
|
|
12
|
+
const findings = prioritize(drafts, crawl.pages.length, {
|
|
13
|
+
gscClicksByUrl: extra?.gsc?.clicksByUrl,
|
|
14
|
+
indexableUrls: indexable,
|
|
15
|
+
});
|
|
16
|
+
const score = computeSiteScore(crawl, drafts, {
|
|
17
|
+
hasGsc: Boolean(extra?.gsc),
|
|
18
|
+
hasCrux: Boolean(extra?.crux),
|
|
19
|
+
});
|
|
20
|
+
return { findings, score };
|
|
21
|
+
}
|
|
22
|
+
export function buildReport(crawl, elapsedMs, extra) {
|
|
23
|
+
const { findings, score } = analyzeCrawl(crawl, extra);
|
|
24
|
+
return {
|
|
25
|
+
url: crawl.startUrl,
|
|
26
|
+
origin: crawl.origin,
|
|
27
|
+
pages: crawl.pagesSeen,
|
|
28
|
+
elapsedMs,
|
|
29
|
+
score,
|
|
30
|
+
findings,
|
|
31
|
+
formula: {
|
|
32
|
+
siteScore: `${SITE_SCORE_VERSION}: ${SITE_SCORE_FORMULA.split("\n")[0]}`,
|
|
33
|
+
priority: PRIORITY_FORMULA,
|
|
34
|
+
},
|
|
35
|
+
};
|
|
36
|
+
}
|
|
37
|
+
export function findingFingerprint(siteId, ruleId, url) {
|
|
38
|
+
return createHash("sha256")
|
|
39
|
+
.update(`${siteId}|${ruleId}|${url ?? "*"}`)
|
|
40
|
+
.digest("hex")
|
|
41
|
+
.slice(0, 32);
|
|
42
|
+
}
|
|
43
|
+
export function flattenForDb(siteId, findings) {
|
|
44
|
+
const rows = [];
|
|
45
|
+
for (const f of findings) {
|
|
46
|
+
if (f.urls.length <= 1) {
|
|
47
|
+
const url = f.urls[0] ?? null;
|
|
48
|
+
rows.push({
|
|
49
|
+
pageUrl: url,
|
|
50
|
+
ruleId: f.ruleId,
|
|
51
|
+
severity: f.severity,
|
|
52
|
+
autonomyTier: f.autonomyTier,
|
|
53
|
+
title: f.title,
|
|
54
|
+
explanation: f.explanation,
|
|
55
|
+
evidence: { ...asObj(f.evidence), priority: f.priority },
|
|
56
|
+
fingerprint: findingFingerprint(siteId, f.ruleId, url),
|
|
57
|
+
});
|
|
58
|
+
continue;
|
|
59
|
+
}
|
|
60
|
+
rows.push({
|
|
61
|
+
pageUrl: null,
|
|
62
|
+
ruleId: f.ruleId,
|
|
63
|
+
severity: f.severity,
|
|
64
|
+
autonomyTier: f.autonomyTier,
|
|
65
|
+
title: f.title,
|
|
66
|
+
explanation: f.explanation,
|
|
67
|
+
evidence: {
|
|
68
|
+
...asObj(f.evidence),
|
|
69
|
+
urls: f.urls,
|
|
70
|
+
priority: f.priority,
|
|
71
|
+
count: f.urls.length,
|
|
72
|
+
},
|
|
73
|
+
fingerprint: findingFingerprint(siteId, f.ruleId, null),
|
|
74
|
+
});
|
|
75
|
+
}
|
|
76
|
+
return rows;
|
|
77
|
+
}
|
|
78
|
+
function asObj(evidence) {
|
|
79
|
+
if (evidence && typeof evidence === "object" && !Array.isArray(evidence)) {
|
|
80
|
+
return evidence;
|
|
81
|
+
}
|
|
82
|
+
return { value: evidence };
|
|
83
|
+
}
|
|
84
|
+
export function catalogueSize() {
|
|
85
|
+
return CHECKS.length;
|
|
86
|
+
}
|
|
87
|
+
//# sourceMappingURL=audit.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"audit.js","sourceRoot":"","sources":["../src/audit.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAEzC,OAAO,EAAE,MAAM,EAAE,MAAM,gBAAgB,CAAC;AACxC,OAAO,EAAE,SAAS,EAAE,MAAM,oBAAoB,CAAC;AAC/C,OAAO,EAAE,UAAU,EAAE,MAAM,eAAe,CAAC;AAC3C,OAAO,EAAE,gBAAgB,EAAE,kBAAkB,EAAE,kBAAkB,EAAE,MAAM,YAAY,CAAC;AACtF,OAAO,EAAE,gBAAgB,EAAE,MAAM,eAAe,CAAC;AACjD,OAAO,EAAE,WAAW,EAAE,MAAM,2BAA2B,CAAC;AAGxD,MAAM,UAAU,YAAY,CAC1B,KAAkB,EAClB,KAAmC;IAKnC,MAAM,GAAG,GAAiB,EAAE,KAAK,EAAE,GAAG,KAAK,EAAE,CAAC;IAC9C,MAAM,MAAM,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC;IAC9B,MAAM,SAAS,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IAC7E,MAAM,QAAQ,GAAG,UAAU,CAAC,MAAM,EAAE,KAAK,CAAC,KAAK,CAAC,MAAM,EAAE;QACtD,cAAc,EAAE,KAAK,EAAE,GAAG,EAAE,WAAW;QACvC,aAAa,EAAE,SAAS;KACzB,CAAC,CAAC;IACH,MAAM,KAAK,GAAG,gBAAgB,CAAC,KAAK,EAAE,MAAM,EAAE;QAC5C,MAAM,EAAE,OAAO,CAAC,KAAK,EAAE,GAAG,CAAC;QAC3B,OAAO,EAAE,OAAO,CAAC,KAAK,EAAE,IAAI,CAAC;KAC9B,CAAC,CAAC;IACH,OAAO,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC;AAC7B,CAAC;AAED,MAAM,UAAU,WAAW,CACzB,KAAkB,EAClB,SAAiB,EACjB,KAAmC;IAEnC,MAAM,EAAE,QAAQ,EAAE,KAAK,EAAE,GAAG,YAAY,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC;IACvD,OAAO;QACL,GAAG,EAAE,KAAK,CAAC,QAAQ;QACnB,MAAM,EAAE,KAAK,CAAC,MAAM;QACpB,KAAK,EAAE,KAAK,CAAC,SAAS;QACtB,SAAS;QACT,KAAK;QACL,QAAQ;QACR,OAAO,EAAE;YACP,SAAS,EAAE,GAAG,kBAAkB,KAAK,kBAAkB,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,EAAE;YACxE,QAAQ,EAAE,gBAAgB;SAC3B;KACF,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,kBAAkB,CAChC,MAAc,EACd,MAAc,EACd,GAAkB;IAElB,OAAO,UAAU,CAAC,QAAQ,CAAC;SACxB,MAAM,CAAC,GAAG,MAAM,IAAI,MAAM,IAAI,GAAG,IAAI,GAAG,EAAE,CAAC;SAC3C,MAAM,CAAC,KAAK,CAAC;SACb,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;AAClB,CAAC;AAED,MAAM,UAAU,YAAY,CAC1B,MAAc,EACd,QAA8B;IAW9B,MAAM,IAAI,GAAG,EAAE,CAAC;IAChB,KAAK,MAAM,CAAC,IAAI,QAAQ,EAAE,CAAC;QACzB,IAAI,CAAC,CAAC,IAAI,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;YACvB,MAAM,GAAG,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC;YAC9B,IAAI,CAAC,IAAI,CAAC;gBACR,OAAO,EAAE,GAAG;gBACZ,MAAM,EAAE,CAAC,CAAC,MAAM;gBAChB,QAAQ,EAAE,CAAC,CAAC,QAAQ;gBACpB,YAAY,EAAE,CAAC,CAAC,YAAY;gBAC5B,KAAK,EAAE,CAAC,CAAC,KAAK;gBACd,WAAW,EAAE,CAAC,CAAC,WAAW;gBAC1B,QAAQ,EAAE,EAAE,GAAG,KAAK,CAAC,CAAC,CAAC,QAAQ,CAAC,EAAE,QAAQ,EAAE,CAAC,CAAC,QAAQ,EAAE;gBACxD,WAAW,EAAE,kBAAkB,CAAC,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,GAAG,CAAC;aACvD,CAAC,CAAC;YACH,SAAS;QACX,CAAC;QACD,IAAI,CAAC,IAAI,CAAC;YACR,OAAO,EAAE,IAAI;YACb,MAAM,EAAE,CAAC,CAAC,MAAM;YAChB,QAAQ,EAAE,CAAC,CAAC,QAAQ;YACpB,YAAY,EAAE,CAAC,CAAC,YAAY;YAC5B,KAAK,EAAE,CAAC,CAAC,KAAK;YACd,WAAW,EAAE,CAAC,CAAC,WAAW;YAC1B,QAAQ,EAAE;gBACR,GAAG,KAAK,CAAC,CAAC,CAAC,QAAQ,CAAC;gBACpB,IAAI,EAAE,CAAC,CAAC,IAAI;gBACZ,QAAQ,EAAE,CAAC,CAAC,QAAQ;gBACpB,KAAK,EAAE,CAAC,CAAC,IAAI,CAAC,MAAM;aACrB;YACD,WAAW,EAAE,kBAAkB,CAAC,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,IAAI,CAAC;SACxD,CAAC,CAAC;IACL,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,SAAS,KAAK,CAAC,QAAiB;IAC9B,IAAI,QAAQ,IAAI,OAAO,QAAQ,KAAK,QAAQ,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,EAAE,CAAC;QACzE,OAAO,QAAmC,CAAC;IAC7C,CAAC;IACD,OAAO,EAAE,KAAK,EAAE,QAAQ,EAAE,CAAC;AAC7B,CAAC;AAED,MAAM,UAAU,aAAa;IAC3B,OAAO,MAAM,CAAC,MAAM,CAAC;AACvB,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"catalogue-data.d.ts","sourceRoot":"","sources":["../src/catalogue-data.ts"],"names":[],"mappings":"AAAA,6EAA6E;AAC7E,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,YAAY,CAAC;AAElD,eAAO,MAAM,MAAM,EAAE,SAAS,eAAe,EA6gK5C,CAAC"}
|