@rivi-software/agentic-seo-check 0.2.0 → 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (76) hide show
  1. package/README.md +8 -4
  2. package/dist/core/aeo/page.d.ts +0 -10
  3. package/dist/core/aeo/page.js +1 -29
  4. package/dist/core/aeo/page.js.map +1 -1
  5. package/dist/core/audit.js +3 -0
  6. package/dist/core/audit.js.map +1 -1
  7. package/dist/core/crawl.d.ts +2 -0
  8. package/dist/core/crawl.js +61 -4
  9. package/dist/core/crawl.js.map +1 -1
  10. package/dist/core/fetch.d.ts +9 -0
  11. package/dist/core/fetch.js +28 -12
  12. package/dist/core/fetch.js.map +1 -1
  13. package/dist/core/html/dom.d.ts +2 -0
  14. package/dist/core/html/dom.js +26 -1
  15. package/dist/core/html/dom.js.map +1 -1
  16. package/dist/core/html/extract.js +18 -3
  17. package/dist/core/html/extract.js.map +1 -1
  18. package/dist/core/html/head.d.ts +10 -0
  19. package/dist/core/html/head.js +206 -0
  20. package/dist/core/html/head.js.map +1 -0
  21. package/dist/core/html/resources.d.ts +17 -0
  22. package/dist/core/html/resources.js +172 -0
  23. package/dist/core/html/resources.js.map +1 -0
  24. package/dist/core/html/structured-data.d.ts +2 -0
  25. package/dist/core/html/structured-data.js +2 -0
  26. package/dist/core/html/structured-data.js.map +1 -1
  27. package/dist/core/robots.js +1 -0
  28. package/dist/core/robots.js.map +1 -1
  29. package/dist/core/rules/aeo.js +2 -2
  30. package/dist/core/rules/aeo.js.map +1 -1
  31. package/dist/core/rules/canonical.d.ts +3 -0
  32. package/dist/core/rules/canonical.js +54 -2
  33. package/dist/core/rules/canonical.js.map +1 -1
  34. package/dist/core/rules/document.js +4 -2
  35. package/dist/core/rules/document.js.map +1 -1
  36. package/dist/core/rules/head.d.ts +1 -0
  37. package/dist/core/rules/head.js +116 -0
  38. package/dist/core/rules/head.js.map +1 -0
  39. package/dist/core/rules/index.js +8 -0
  40. package/dist/core/rules/index.js.map +1 -1
  41. package/dist/core/rules/indexing.d.ts +10 -0
  42. package/dist/core/rules/indexing.js +171 -9
  43. package/dist/core/rules/indexing.js.map +1 -1
  44. package/dist/core/rules/loading.d.ts +1 -0
  45. package/dist/core/rules/loading.js +138 -0
  46. package/dist/core/rules/loading.js.map +1 -0
  47. package/dist/core/rules/rule.d.ts +2 -0
  48. package/dist/core/rules/rule.js +2 -0
  49. package/dist/core/rules/rule.js.map +1 -1
  50. package/dist/core/rules/structured-data-values.d.ts +3 -0
  51. package/dist/core/rules/structured-data-values.js +316 -0
  52. package/dist/core/rules/structured-data-values.js.map +1 -0
  53. package/dist/core/rules/structured-data.js +50 -20
  54. package/dist/core/rules/structured-data.js.map +1 -1
  55. package/dist/core/rules/urls.d.ts +1 -0
  56. package/dist/core/rules/urls.js +293 -0
  57. package/dist/core/rules/urls.js.map +1 -0
  58. package/dist/core/site-probes.d.ts +33 -0
  59. package/dist/core/site-probes.js +144 -0
  60. package/dist/core/site-probes.js.map +1 -0
  61. package/dist/core/site-rules.d.ts +18 -0
  62. package/dist/core/site-rules.js +479 -5
  63. package/dist/core/site-rules.js.map +1 -1
  64. package/dist/core/sitemap.d.ts +38 -1
  65. package/dist/core/sitemap.js +177 -43
  66. package/dist/core/sitemap.js.map +1 -1
  67. package/dist/core/static.d.ts +6 -0
  68. package/dist/core/static.js +121 -12
  69. package/dist/core/static.js.map +1 -1
  70. package/dist/core/types.d.ts +91 -0
  71. package/dist/core/types.js.map +1 -1
  72. package/dist/core/url.d.ts +2 -0
  73. package/dist/core/url.js +8 -0
  74. package/dist/core/url.js.map +1 -1
  75. package/package.json +1 -1
  76. package/skills/seo-audit/SKILL.md +1 -0
@@ -4,6 +4,8 @@ export type { AnyNode, Document, Element } from "domhandler";
4
4
  export declare const HIDDEN_ELEMENTS: Set<string>;
5
5
  /** The text a reader sees under `root`: hidden elements left out, whitespace collapsed. */
6
6
  export declare const visibleText: (root: AnyNode) => string;
7
+ /** For each <img> under `root`, the number of visible words that come before it in the document. */
8
+ export declare const wordsBeforeImages: (root: AnyNode) => Map<Element, number>;
7
9
  /** Parse a full HTML document with the WHATWG algorithm (parse5), scripting enabled like a browser. */
8
10
  export declare const parseDocument: (html: string) => Document;
9
11
  /** Descendants of `root` matching a CSS selector, in document order. */
@@ -3,7 +3,7 @@ import { hasChildren, isTag, isText } from "domhandler";
3
3
  import { textContent } from "domutils";
4
4
  import { parse } from "parse5";
5
5
  import { adapter } from "parse5-htmlparser2-tree-adapter";
6
- import { collapseWhitespace } from "./text.js";
6
+ import { collapseWhitespace, wordCount } from "./text.js";
7
7
  /** Elements whose content is never visible page text. */
8
8
  export const HIDDEN_ELEMENTS = new Set([
9
9
  "script",
@@ -69,6 +69,31 @@ export const visibleText = (root) => {
69
69
  walk(hasChildren(root) ? root.children : [root]);
70
70
  return collapseWhitespace(parts.join(""));
71
71
  };
72
+ /** For each <img> under `root`, the number of visible words that come before it in the document. */
73
+ export const wordsBeforeImages = (root) => {
74
+ const out = new Map();
75
+ let words = 0;
76
+ const walk = (nodes) => {
77
+ for (const node of nodes) {
78
+ if (isText(node)) {
79
+ words += wordCount(node.data);
80
+ }
81
+ else if (isTag(node)) {
82
+ if (node.name === "img") {
83
+ out.set(node, words);
84
+ }
85
+ else if (!HIDDEN_ELEMENTS.has(node.name)) {
86
+ walk(node.children);
87
+ }
88
+ }
89
+ else if (hasChildren(node)) {
90
+ walk(node.children);
91
+ }
92
+ }
93
+ };
94
+ walk(hasChildren(root) ? root.children : [root]);
95
+ return out;
96
+ };
72
97
  /** Parse a full HTML document with the WHATWG algorithm (parse5), scripting enabled like a browser. */
73
98
  export const parseDocument = (html) => parse(html, { scriptingEnabled: true, treeAdapter: adapter });
74
99
  /** Descendants of `root` matching a CSS selector, in document order. */
@@ -1 +1 @@
1
- {"version":3,"file":"dom.js","sourceRoot":"","sources":["../../../src/core/html/dom.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AAClD,OAAO,EAAE,WAAW,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,YAAY,CAAC;AAExD,OAAO,EAAE,WAAW,EAAE,MAAM,UAAU,CAAC;AACvC,OAAO,EAAE,KAAK,EAAE,MAAM,QAAQ,CAAC;AAC/B,OAAO,EAAE,OAAO,EAAE,MAAM,iCAAiC,CAAC;AAE1D,OAAO,EAAE,kBAAkB,EAAE,MAAM,WAAW,CAAC;AAI/C,yDAAyD;AACzD,MAAM,CAAC,MAAM,eAAe,GAAG,IAAI,GAAG,CAAC;IACrC,QAAQ;IACR,OAAO;IACP,UAAU;IACV,UAAU;IACV,KAAK;IACL,QAAQ;IACR,MAAM;CACP,CAAC,CAAC;AAEH,wFAAwF;AACxF,MAAM,cAAc,GAAG,IAAI,GAAG,CAAC;IAC7B,GAAG;IACH,KAAK;IACL,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,SAAS;IACT,SAAS;IACT,QAAQ;IACR,QAAQ;IACR,KAAK;IACL,YAAY;IACZ,KAAK;IACL,IAAI;IACJ,IAAI;IACJ,YAAY;CACb,CAAC,CAAC;AAEH,2FAA2F;AAC3F,MAAM,CAAC,MAAM,WAAW,GAAG,CAAC,IAAa,EAAU,EAAE;IACnD,MAAM,KAAK,GAAa,EAAE,CAAC;IAE3B,MAAM,IAAI,GAAG,CAAC,KAAgB,EAAQ,EAAE;QACtC,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,IAAI,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC;gBACjB,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACxB,CAAC;iBAAM,IAAI,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC;gBACvB,IAAI,eAAe,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;oBACnC,SAAS;gBACX,CAAC;gBAED,MAAM,KAAK,GAAG,cAAc,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;gBAE5C,IAAI,KAAK,EAAE,CAAC;oBACV,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;gBAClB,CAAC;gBAED,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;gBAEpB,IAAI,KAAK,EAAE,CAAC;oBACV,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;gBAClB,CAAC;YACH,CAAC;iBAAM,IAAI,WAAW,CAAC,IAAI,CAAC,EAAE,CAAC;gBAC7B,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;YACtB,CAAC;QACH,CAAC;IACH,CAAC,CAAC;IAEF,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;IAEjD,OAAO,kBAAkB,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC;AAC5C,CAAC,CAAC;AAEF,uGAAuG;AACvG,MAAM,CAAC,MAAM,aAAa,GAAG,CAAC,IAAY,EAAY,EAAE,CACtD,KAAK,CAAC,IAAI,EAAE,EAAE,gBAAgB,EAAE,IAAI,EAAE,WAAW,EAAE,OAAO,EAAE,CAAC,CAAC;AAEhE,wEAAwE;AACxE,MAAM,CAAC,MAAM,QAAQ,GAAG,CAAC,IAAa,EAAE,QAAgB,EAAa,EAAE,CACrE,SAAS,CAAmB,QAAQ,EAAE,IAAI,CAAC,CAAC;AAE9C,0DAA0D;AAC1D,MAAM,CAAC,MAAM,QAAQ,GAAG,CAAC,IAAa,EAAE,QAAgB,EAAkB,EAAE,CAC1E,SAAS,CAAmB,QAAQ,EAAE,IAAI,CAAC,CAAC;AAE9C,MAAM,CAAC,MAAM,IAAI,GAAG,CAAC,EAAW,EAAE,IAAY,EAAsB,EAAE,CACpE,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;AAEjE,MAAM,CAAC,MAAM,OAAO,GAAG,CAAC,EAAW,EAAE,IAAY,EAAW,EAAE,CAC5D,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC;AAElC,kFAAkF;AAClF,MAAM,CAAC,MAAM,IAAI,GAAG,CAAC,IAAyB,EAAU,EAAE,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC;AAE7E,MAAM,CAAC,MAAM,aAAa,GAAG,CAAC,EAAW,EAAa,EAAE,CACtD,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC"}
1
+ {"version":3,"file":"dom.js","sourceRoot":"","sources":["../../../src/core/html/dom.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AAClD,OAAO,EAAE,WAAW,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,YAAY,CAAC;AAExD,OAAO,EAAE,WAAW,EAAE,MAAM,UAAU,CAAC;AACvC,OAAO,EAAE,KAAK,EAAE,MAAM,QAAQ,CAAC;AAC/B,OAAO,EAAE,OAAO,EAAE,MAAM,iCAAiC,CAAC;AAE1D,OAAO,EAAE,kBAAkB,EAAE,SAAS,EAAE,MAAM,WAAW,CAAC;AAI1D,yDAAyD;AACzD,MAAM,CAAC,MAAM,eAAe,GAAG,IAAI,GAAG,CAAC;IACrC,QAAQ;IACR,OAAO;IACP,UAAU;IACV,UAAU;IACV,KAAK;IACL,QAAQ;IACR,MAAM;CACP,CAAC,CAAC;AAEH,wFAAwF;AACxF,MAAM,cAAc,GAAG,IAAI,GAAG,CAAC;IAC7B,GAAG;IACH,KAAK;IACL,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,SAAS;IACT,SAAS;IACT,QAAQ;IACR,QAAQ;IACR,KAAK;IACL,YAAY;IACZ,KAAK;IACL,IAAI;IACJ,IAAI;IACJ,YAAY;CACb,CAAC,CAAC;AAEH,2FAA2F;AAC3F,MAAM,CAAC,MAAM,WAAW,GAAG,CAAC,IAAa,EAAU,EAAE;IACnD,MAAM,KAAK,GAAa,EAAE,CAAC;IAE3B,MAAM,IAAI,GAAG,CAAC,KAAgB,EAAQ,EAAE;QACtC,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,IAAI,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC;gBACjB,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACxB,CAAC;iBAAM,IAAI,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC;gBACvB,IAAI,eAAe,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;oBACnC,SAAS;gBACX,CAAC;gBAED,MAAM,KAAK,GAAG,cAAc,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;gBAE5C,IAAI,KAAK,EAAE,CAAC;oBACV,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;gBAClB,CAAC;gBAED,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;gBAEpB,IAAI,KAAK,EAAE,CAAC;oBACV,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;gBAClB,CAAC;YACH,CAAC;iBAAM,IAAI,WAAW,CAAC,IAAI,CAAC,EAAE,CAAC;gBAC7B,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;YACtB,CAAC;QACH,CAAC;IACH,CAAC,CAAC;IAEF,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;IAEjD,OAAO,kBAAkB,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC;AAC5C,CAAC,CAAC;AAEF,oGAAoG;AACpG,MAAM,CAAC,MAAM,iBAAiB,GAAG,CAAC,IAAa,EAAwB,EAAE;IACvE,MAAM,GAAG,GAAG,IAAI,GAAG,EAAmB,CAAC;IACvC,IAAI,KAAK,GAAG,CAAC,CAAC;IAEd,MAAM,IAAI,GAAG,CAAC,KAAgB,EAAQ,EAAE;QACtC,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,IAAI,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC;gBACjB,KAAK,IAAI,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAChC,CAAC;iBAAM,IAAI,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC;gBACvB,IAAI,IAAI,CAAC,IAAI,KAAK,KAAK,EAAE,CAAC;oBACxB,GAAG,CAAC,GAAG,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;gBACvB,CAAC;qBAAM,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;oBAC3C,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;gBACtB,CAAC;YACH,CAAC;iBAAM,IAAI,WAAW,CAAC,IAAI,CAAC,EAAE,CAAC;gBAC7B,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;YACtB,CAAC;QACH,CAAC;IACH,CAAC,CAAC;IAEF,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;IAEjD,OAAO,GAAG,CAAC;AACb,CAAC,CAAC;AAEF,uGAAuG;AACvG,MAAM,CAAC,MAAM,aAAa,GAAG,CAAC,IAAY,EAAY,EAAE,CACtD,KAAK,CAAC,IAAI,EAAE,EAAE,gBAAgB,EAAE,IAAI,EAAE,WAAW,EAAE,OAAO,EAAE,CAAC,CAAC;AAEhE,wEAAwE;AACxE,MAAM,CAAC,MAAM,QAAQ,GAAG,CAAC,IAAa,EAAE,QAAgB,EAAa,EAAE,CACrE,SAAS,CAAmB,QAAQ,EAAE,IAAI,CAAC,CAAC;AAE9C,0DAA0D;AAC1D,MAAM,CAAC,MAAM,QAAQ,GAAG,CAAC,IAAa,EAAE,QAAgB,EAAkB,EAAE,CAC1E,SAAS,CAAmB,QAAQ,EAAE,IAAI,CAAC,CAAC;AAE9C,MAAM,CAAC,MAAM,IAAI,GAAG,CAAC,EAAW,EAAE,IAAY,EAAsB,EAAE,CACpE,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;AAEjE,MAAM,CAAC,MAAM,OAAO,GAAG,CAAC,EAAW,EAAE,IAAY,EAAW,EAAE,CAC5D,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC;AAElC,kFAAkF;AAClF,MAAM,CAAC,MAAM,IAAI,GAAG,CAAC,IAAyB,EAAU,EAAE,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC;AAE7E,MAAM,CAAC,MAAM,aAAa,GAAG,CAAC,EAAW,EAAa,EAAE,CACtD,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC"}
@@ -1,6 +1,9 @@
1
+ import { isTag } from "domhandler";
1
2
  import { isSameSite, resolveUrl, hostOf } from "../url.js";
2
3
  import { extractAnswerSignals } from "./answer.js";
3
- import { attr, parseDocument, queryAll, queryOne, text, visibleText, } from "./dom.js";
4
+ import { attr, parseDocument, queryAll, queryOne, text, visibleText, wordsBeforeImages, } from "./dom.js";
5
+ import { extractHeadSignals, isHtmlElement } from "./head.js";
6
+ import { extractMissingFragments, extractResources, upgradesInsecureRequests, } from "./resources.js";
4
7
  import { extractStructuredData } from "./structured-data.js";
5
8
  import { collapseWhitespace, wordCount } from "./text.js";
6
9
  const relList = (rel) => (rel ?? "").toLowerCase().split(/\s+/u).filter(Boolean);
@@ -31,7 +34,10 @@ export const extractPage = (html, options) => {
31
34
  const baseHref = baseEl ? attr(baseEl, "href") : undefined;
32
35
  const base = (baseHref && resolveUrl(baseHref, options.url)) || options.url;
33
36
  const siteHost = options.siteHost ?? hostOf(options.url) ?? "";
34
- const titles = queryAll(doc, "title").map((el) => collapseWhitespace(text(el)));
37
+ // An inline SVG icon's <title> is its accessible name, not a second page title.
38
+ const titles = queryAll(doc, "title")
39
+ .filter(isHtmlElement)
40
+ .map((el) => collapseWhitespace(text(el)));
35
41
  const metaDescriptions = queryAll(doc, "meta[name]")
36
42
  .filter((el) => (attr(el, "name") ?? "").trim().toLowerCase() === "description")
37
43
  .map((el) => collapseWhitespace(attr(el, "content") ?? ""));
@@ -63,12 +69,18 @@ export const extractPage = (html, options) => {
63
69
  twitter[key.slice(8)] = content;
64
70
  }
65
71
  }
72
+ const body = queryOne(doc, "body");
73
+ const position = body ? wordsBeforeImages(body) : new Map();
66
74
  const images = queryAll(doc, "img").map((el) => ({
67
75
  src: attr(el, "src") ?? attr(el, "data-src") ?? null,
68
76
  alt: attr(el, "alt") ?? null,
69
77
  width: attr(el, "width") ?? null,
70
78
  height: attr(el, "height") ?? null,
71
79
  loading: attr(el, "loading") ?? null,
80
+ srcset: attr(el, "srcset") ?? null,
81
+ inPicture: el.parent !== null && isTag(el.parent) && el.parent.name === "picture",
82
+ fetchpriority: attr(el, "fetchpriority")?.trim().toLowerCase() ?? null,
83
+ wordsBefore: position.get(el) ?? null,
72
84
  }));
73
85
  const links = queryAll(doc, "a[href]").map((el) => {
74
86
  const href = (attr(el, "href") ?? "").trim();
@@ -105,7 +117,6 @@ export const extractPage = (html, options) => {
105
117
  };
106
118
  });
107
119
  const structured = extractStructuredData(doc);
108
- const body = queryOne(doc, "body");
109
120
  const bodyText = body ? visibleText(body) : "";
110
121
  const htmlEl = queryOne(doc, "html");
111
122
  const viewportEl = queryOne(doc, 'meta[name="viewport" i]');
@@ -129,12 +140,16 @@ export const extractPage = (html, options) => {
129
140
  images,
130
141
  links,
131
142
  hreflang,
143
+ resources: extractResources(doc, base),
144
+ upgradesInsecureRequests: upgradesInsecureRequests(doc),
145
+ missingFragments: extractMissingFragments(doc),
132
146
  structuredData: structured.items,
133
147
  structuredDataErrors: structured.errors,
134
148
  text: bodyText,
135
149
  wordCount: wordCount(bodyText),
136
150
  htmlBytes: Buffer.byteLength(html, "utf-8"),
137
151
  hasBaseHref: Boolean(baseHref),
152
+ head: extractHeadSignals(html, doc),
138
153
  answer: extractAnswerSignals(doc, bodyText),
139
154
  };
140
155
  };
@@ -1 +1 @@
1
- {"version":3,"file":"extract.js","sourceRoot":"","sources":["../../../src/core/html/extract.ts"],"names":[],"mappings":"AAOA,OAAO,EAAE,UAAU,EAAE,UAAU,EAAE,MAAM,EAAE,MAAM,WAAW,CAAC;AAC3D,OAAO,EAAE,oBAAoB,EAAE,MAAM,aAAa,CAAC;AACnD,OAAO,EACL,IAAI,EACJ,aAAa,EACb,QAAQ,EACR,QAAQ,EACR,IAAI,EACJ,WAAW,GACZ,MAAM,UAAU,CAAC;AAElB,OAAO,EAAE,qBAAqB,EAAE,MAAM,sBAAsB,CAAC;AAC7D,OAAO,EAAE,kBAAkB,EAAE,SAAS,EAAE,MAAM,WAAW,CAAC;AAW1D,MAAM,OAAO,GAAG,CAAC,GAA8B,EAAY,EAAE,CAC3D,CAAC,GAAG,IAAI,EAAE,CAAC,CAAC,WAAW,EAAE,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;AAE1D,MAAM,UAAU,GAAG,CAAC,KAAyB,EAAiB,EAAE,CAC9D,KAAK,EAAE,IAAI,EAAE,IAAI,IAAI,CAAC;AAExB,MAAM,aAAa,GAAG,CAAC,MAAgB,EAAiB,EAAE;IACxD,KAAK,MAAM,CAAC,IAAI,MAAM,EAAE,CAAC;QACvB,IAAI,CAAC,EAAE,CAAC;YACN,OAAO,CAAC,CAAC;QACX,CAAC;IACH,CAAC;IAED,OAAO,IAAI,CAAC;AACd,CAAC,CAAC;AAEF,MAAM,aAAa,GAAG,CAAC,GAAa,EAAiB,EAAE;IACrD,MAAM,WAAW,GAAG,QAAQ,CAAC,GAAG,EAAE,eAAe,CAAC,CAAC;IACnD,MAAM,MAAM,GAAG,WAAW,CAAC,CAAC,CAAC,IAAI,CAAC,WAAW,EAAE,SAAS,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;IAEtE,IAAI,MAAM,EAAE,CAAC;QACX,OAAO,MAAM,CAAC,IAAI,EAAE,CAAC;IACvB,CAAC;IAED,MAAM,SAAS,GAAG,QAAQ,CAAC,GAAG,EAAE,kBAAkB,CAAC,CAAC,IAAI,CACtD,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,EAAE,EAAE,YAAY,CAAC,IAAI,EAAE,CAAC,CAAC,WAAW,EAAE,KAAK,cAAc,CACxE,CAAC;IAEF,MAAM,OAAO,GAAG,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,EAAE,SAAS,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;IACnE,MAAM,KAAK,GAAG,OAAO,EAAE,KAAK,CAAC,8BAA8B,CAAC,CAAC;IAE7D,OAAO,KAAK,EAAE,MAAM,EAAE,OAAO,IAAI,IAAI,CAAC;AACxC,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,WAAW,GAAG,CACzB,IAAY,EACZ,OAAuB,EACR,EAAE;IACjB,MAAM,GAAG,GAAG,aAAa,CAAC,IAAI,CAAC,CAAC;IAChC,MAAM,UAAU,GAAG,wBAAwB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACvD,MAAM,MAAM,GAAG,QAAQ,CAAC,GAAG,EAAE,YAAY,CAAC,CAAC;IAC3C,MAAM,QAAQ,GAAG,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;IAC3D,MAAM,IAAI,GAAG,CAAC,QAAQ,IAAI,UAAU,CAAC,QAAQ,EAAE,OAAO,CAAC,GAAG,CAAC,CAAC,IAAI,OAAO,CAAC,GAAG,CAAC;IAC5E,MAAM,QAAQ,GAAG,OAAO,CAAC,QAAQ,IAAI,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC;IAE/D,MAAM,MAAM,GAAG,QAAQ,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAC/C,kBAAkB,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAC7B,CAAC;IAEF,MAAM,gBAAgB,GAAG,QAAQ,CAAC,GAAG,EAAE,YAAY,CAAC;SACjD,MAAM,CACL,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,EAAE,EAAE,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,KAAK,aAAa,CACxE;SACA,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,kBAAkB,CAAC,IAAI,CAAC,EAAE,EAAE,SAAS,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;IAE9D,MAAM,UAAU,GAAG,QAAQ,CAAC,GAAG,EAAE,YAAY,CAAC;SAC3C,MAAM,CAAC,CAAC,EAAE,EAAE,EAAE,CACb,CAAC,QAAQ,EAAE,WAAW,EAAE,gBAAgB,EAAE,SAAS,CAAC,CAAC,QAAQ,CAC3D,CAAC,IAAI,CAAC,EAAE,EAAE,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAC9C,CACF;SACA,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,kBAAkB,CAAC,IAAI,CAAC,EAAE,EAAE,SAAS,CAAC,IAAI,EAAE,CAAC,CAAC,WAAW,EAAE,CAAC;SACxE,MAAM,CAAC,OAAO,CAAC,CAAC;IAEnB,MAAM,UAAU,GAAG,QAAQ,CAAC,GAAG,EAAE,WAAW,CAAC;SAC1C,MAAM,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,EAAE,KAAK,CAAC,CAAC,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC;SAC9D,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,EAAE,EAAE,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;IAEhD,MAAM,QAAQ,GAAkB,QAAQ,CAAC,GAAG,EAAE,wBAAwB,CAAC,CAAC,GAAG,CACzE,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC;QACP,KAAK,EAAE,MAAM,CAAC,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;QAC/B,IAAI,EAAE,kBAAkB,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;KACnC,CAAC,CACH,CAAC;IAEF,MAAM,EAAE,GAA2B,EAAE,CAAC;IACtC,MAAM,OAAO,GAA2B,EAAE,CAAC;IAE3C,KAAK,MAAM,EAAE,IAAI,QAAQ,CAAC,GAAG,EAAE,4BAA4B,CAAC,EAAE,CAAC;QAC7D,MAAM,GAAG,GAAG,CAAC,IAAI,CAAC,EAAE,EAAE,UAAU,CAAC,IAAI,IAAI,CAAC,EAAE,EAAE,MAAM,CAAC,IAAI,EAAE,CAAC;aACzD,IAAI,EAAE;aACN,WAAW,EAAE,CAAC;QAEjB,MAAM,OAAO,GAAG,kBAAkB,CAAC,IAAI,CAAC,EAAE,EAAE,SAAS,CAAC,IAAI,EAAE,CAAC,CAAC;QAE9D,IAAI,CAAC,GAAG,EAAE,CAAC;YACT,SAAS;QACX,CAAC;QAED,IAAI,GAAG,CAAC,UAAU,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,GAAG,IAAI,EAAE,CAAC,EAAE,CAAC;YAC1C,EAAE,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,GAAG,OAAO,CAAC;QAC7B,CAAC;aAAM,IAAI,GAAG,CAAC,UAAU,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,GAAG,IAAI,OAAO,CAAC,EAAE,CAAC;YAC3D,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,GAAG,OAAO,CAAC;QAClC,CAAC;IACH,CAAC;IAED,MAAM,MAAM,GAAgB,QAAQ,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC;QAC5D,GAAG,EAAE,IAAI,CAAC,EAAE,EAAE,KAAK,CAAC,IAAI,IAAI,CAAC,EAAE,EAAE,UAAU,CAAC,IAAI,IAAI;QACpD,GAAG,EAAE,IAAI,CAAC,EAAE,EAAE,KAAK,CAAC,IAAI,IAAI;QAC5B,KAAK,EAAE,IAAI,CAAC,EAAE,EAAE,OAAO,CAAC,IAAI,IAAI;QAChC,MAAM,EAAE,IAAI,CAAC,EAAE,EAAE,QAAQ,CAAC,IAAI,IAAI;QAClC,OAAO,EAAE,IAAI,CAAC,EAAE,EAAE,SAAS,CAAC,IAAI,IAAI;KACrC,CAAC,CAAC,CAAC;IAEJ,MAAM,KAAK,GAAe,QAAQ,CAAC,GAAG,EAAE,SAAS,CAAC,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE;QAC5D,MAAM,IAAI,GAAG,CAAC,IAAI,CAAC,EAAE,EAAE,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;QAE7C,kEAAkE;QAClE,MAAM,GAAG,GACP,IAAI,KAAK,EAAE,IAAI,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,UAAU,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;QAEtE,MAAM,GAAG,GAAG,IAAI,CAAC,EAAE,EAAE,KAAK,CAAC,IAAI,IAAI,CAAC;QACpC,MAAM,SAAS,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC;QAC/B,MAAM,QAAQ,GAAG,kBAAkB,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC;QAC9C,MAAM,SAAS,GAAG,kBAAkB,CAAC,IAAI,CAAC,EAAE,EAAE,YAAY,CAAC,IAAI,EAAE,CAAC,CAAC;QACnE,MAAM,KAAK,GAAG,kBAAkB,CAAC,IAAI,CAAC,EAAE,EAAE,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC;QAE1D,MAAM,MAAM,GAAG,QAAQ,CAAC,EAAE,EAAE,UAAU,CAAC;aACpC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,kBAAkB,CAAC,IAAI,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC;aACxD,IAAI,CAAC,OAAO,CAAC,CAAC;QAEjB,OAAO;YACL,IAAI;YACJ,GAAG;YACH,IAAI,EAAE,QAAQ;YACd,GAAG;YACH,QAAQ,EAAE,GAAG;gBACX,CAAC,CAAC,UAAU,CAAC,GAAG,EAAE,QAAQ,EAAE,OAAO,CAAC,iBAAiB,CAAC;gBACtD,CAAC,CAAC,KAAK;YACT,QAAQ,EAAE,SAAS,CAAC,QAAQ,CAAC,UAAU,CAAC;YACxC,iBAAiB,EAAE,OAAO,CAAC,QAAQ,IAAI,SAAS,IAAI,KAAK,IAAI,MAAM,CAAC;SACrE,CAAC;IACJ,CAAC,CAAC,CAAC;IAEH,MAAM,QAAQ,GAAmB,QAAQ,CAAC,GAAG,EAAE,WAAW,CAAC;SACxD,MAAM,CACL,CAAC,EAAE,EAAE,EAAE,CACL,OAAO,CAAC,IAAI,CAAC,EAAE,EAAE,KAAK,CAAC,CAAC,CAAC,QAAQ,CAAC,WAAW,CAAC,IAAI,IAAI,CAAC,EAAE,EAAE,UAAU,CAAC,CACzE;SACA,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE;QACV,MAAM,IAAI,GAAG,CAAC,IAAI,CAAC,EAAE,EAAE,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;QAE7C,OAAO;YACL,IAAI,EAAE,CAAC,IAAI,CAAC,EAAE,EAAE,UAAU,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE;YACzC,IAAI;YACJ,GAAG,EAAE,UAAU,CAAC,IAAI,EAAE,IAAI,CAAC;SAC5B,CAAC;IACJ,CAAC,CAAC,CAAC;IAEL,MAAM,UAAU,GAAG,qBAAqB,CAAC,GAAG,CAAC,CAAC;IAE9C,MAAM,IAAI,GAAG,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC;IACnC,MAAM,QAAQ,GAAG,IAAI,CAAC,CAAC,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;IAE/C,MAAM,MAAM,GAAG,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC;IACrC,MAAM,UAAU,GAAG,QAAQ,CAAC,GAAG,EAAE,yBAAyB,CAAC,CAAC;IAE5D,OAAO;QACL,GAAG,EAAE,OAAO,CAAC,GAAG;QAChB,UAAU;QACV,IAAI,EAAE,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;QAC3D,OAAO,EAAE,aAAa,CAAC,GAAG,CAAC;QAC3B,QAAQ,EAAE,UAAU,CAAC,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC,UAAU,EAAE,SAAS,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;QAC1E,MAAM;QACN,KAAK,EAAE,aAAa,CAAC,MAAM,CAAC;QAC5B,gBAAgB;QAChB,eAAe,EAAE,aAAa,CAAC,gBAAgB,CAAC;QAChD,UAAU;QACV,UAAU;QACV,SAAS,EAAE,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,UAAU,CAAC,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI;QACjE,QAAQ;QACR,GAAG,EAAE,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;QAC7D,EAAE;QACF,OAAO;QACP,MAAM;QACN,KAAK;QACL,QAAQ;QACR,cAAc,EAAE,UAAU,CAAC,KAAK;QAChC,oBAAoB,EAAE,UAAU,CAAC,MAAM;QACvC,IAAI,EAAE,QAAQ;QACd,SAAS,EAAE,SAAS,CAAC,QAAQ,CAAC;QAC9B,SAAS,EAAE,MAAM,CAAC,UAAU,CAAC,IAAI,EAAE,OAAO,CAAC;QAC3C,WAAW,EAAE,OAAO,CAAC,QAAQ,CAAC;QAC9B,MAAM,EAAE,oBAAoB,CAAC,GAAG,EAAE,QAAQ,CAAC;KAC5C,CAAC;AACJ,CAAC,CAAC"}
1
+ {"version":3,"file":"extract.js","sourceRoot":"","sources":["../../../src/core/html/extract.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,MAAM,YAAY,CAAC;AASnC,OAAO,EAAE,UAAU,EAAE,UAAU,EAAE,MAAM,EAAE,MAAM,WAAW,CAAC;AAC3D,OAAO,EAAE,oBAAoB,EAAE,MAAM,aAAa,CAAC;AACnD,OAAO,EACL,IAAI,EACJ,aAAa,EACb,QAAQ,EACR,QAAQ,EACR,IAAI,EACJ,WAAW,EACX,iBAAiB,GAClB,MAAM,UAAU,CAAC;AAElB,OAAO,EAAE,kBAAkB,EAAE,aAAa,EAAE,MAAM,WAAW,CAAC;AAC9D,OAAO,EACL,uBAAuB,EACvB,gBAAgB,EAChB,wBAAwB,GACzB,MAAM,gBAAgB,CAAC;AACxB,OAAO,EAAE,qBAAqB,EAAE,MAAM,sBAAsB,CAAC;AAC7D,OAAO,EAAE,kBAAkB,EAAE,SAAS,EAAE,MAAM,WAAW,CAAC;AAW1D,MAAM,OAAO,GAAG,CAAC,GAA8B,EAAY,EAAE,CAC3D,CAAC,GAAG,IAAI,EAAE,CAAC,CAAC,WAAW,EAAE,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;AAE1D,MAAM,UAAU,GAAG,CAAC,KAAyB,EAAiB,EAAE,CAC9D,KAAK,EAAE,IAAI,EAAE,IAAI,IAAI,CAAC;AAExB,MAAM,aAAa,GAAG,CAAC,MAAgB,EAAiB,EAAE;IACxD,KAAK,MAAM,CAAC,IAAI,MAAM,EAAE,CAAC;QACvB,IAAI,CAAC,EAAE,CAAC;YACN,OAAO,CAAC,CAAC;QACX,CAAC;IACH,CAAC;IAED,OAAO,IAAI,CAAC;AACd,CAAC,CAAC;AAEF,MAAM,aAAa,GAAG,CAAC,GAAa,EAAiB,EAAE;IACrD,MAAM,WAAW,GAAG,QAAQ,CAAC,GAAG,EAAE,eAAe,CAAC,CAAC;IACnD,MAAM,MAAM,GAAG,WAAW,CAAC,CAAC,CAAC,IAAI,CAAC,WAAW,EAAE,SAAS,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;IAEtE,IAAI,MAAM,EAAE,CAAC;QACX,OAAO,MAAM,CAAC,IAAI,EAAE,CAAC;IACvB,CAAC;IAED,MAAM,SAAS,GAAG,QAAQ,CAAC,GAAG,EAAE,kBAAkB,CAAC,CAAC,IAAI,CACtD,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,EAAE,EAAE,YAAY,CAAC,IAAI,EAAE,CAAC,CAAC,WAAW,EAAE,KAAK,cAAc,CACxE,CAAC;IAEF,MAAM,OAAO,GAAG,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,EAAE,SAAS,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;IACnE,MAAM,KAAK,GAAG,OAAO,EAAE,KAAK,CAAC,8BAA8B,CAAC,CAAC;IAE7D,OAAO,KAAK,EAAE,MAAM,EAAE,OAAO,IAAI,IAAI,CAAC;AACxC,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,WAAW,GAAG,CACzB,IAAY,EACZ,OAAuB,EACR,EAAE;IACjB,MAAM,GAAG,GAAG,aAAa,CAAC,IAAI,CAAC,CAAC;IAChC,MAAM,UAAU,GAAG,wBAAwB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACvD,MAAM,MAAM,GAAG,QAAQ,CAAC,GAAG,EAAE,YAAY,CAAC,CAAC;IAC3C,MAAM,QAAQ,GAAG,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;IAC3D,MAAM,IAAI,GAAG,CAAC,QAAQ,IAAI,UAAU,CAAC,QAAQ,EAAE,OAAO,CAAC,GAAG,CAAC,CAAC,IAAI,OAAO,CAAC,GAAG,CAAC;IAC5E,MAAM,QAAQ,GAAG,OAAO,CAAC,QAAQ,IAAI,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC;IAE/D,gFAAgF;IAChF,MAAM,MAAM,GAAG,QAAQ,CAAC,GAAG,EAAE,OAAO,CAAC;SAClC,MAAM,CAAC,aAAa,CAAC;SACrB,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,kBAAkB,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;IAE7C,MAAM,gBAAgB,GAAG,QAAQ,CAAC,GAAG,EAAE,YAAY,CAAC;SACjD,MAAM,CACL,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,EAAE,EAAE,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,KAAK,aAAa,CACxE;SACA,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,kBAAkB,CAAC,IAAI,CAAC,EAAE,EAAE,SAAS,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;IAE9D,MAAM,UAAU,GAAG,QAAQ,CAAC,GAAG,EAAE,YAAY,CAAC;SAC3C,MAAM,CAAC,CAAC,EAAE,EAAE,EAAE,CACb,CAAC,QAAQ,EAAE,WAAW,EAAE,gBAAgB,EAAE,SAAS,CAAC,CAAC,QAAQ,CAC3D,CAAC,IAAI,CAAC,EAAE,EAAE,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAC9C,CACF;SACA,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,kBAAkB,CAAC,IAAI,CAAC,EAAE,EAAE,SAAS,CAAC,IAAI,EAAE,CAAC,CAAC,WAAW,EAAE,CAAC;SACxE,MAAM,CAAC,OAAO,CAAC,CAAC;IAEnB,MAAM,UAAU,GAAG,QAAQ,CAAC,GAAG,EAAE,WAAW,CAAC;SAC1C,MAAM,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,EAAE,KAAK,CAAC,CAAC,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC;SAC9D,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,EAAE,EAAE,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;IAEhD,MAAM,QAAQ,GAAkB,QAAQ,CAAC,GAAG,EAAE,wBAAwB,CAAC,CAAC,GAAG,CACzE,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC;QACP,KAAK,EAAE,MAAM,CAAC,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;QAC/B,IAAI,EAAE,kBAAkB,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;KACnC,CAAC,CACH,CAAC;IAEF,MAAM,EAAE,GAA2B,EAAE,CAAC;IACtC,MAAM,OAAO,GAA2B,EAAE,CAAC;IAE3C,KAAK,MAAM,EAAE,IAAI,QAAQ,CAAC,GAAG,EAAE,4BAA4B,CAAC,EAAE,CAAC;QAC7D,MAAM,GAAG,GAAG,CAAC,IAAI,CAAC,EAAE,EAAE,UAAU,CAAC,IAAI,IAAI,CAAC,EAAE,EAAE,MAAM,CAAC,IAAI,EAAE,CAAC;aACzD,IAAI,EAAE;aACN,WAAW,EAAE,CAAC;QAEjB,MAAM,OAAO,GAAG,kBAAkB,CAAC,IAAI,CAAC,EAAE,EAAE,SAAS,CAAC,IAAI,EAAE,CAAC,CAAC;QAE9D,IAAI,CAAC,GAAG,EAAE,CAAC;YACT,SAAS;QACX,CAAC;QAED,IAAI,GAAG,CAAC,UAAU,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,GAAG,IAAI,EAAE,CAAC,EAAE,CAAC;YAC1C,EAAE,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,GAAG,OAAO,CAAC;QAC7B,CAAC;aAAM,IAAI,GAAG,CAAC,UAAU,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,GAAG,IAAI,OAAO,CAAC,EAAE,CAAC;YAC3D,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,GAAG,OAAO,CAAC;QAClC,CAAC;IACH,CAAC;IAED,MAAM,IAAI,GAAG,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC;IACnC,MAAM,QAAQ,GAAG,IAAI,CAAC,CAAC,CAAC,iBAAiB,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,GAAG,EAAmB,CAAC;IAE7E,MAAM,MAAM,GAAgB,QAAQ,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC;QAC5D,GAAG,EAAE,IAAI,CAAC,EAAE,EAAE,KAAK,CAAC,IAAI,IAAI,CAAC,EAAE,EAAE,UAAU,CAAC,IAAI,IAAI;QACpD,GAAG,EAAE,IAAI,CAAC,EAAE,EAAE,KAAK,CAAC,IAAI,IAAI;QAC5B,KAAK,EAAE,IAAI,CAAC,EAAE,EAAE,OAAO,CAAC,IAAI,IAAI;QAChC,MAAM,EAAE,IAAI,CAAC,EAAE,EAAE,QAAQ,CAAC,IAAI,IAAI;QAClC,OAAO,EAAE,IAAI,CAAC,EAAE,EAAE,SAAS,CAAC,IAAI,IAAI;QACpC,MAAM,EAAE,IAAI,CAAC,EAAE,EAAE,QAAQ,CAAC,IAAI,IAAI;QAClC,SAAS,EACP,EAAE,CAAC,MAAM,KAAK,IAAI,IAAI,KAAK,CAAC,EAAE,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC,MAAM,CAAC,IAAI,KAAK,SAAS;QACxE,aAAa,EAAE,IAAI,CAAC,EAAE,EAAE,eAAe,CAAC,EAAE,IAAI,EAAE,CAAC,WAAW,EAAE,IAAI,IAAI;QACtE,WAAW,EAAE,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,IAAI;KACtC,CAAC,CAAC,CAAC;IAEJ,MAAM,KAAK,GAAe,QAAQ,CAAC,GAAG,EAAE,SAAS,CAAC,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE;QAC5D,MAAM,IAAI,GAAG,CAAC,IAAI,CAAC,EAAE,EAAE,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;QAE7C,kEAAkE;QAClE,MAAM,GAAG,GACP,IAAI,KAAK,EAAE,IAAI,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,UAAU,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;QAEtE,MAAM,GAAG,GAAG,IAAI,CAAC,EAAE,EAAE,KAAK,CAAC,IAAI,IAAI,CAAC;QACpC,MAAM,SAAS,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC;QAC/B,MAAM,QAAQ,GAAG,kBAAkB,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC;QAC9C,MAAM,SAAS,GAAG,kBAAkB,CAAC,IAAI,CAAC,EAAE,EAAE,YAAY,CAAC,IAAI,EAAE,CAAC,CAAC;QACnE,MAAM,KAAK,GAAG,kBAAkB,CAAC,IAAI,CAAC,EAAE,EAAE,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC;QAE1D,MAAM,MAAM,GAAG,QAAQ,CAAC,EAAE,EAAE,UAAU,CAAC;aACpC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,kBAAkB,CAAC,IAAI,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC;aACxD,IAAI,CAAC,OAAO,CAAC,CAAC;QAEjB,OAAO;YACL,IAAI;YACJ,GAAG;YACH,IAAI,EAAE,QAAQ;YACd,GAAG;YACH,QAAQ,EAAE,GAAG;gBACX,CAAC,CAAC,UAAU,CAAC,GAAG,EAAE,QAAQ,EAAE,OAAO,CAAC,iBAAiB,CAAC;gBACtD,CAAC,CAAC,KAAK;YACT,QAAQ,EAAE,SAAS,CAAC,QAAQ,CAAC,UAAU,CAAC;YACxC,iBAAiB,EAAE,OAAO,CAAC,QAAQ,IAAI,SAAS,IAAI,KAAK,IAAI,MAAM,CAAC;SACrE,CAAC;IACJ,CAAC,CAAC,CAAC;IAEH,MAAM,QAAQ,GAAmB,QAAQ,CAAC,GAAG,EAAE,WAAW,CAAC;SACxD,MAAM,CACL,CAAC,EAAE,EAAE,EAAE,CACL,OAAO,CAAC,IAAI,CAAC,EAAE,EAAE,KAAK,CAAC,CAAC,CAAC,QAAQ,CAAC,WAAW,CAAC,IAAI,IAAI,CAAC,EAAE,EAAE,UAAU,CAAC,CACzE;SACA,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE;QACV,MAAM,IAAI,GAAG,CAAC,IAAI,CAAC,EAAE,EAAE,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;QAE7C,OAAO;YACL,IAAI,EAAE,CAAC,IAAI,CAAC,EAAE,EAAE,UAAU,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE;YACzC,IAAI;YACJ,GAAG,EAAE,UAAU,CAAC,IAAI,EAAE,IAAI,CAAC;SAC5B,CAAC;IACJ,CAAC,CAAC,CAAC;IAEL,MAAM,UAAU,GAAG,qBAAqB,CAAC,GAAG,CAAC,CAAC;IAE9C,MAAM,QAAQ,GAAG,IAAI,CAAC,CAAC,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;IAE/C,MAAM,MAAM,GAAG,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC;IACrC,MAAM,UAAU,GAAG,QAAQ,CAAC,GAAG,EAAE,yBAAyB,CAAC,CAAC;IAE5D,OAAO;QACL,GAAG,EAAE,OAAO,CAAC,GAAG;QAChB,UAAU;QACV,IAAI,EAAE,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;QAC3D,OAAO,EAAE,aAAa,CAAC,GAAG,CAAC;QAC3B,QAAQ,EAAE,UAAU,CAAC,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC,UAAU,EAAE,SAAS,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;QAC1E,MAAM;QACN,KAAK,EAAE,aAAa,CAAC,MAAM,CAAC;QAC5B,gBAAgB;QAChB,eAAe,EAAE,aAAa,CAAC,gBAAgB,CAAC;QAChD,UAAU;QACV,UAAU;QACV,SAAS,EAAE,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,UAAU,CAAC,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI;QACjE,QAAQ;QACR,GAAG,EAAE,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;QAC7D,EAAE;QACF,OAAO;QACP,MAAM;QACN,KAAK;QACL,QAAQ;QACR,SAAS,EAAE,gBAAgB,CAAC,GAAG,EAAE,IAAI,CAAC;QACtC,wBAAwB,EAAE,wBAAwB,CAAC,GAAG,CAAC;QACvD,gBAAgB,EAAE,uBAAuB,CAAC,GAAG,CAAC;QAC9C,cAAc,EAAE,UAAU,CAAC,KAAK;QAChC,oBAAoB,EAAE,UAAU,CAAC,MAAM;QACvC,IAAI,EAAE,QAAQ;QACd,SAAS,EAAE,SAAS,CAAC,QAAQ,CAAC;QAC9B,SAAS,EAAE,MAAM,CAAC,UAAU,CAAC,IAAI,EAAE,OAAO,CAAC;QAC3C,WAAW,EAAE,OAAO,CAAC,QAAQ,CAAC;QAC9B,IAAI,EAAE,kBAAkB,CAAC,IAAI,EAAE,GAAG,CAAC;QACnC,MAAM,EAAE,oBAAoB,CAAC,GAAG,EAAE,QAAQ,CAAC;KAC5C,CAAC;AACJ,CAAC,CAAC"}
@@ -0,0 +1,10 @@
1
+ import type { HeadSignals } from "../types.js";
2
+ import type { Document, Element } from "./dom.js";
3
+ /** False for SVG and MathML elements such as an icon's `<title>`, which are not page metadata. */
4
+ export declare const isHtmlElement: (el: Element) => boolean;
5
+ /**
6
+ * The metadata element as it is written (`<meta name="description">`, `<link rel="canonical">`),
7
+ * or null for elements that also belong in <body>, such as microdata `<meta itemprop>`.
8
+ */
9
+ export declare const describeMetadata: (el: Element) => string | null;
10
+ export declare const extractHeadSignals: (html: string, doc: Document) => HeadSignals;
@@ -0,0 +1,206 @@
1
+ import { isTag, isText } from "domhandler";
2
+ import { parse } from "parse5";
3
+ import { adapter } from "parse5-htmlparser2-tree-adapter";
4
+ import { attr, queryAll, queryOne } from "./dom.js";
5
+ import { collapseWhitespace } from "./text.js";
6
+ const HTML_NAMESPACE = "http://www.w3.org/1999/xhtml";
7
+ /**
8
+ * Meta names search engines read in <head> only. Robots meta tags are left out: Google honours
9
+ * them in <body> too.
10
+ */
11
+ const HEAD_META_NAMES = new Set(["description", "viewport"]);
12
+ /*
13
+ * Where the author's <head> ends: the first `</head>` or `<body>` tag. Both alternatives are fixed
14
+ * words followed by one character, so the search stays linear on long documents.
15
+ */
16
+ const HEAD_END = /<\/head[\s/>]|<body[\s/>]/iu;
17
+ /** Longest stretch of source quoted for the element or text that ends <head>. */
18
+ const MAX_QUOTE = 120;
19
+ /** False for SVG and MathML elements such as an icon's `<title>`, which are not page metadata. */
20
+ export const isHtmlElement = (el) => !el.namespace || el.namespace === HTML_NAMESPACE;
21
+ const relTokens = (el) => (attr(el, "rel") ?? "").toLowerCase().split(/\s+/u).filter(Boolean);
22
+ /**
23
+ * The metadata element as it is written (`<meta name="description">`, `<link rel="canonical">`),
24
+ * or null for elements that also belong in <body>, such as microdata `<meta itemprop>`.
25
+ */
26
+ export const describeMetadata = (el) => {
27
+ if (!isHtmlElement(el)) {
28
+ return null;
29
+ }
30
+ if (el.name === "title") {
31
+ return "<title>";
32
+ }
33
+ if (el.name === "meta") {
34
+ const name = (attr(el, "name") ?? "").trim().toLowerCase();
35
+ return HEAD_META_NAMES.has(name) ? `<meta name="${name}">` : null;
36
+ }
37
+ if (el.name === "link") {
38
+ const rel = relTokens(el);
39
+ if (rel.includes("canonical")) {
40
+ return '<link rel="canonical">';
41
+ }
42
+ const hreflang = attr(el, "hreflang")?.trim();
43
+ if (rel.includes("alternate") && hreflang) {
44
+ return `<link rel="alternate" hreflang="${hreflang.slice(0, 20)}">`;
45
+ }
46
+ }
47
+ return null;
48
+ };
49
+ const isInsideHidden = (el) => {
50
+ for (let node = el.parent; node && isTag(node); node = node.parent) {
51
+ if (node.name === "body") {
52
+ return false;
53
+ }
54
+ if (attr(node, "hidden") !== undefined) {
55
+ return true;
56
+ }
57
+ }
58
+ return false;
59
+ };
60
+ const metadataElements = (root) => queryAll(root, "title, meta[name], link[rel]").flatMap((el) => {
61
+ const described = describeMetadata(el);
62
+ return described ? [{ described, streamed: isInsideHidden(el) }] : [];
63
+ });
64
+ /** Metadata elements under `root`, in document order, as `describeMetadata` writes them. */
65
+ const metadataIn = (root) => metadataElements(root).map((m) => m.described);
66
+ /** Characters that look like whitespace but end <head>, by the name a reader recognizes. */
67
+ const INVISIBLE_NAMES = new Map([
68
+ [0xfe_ff, "byte order mark"],
69
+ [0xa0, "no-break space, &nbsp;"],
70
+ ]);
71
+ /** "U+FEFF (byte order mark)" for the characters in text that is not ASCII whitespace. */
72
+ const invisibleCharacters = (data) => [...new Set([...data].filter((c) => !"\t\n\f\r ".includes(c)))]
73
+ .slice(0, 3)
74
+ .map((c) => {
75
+ const code = c.codePointAt(0) ?? 0;
76
+ const name = INVISIBLE_NAMES.get(code);
77
+ const hex = code.toString(16).toUpperCase().padStart(4, "0");
78
+ return name ? `U+${hex} (${name})` : `U+${hex}`;
79
+ })
80
+ .join(", ");
81
+ /** The source of an element's start tag, or a text node's first words. */
82
+ const quote = (node, source) => {
83
+ if (isText(node)) {
84
+ const words = collapseWhitespace(node.data);
85
+ if (!words) {
86
+ return `invisible character ${invisibleCharacters(node.data)}`;
87
+ }
88
+ return `text "${words.length > 40 ? `${words.slice(0, 40)}…` : words}"`;
89
+ }
90
+ const location = isTag(node) ? node.sourceCodeLocation : undefined;
91
+ if (!location) {
92
+ return isTag(node) ? `<${node.name}>` : "unknown";
93
+ }
94
+ const end = Math.min(location.startTag?.endOffset ?? location.endOffset, location.startOffset + MAX_QUOTE);
95
+ return collapseWhitespace(source.slice(location.startOffset, end));
96
+ };
97
+ /**
98
+ * Parse only what the author wrote before `</head>` or `<body>`. Everything the parser puts in
99
+ * <body> then sat inside <head> in the source: the first such node is what ended <head>, the way
100
+ * Google stops reading <head> at the first element that does not belong there.
101
+ */
102
+ const parseHead = (prefix, scriptingEnabled) => {
103
+ const doc = parse(prefix, {
104
+ scriptingEnabled,
105
+ sourceCodeLocationInfo: true,
106
+ treeAdapter: adapter,
107
+ });
108
+ const body = queryOne(doc, "body");
109
+ // Only ASCII whitespace keeps <head> open; a stray byte order mark or &nbsp; ends it.
110
+ const closedBy = body?.children.find((node) => isTag(node) || (isText(node) && /[^\t\n\f\r ]/u.test(node.data))) ?? null;
111
+ return {
112
+ closedBy,
113
+ afterClose: body && closedBy ? metadataIn(body) : [],
114
+ doc,
115
+ };
116
+ };
117
+ /** `<meta http-equiv="refresh" content="5; url=/next">` as seconds and target. */
118
+ const parseRefresh = (content) => {
119
+ const value = content.slice(0, 2048).trim();
120
+ const separator = value.search(/[;,]/u);
121
+ const time = (separator === -1 ? value : value.slice(0, separator)).trim();
122
+ let target = separator === -1 ? "" : value.slice(separator + 1).trim();
123
+ if (/^url\s*=/iu.test(target)) {
124
+ target = target.slice(target.indexOf("=") + 1).trim();
125
+ }
126
+ if (/^["']/u.test(target)) {
127
+ const closing = target.indexOf(target.charAt(0), 1);
128
+ target = target.slice(1, closing === -1 ? undefined : closing);
129
+ }
130
+ const seconds = /^\d+(?:\.\d*)?$/u.test(time)
131
+ ? Math.floor(Number(time))
132
+ : null;
133
+ return { seconds, url: target.trim() || null };
134
+ };
135
+ /** Byte offset where the element declaring the character encoding ends, in the source of <head>. */
136
+ const charsetEnd = (doc, source) => {
137
+ const declaration = queryAll(doc, "meta").find((el) => attr(el, "charset") !== undefined ||
138
+ (attr(el, "http-equiv") ?? "").trim().toLowerCase() === "content-type");
139
+ const end = declaration?.sourceCodeLocation?.endOffset;
140
+ return end === undefined ? null : Buffer.byteLength(source.slice(0, end));
141
+ };
142
+ /** Metadata in <body>, apart from what a framework streams there for its JavaScript to move. */
143
+ const splitStreamed = (items) => ({
144
+ inBody: items.flatMap((m) => (m.streamed ? [] : [m.described])),
145
+ streamedInBody: items.flatMap((m) => (m.streamed ? [m.described] : [])),
146
+ });
147
+ /** Types that make a <script> run as JavaScript; anything else (JSON, templates) never blocks. */
148
+ const isJavaScriptType = (type) => {
149
+ const t = (type ?? "").trim().toLowerCase();
150
+ return t === "" || t.endsWith("javascript") || t.endsWith("ecmascript");
151
+ };
152
+ /**
153
+ * Scripts in <head> the parser waits for: external, classic, neither async nor defer. A nomodule
154
+ * script is a fallback for old browsers, which current ones neither download nor run.
155
+ */
156
+ const blockingScripts = (doc) => queryAll(doc, "head script[src]")
157
+ .filter((el) => isJavaScriptType(attr(el, "type")) &&
158
+ attr(el, "async") === undefined &&
159
+ attr(el, "defer") === undefined &&
160
+ attr(el, "nomodule") === undefined)
161
+ .map((el) => (attr(el, "src") ?? "").trim());
162
+ const ICON_RELS = new Set([
163
+ "icon",
164
+ "apple-touch-icon",
165
+ "apple-touch-icon-precomposed",
166
+ ]);
167
+ const iconLinks = (doc) => queryAll(doc, "link[rel][href]")
168
+ .filter((el) => relTokens(el).some((rel) => ICON_RELS.has(rel)))
169
+ .map((el) => (attr(el, "href") ?? "").trim());
170
+ export const extractHeadSignals = (html, doc) => {
171
+ const refresh = queryAll(doc, "meta[http-equiv]")
172
+ .filter((el) => (attr(el, "http-equiv") ?? "").trim().toLowerCase() === "refresh")
173
+ .map((el) => parseRefresh(attr(el, "content") ?? ""));
174
+ const body = queryOne(doc, "body");
175
+ const metadataInBody = body ? metadataElements(body) : [];
176
+ const end = HEAD_END.exec(html);
177
+ if (!end) {
178
+ return {
179
+ closedBy: null,
180
+ closedInNoscript: false,
181
+ afterClose: [],
182
+ ...splitStreamed(metadataInBody),
183
+ refresh,
184
+ charsetEndByte: null,
185
+ blockingScripts: blockingScripts(doc),
186
+ icons: iconLinks(doc),
187
+ };
188
+ }
189
+ const prefix = html.slice(0, end.index);
190
+ const browser = parseHead(prefix, true);
191
+ // A <noscript> in <head> is text to a browser that runs scripts and markup to one that does not.
192
+ const plain = browser.closedBy ? browser : parseHead(prefix, false);
193
+ const { closedBy } = plain;
194
+ return {
195
+ closedBy: closedBy ? quote(closedBy, prefix) : null,
196
+ closedInNoscript: !browser.closedBy && Boolean(closedBy),
197
+ afterClose: plain.afterClose,
198
+ // The parse of the whole page puts metadata after `closedBy` in <body> too; count it once.
199
+ ...splitStreamed(metadataInBody.slice(browser.afterClose.length)),
200
+ refresh,
201
+ charsetEndByte: charsetEnd(browser.doc, prefix),
202
+ blockingScripts: blockingScripts(doc),
203
+ icons: iconLinks(doc),
204
+ };
205
+ };
206
+ //# sourceMappingURL=head.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"head.js","sourceRoot":"","sources":["../../../src/core/html/head.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,YAAY,CAAC;AAC3C,OAAO,EAAE,KAAK,EAAE,MAAM,QAAQ,CAAC;AAC/B,OAAO,EAAE,OAAO,EAAE,MAAM,iCAAiC,CAAC;AAG1D,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,QAAQ,EAAE,MAAM,UAAU,CAAC;AAEpD,OAAO,EAAE,kBAAkB,EAAE,MAAM,WAAW,CAAC;AAE/C,MAAM,cAAc,GAAG,8BAA8B,CAAC;AAEtD;;;GAGG;AACH,MAAM,eAAe,GAAG,IAAI,GAAG,CAAC,CAAC,aAAa,EAAE,UAAU,CAAC,CAAC,CAAC;AAE7D;;;GAGG;AACH,MAAM,QAAQ,GAAG,6BAA6B,CAAC;AAE/C,iFAAiF;AACjF,MAAM,SAAS,GAAG,GAAG,CAAC;AAEtB,kGAAkG;AAClG,MAAM,CAAC,MAAM,aAAa,GAAG,CAAC,EAAW,EAAW,EAAE,CACpD,CAAC,EAAE,CAAC,SAAS,IAAI,EAAE,CAAC,SAAS,KAAK,cAAc,CAAC;AAEnD,MAAM,SAAS,GAAG,CAAC,EAAW,EAAY,EAAE,CAC1C,CAAC,IAAI,CAAC,EAAE,EAAE,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC,WAAW,EAAE,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;AAEtE;;;GAGG;AACH,MAAM,CAAC,MAAM,gBAAgB,GAAG,CAAC,EAAW,EAAiB,EAAE;IAC7D,IAAI,CAAC,aAAa,CAAC,EAAE,CAAC,EAAE,CAAC;QACvB,OAAO,IAAI,CAAC;IACd,CAAC;IAED,IAAI,EAAE,CAAC,IAAI,KAAK,OAAO,EAAE,CAAC;QACxB,OAAO,SAAS,CAAC;IACnB,CAAC;IAED,IAAI,EAAE,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;QACvB,MAAM,IAAI,GAAG,CAAC,IAAI,CAAC,EAAE,EAAE,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;QAE3D,OAAO,eAAe,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,eAAe,IAAI,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC;IACpE,CAAC;IAED,IAAI,EAAE,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;QACvB,MAAM,GAAG,GAAG,SAAS,CAAC,EAAE,CAAC,CAAC;QAE1B,IAAI,GAAG,CAAC,QAAQ,CAAC,WAAW,CAAC,EAAE,CAAC;YAC9B,OAAO,wBAAwB,CAAC;QAClC,CAAC;QAED,MAAM,QAAQ,GAAG,IAAI,CAAC,EAAE,EAAE,UAAU,CAAC,EAAE,IAAI,EAAE,CAAC;QAE9C,IAAI,GAAG,CAAC,QAAQ,CAAC,WAAW,CAAC,IAAI,QAAQ,EAAE,CAAC;YAC1C,OAAO,mCAAmC,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC;QACtE,CAAC;IACH,CAAC;IAED,OAAO,IAAI,CAAC;AACd,CAAC,CAAC;AAQF,MAAM,cAAc,GAAG,CAAC,EAAW,EAAW,EAAE;IAC9C,KAAK,IAAI,IAAI,GAAG,EAAE,CAAC,MAAM,EAAE,IAAI,IAAI,KAAK,CAAC,IAAI,CAAC,EAAE,IAAI,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC;QACnE,IAAI,IAAI,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;YACzB,OAAO,KAAK,CAAC;QACf,CAAC;QAED,IAAI,IAAI,CAAC,IAAI,EAAE,QAAQ,CAAC,KAAK,SAAS,EAAE,CAAC;YACvC,OAAO,IAAI,CAAC;QACd,CAAC;IACH,CAAC;IAED,OAAO,KAAK,CAAC;AACf,CAAC,CAAC;AAEF,MAAM,gBAAgB,GAAG,CAAC,IAAa,EAAkB,EAAE,CACzD,QAAQ,CAAC,IAAI,EAAE,8BAA8B,CAAC,CAAC,OAAO,CAAC,CAAC,EAAE,EAAE,EAAE;IAC5D,MAAM,SAAS,GAAG,gBAAgB,CAAC,EAAE,CAAC,CAAC;IAEvC,OAAO,SAAS,CAAC,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,QAAQ,EAAE,cAAc,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;AACxE,CAAC,CAAC,CAAC;AAEL,4FAA4F;AAC5F,MAAM,UAAU,GAAG,CAAC,IAAa,EAAY,EAAE,CAC7C,gBAAgB,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;AAEjD,4FAA4F;AAC5F,MAAM,eAAe,GAAG,IAAI,GAAG,CAAC;IAC9B,CAAC,OAAO,EAAE,iBAAiB,CAAC;IAC5B,CAAC,IAAI,EAAE,wBAAwB,CAAC;CACjC,CAAC,CAAC;AAEH,0FAA0F;AAC1F,MAAM,mBAAmB,GAAG,CAAC,IAAY,EAAU,EAAE,CACnD,CAAC,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;KAC5D,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC;KACX,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;IACT,MAAM,IAAI,GAAG,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;IACnC,MAAM,IAAI,GAAG,eAAe,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;IACvC,MAAM,GAAG,GAAG,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;IAE7D,OAAO,IAAI,CAAC,CAAC,CAAC,KAAK,GAAG,KAAK,IAAI,GAAG,CAAC,CAAC,CAAC,KAAK,GAAG,EAAE,CAAC;AAClD,CAAC,CAAC;KACD,IAAI,CAAC,IAAI,CAAC,CAAC;AAEhB,0EAA0E;AAC1E,MAAM,KAAK,GAAG,CAAC,IAAa,EAAE,MAAc,EAAU,EAAE;IACtD,IAAI,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC;QACjB,MAAM,KAAK,GAAG,kBAAkB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAE5C,IAAI,CAAC,KAAK,EAAE,CAAC;YACX,OAAO,uBAAuB,mBAAmB,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;QACjE,CAAC;QAED,OAAO,SAAS,KAAK,CAAC,MAAM,GAAG,EAAE,CAAC,CAAC,CAAC,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC;IAC1E,CAAC;IAED,MAAM,QAAQ,GAAG,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAC,CAAC,SAAS,CAAC;IAEnE,IAAI,CAAC,QAAQ,EAAE,CAAC;QACd,OAAO,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,IAAI,GAAG,CAAC,CAAC,CAAC,SAAS,CAAC;IACpD,CAAC;IAED,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAClB,QAAQ,CAAC,QAAQ,EAAE,SAAS,IAAI,QAAQ,CAAC,SAAS,EAClD,QAAQ,CAAC,WAAW,GAAG,SAAS,CACjC,CAAC;IAEF,OAAO,kBAAkB,CAAC,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC,WAAW,EAAE,GAAG,CAAC,CAAC,CAAC;AACrE,CAAC,CAAC;AAUF;;;;GAIG;AACH,MAAM,SAAS,GAAG,CAAC,MAAc,EAAE,gBAAyB,EAAa,EAAE;IACzE,MAAM,GAAG,GAAG,KAAK,CAAC,MAAM,EAAE;QACxB,gBAAgB;QAChB,sBAAsB,EAAE,IAAI;QAC5B,WAAW,EAAE,OAAO;KACrB,CAAC,CAAC;IAEH,MAAM,IAAI,GAAG,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC;IAEnC,sFAAsF;IACtF,MAAM,QAAQ,GACZ,IAAI,EAAE,QAAQ,CAAC,IAAI,CACjB,CAAC,IAAI,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAC3E,IAAI,IAAI,CAAC;IAEZ,OAAO;QACL,QAAQ;QACR,UAAU,EAAE,IAAI,IAAI,QAAQ,CAAC,CAAC,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,EAAE;QACpD,GAAG;KACJ,CAAC;AACJ,CAAC,CAAC;AAEF,kFAAkF;AAClF,MAAM,YAAY,GAAG,CAAC,OAAe,EAAe,EAAE;IACpD,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC,IAAI,EAAE,CAAC;IAC5C,MAAM,SAAS,GAAG,KAAK,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;IACxC,MAAM,IAAI,GAAG,CAAC,SAAS,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;IAC3E,IAAI,MAAM,GAAG,SAAS,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,SAAS,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;IAEvE,IAAI,YAAY,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC;QAC9B,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;IACxD,CAAC;IAED,IAAI,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC;QAC1B,MAAM,OAAO,GAAG,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;QACpD,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,OAAO,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC;IACjE,CAAC;IAED,MAAM,OAAO,GAAG,kBAAkB,CAAC,IAAI,CAAC,IAAI,CAAC;QAC3C,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;QAC1B,CAAC,CAAC,IAAI,CAAC;IAET,OAAO,EAAE,OAAO,EAAE,GAAG,EAAE,MAAM,CAAC,IAAI,EAAE,IAAI,IAAI,EAAE,CAAC;AACjD,CAAC,CAAC;AAEF,oGAAoG;AACpG,MAAM,UAAU,GAAG,CAAC,GAAa,EAAE,MAAc,EAAiB,EAAE;IAClE,MAAM,WAAW,GAAG,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC,IAAI,CAC5C,CAAC,EAAE,EAAE,EAAE,CACL,IAAI,CAAC,EAAE,EAAE,SAAS,CAAC,KAAK,SAAS;QACjC,CAAC,IAAI,CAAC,EAAE,EAAE,YAAY,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,KAAK,cAAc,CACzE,CAAC;IAEF,MAAM,GAAG,GAAG,WAAW,EAAE,kBAAkB,EAAE,SAAS,CAAC;IAEvD,OAAO,GAAG,KAAK,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,UAAU,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC;AAC5E,CAAC,CAAC;AAEF,gGAAgG;AAChG,MAAM,aAAa,GAAG,CACpB,KAAqB,EAC2B,EAAE,CAAC,CAAC;IACpD,MAAM,EAAE,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC;IAC/D,cAAc,EAAE,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;CACxE,CAAC,CAAC;AAEH,kGAAkG;AAClG,MAAM,gBAAgB,GAAG,CAAC,IAAwB,EAAW,EAAE;IAC7D,MAAM,CAAC,GAAG,CAAC,IAAI,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;IAE5C,OAAO,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC,QAAQ,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC,QAAQ,CAAC,YAAY,CAAC,CAAC;AAC1E,CAAC,CAAC;AAEF;;;GAGG;AACH,MAAM,eAAe,GAAG,CAAC,GAAa,EAAY,EAAE,CAClD,QAAQ,CAAC,GAAG,EAAE,kBAAkB,CAAC;KAC9B,MAAM,CACL,CAAC,EAAE,EAAE,EAAE,CACL,gBAAgB,CAAC,IAAI,CAAC,EAAE,EAAE,MAAM,CAAC,CAAC;IAClC,IAAI,CAAC,EAAE,EAAE,OAAO,CAAC,KAAK,SAAS;IAC/B,IAAI,CAAC,EAAE,EAAE,OAAO,CAAC,KAAK,SAAS;IAC/B,IAAI,CAAC,EAAE,EAAE,UAAU,CAAC,KAAK,SAAS,CACrC;KACA,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,EAAE,EAAE,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;AAEjD,MAAM,SAAS,GAAG,IAAI,GAAG,CAAC;IACxB,MAAM;IACN,kBAAkB;IAClB,8BAA8B;CAC/B,CAAC,CAAC;AAEH,MAAM,SAAS,GAAG,CAAC,GAAa,EAAY,EAAE,CAC5C,QAAQ,CAAC,GAAG,EAAE,iBAAiB,CAAC;KAC7B,MAAM,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,SAAS,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC;KAC/D,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,EAAE,EAAE,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;AAElD,MAAM,CAAC,MAAM,kBAAkB,GAAG,CAChC,IAAY,EACZ,GAAa,EACA,EAAE;IACf,MAAM,OAAO,GAAG,QAAQ,CAAC,GAAG,EAAE,kBAAkB,CAAC;SAC9C,MAAM,CACL,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,EAAE,EAAE,YAAY,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,KAAK,SAAS,CAC1E;SACA,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,YAAY,CAAC,IAAI,CAAC,EAAE,EAAE,SAAS,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;IAExD,MAAM,IAAI,GAAG,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC;IACnC,MAAM,cAAc,GAAG,IAAI,CAAC,CAAC,CAAC,gBAAgB,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;IAE1D,MAAM,GAAG,GAAG,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAEhC,IAAI,CAAC,GAAG,EAAE,CAAC;QACT,OAAO;YACL,QAAQ,EAAE,IAAI;YACd,gBAAgB,EAAE,KAAK;YACvB,UAAU,EAAE,EAAE;YACd,GAAG,aAAa,CAAC,cAAc,CAAC;YAChC,OAAO;YACP,cAAc,EAAE,IAAI;YACpB,eAAe,EAAE,eAAe,CAAC,GAAG,CAAC;YACrC,KAAK,EAAE,SAAS,CAAC,GAAG,CAAC;SACtB,CAAC;IACJ,CAAC;IAED,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,KAAK,CAAC,CAAC;IACxC,MAAM,OAAO,GAAG,SAAS,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;IAExC,iGAAiG;IACjG,MAAM,KAAK,GAAG,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,SAAS,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC;IACpE,MAAM,EAAE,QAAQ,EAAE,GAAG,KAAK,CAAC;IAE3B,OAAO;QACL,QAAQ,EAAE,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,IAAI;QACnD,gBAAgB,EAAE,CAAC,OAAO,CAAC,QAAQ,IAAI,OAAO,CAAC,QAAQ,CAAC;QACxD,UAAU,EAAE,KAAK,CAAC,UAAU;QAC5B,2FAA2F;QAC3F,GAAG,aAAa,CAAC,cAAc,CAAC,KAAK,CAAC,OAAO,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC;QACjE,OAAO;QACP,cAAc,EAAE,UAAU,CAAC,OAAO,CAAC,GAAG,EAAE,MAAM,CAAC;QAC/C,eAAe,EAAE,eAAe,CAAC,GAAG,CAAC;QACrC,KAAK,EAAE,SAAS,CAAC,GAAG,CAAC;KACtB,CAAC;AACJ,CAAC,CAAC"}
@@ -0,0 +1,17 @@
1
+ import type { ResourceInfo } from "../types.js";
2
+ import type { Document } from "./dom.js";
3
+ /**
4
+ * The URLs of a srcset, read the way the HTML spec does: a URL runs to the next whitespace, so a
5
+ * data: URL keeps its commas, and a trailing comma ends a candidate that has no descriptor. One
6
+ * pass over the attribute.
7
+ */
8
+ export declare const srcsetUrls: (srcset: string) => string[];
9
+ /**
10
+ * A Content-Security-Policy meta tag with upgrade-insecure-requests: browsers then request every
11
+ * http:// resource of the page over https instead of blocking it.
12
+ */
13
+ export declare const upgradesInsecureRequests: (doc: Document) => boolean;
14
+ /** Scripts, stylesheets, frames, images and media the page loads, with absolute URLs. */
15
+ export declare const extractResources: (doc: Document, base: string) => ResourceInfo[];
16
+ /** `href="#name"` links on the page whose target has no element with that id (or `<a name>`). */
17
+ export declare const extractMissingFragments: (doc: Document) => string[];
@@ -0,0 +1,172 @@
1
+ import { isTag } from "domhandler";
2
+ import { resolveUrl } from "../url.js";
3
+ import { attr, queryAll } from "./dom.js";
4
+ /** URLs in a srcset: each candidate is a URL, then an optional width or density. */
5
+ const isSpace = (c) => c === " " || c === "\t" || c === "\n" || c === "\r" || c === "\f";
6
+ /**
7
+ * The URLs of a srcset, read the way the HTML spec does: a URL runs to the next whitespace, so a
8
+ * data: URL keeps its commas, and a trailing comma ends a candidate that has no descriptor. One
9
+ * pass over the attribute.
10
+ */
11
+ export const srcsetUrls = (srcset) => {
12
+ const urls = [];
13
+ let i = 0;
14
+ while (i < srcset.length) {
15
+ while (i < srcset.length &&
16
+ (isSpace(srcset[i] ?? "") || srcset[i] === ",")) {
17
+ i += 1;
18
+ }
19
+ const start = i;
20
+ while (i < srcset.length && !isSpace(srcset[i] ?? "")) {
21
+ i += 1;
22
+ }
23
+ let url = srcset.slice(start, i);
24
+ if (url.endsWith(",")) {
25
+ url = url.replace(/,+$/u, "");
26
+ }
27
+ else {
28
+ // Skip the descriptors ("2x", "480w") up to the comma that ends this candidate.
29
+ let depth = 0;
30
+ while (i < srcset.length && (srcset[i] !== "," || depth > 0)) {
31
+ if (srcset[i] === "(") {
32
+ depth += 1;
33
+ }
34
+ else if (srcset[i] === ")") {
35
+ depth = Math.max(0, depth - 1);
36
+ }
37
+ i += 1;
38
+ }
39
+ }
40
+ if (url) {
41
+ urls.push(url);
42
+ }
43
+ }
44
+ return urls;
45
+ };
46
+ const isInPicture = (el) => el.parent !== null && isTag(el.parent) && el.parent.name === "picture";
47
+ /**
48
+ * Why browsers treat the resource the way they do: images, audio and video are upgraded to https
49
+ * (an image with srcset or in a <picture> is not), everything else is blocked.
50
+ */
51
+ const kindOf = (el) => {
52
+ switch (el.name) {
53
+ case "script": {
54
+ return "script";
55
+ }
56
+ case "link": {
57
+ return "stylesheet";
58
+ }
59
+ case "iframe": {
60
+ return "frame";
61
+ }
62
+ case "embed":
63
+ case "object": {
64
+ return "object";
65
+ }
66
+ case "img": {
67
+ return attr(el, "srcset") || isInPicture(el)
68
+ ? "responsive-image"
69
+ : "image";
70
+ }
71
+ case "source": {
72
+ return isInPicture(el) ? "responsive-image" : "media";
73
+ }
74
+ case "audio":
75
+ case "video": {
76
+ return "media";
77
+ }
78
+ // Subtitles are not among the destinations browsers upgrade (image, audio, video).
79
+ case "track": {
80
+ return "track";
81
+ }
82
+ default: {
83
+ return null;
84
+ }
85
+ }
86
+ };
87
+ const urlsOf = (el) => {
88
+ if (el.name === "link") {
89
+ return [attr(el, "href") ?? ""];
90
+ }
91
+ if (el.name === "object") {
92
+ return [attr(el, "data") ?? ""];
93
+ }
94
+ return [
95
+ attr(el, "src") ?? "",
96
+ ...srcsetUrls(attr(el, "srcset") ?? ""),
97
+ ...(el.name === "video" ? [attr(el, "poster") ?? ""] : []),
98
+ ];
99
+ };
100
+ /**
101
+ * A Content-Security-Policy meta tag with upgrade-insecure-requests: browsers then request every
102
+ * http:// resource of the page over https instead of blocking it.
103
+ */
104
+ export const upgradesInsecureRequests = (doc) => queryAll(doc, "meta[http-equiv]").some((el) => (attr(el, "http-equiv") ?? "").trim().toLowerCase() ===
105
+ "content-security-policy" &&
106
+ /(?:^|;)\s*upgrade-insecure-requests\s*(?:;|$)/iu.test(attr(el, "content") ?? ""));
107
+ /** Scripts, stylesheets, frames, images and media the page loads, with absolute URLs. */
108
+ export const extractResources = (doc, base) => {
109
+ const elements = queryAll(doc, 'script[src], link[rel~="stylesheet" i][href], iframe[src], embed[src], object[data], img, source, audio[src], video, track[src]');
110
+ const out = [];
111
+ for (const el of elements) {
112
+ const kind = kindOf(el);
113
+ if (!kind) {
114
+ continue;
115
+ }
116
+ for (const raw of urlsOf(el)) {
117
+ const url = raw.trim() ? resolveUrl(raw, base) : null;
118
+ if (url) {
119
+ out.push({ kind, url });
120
+ }
121
+ }
122
+ }
123
+ return out;
124
+ };
125
+ /** The text a fragment link targets, or null for links that do not target an element. */
126
+ const fragmentTarget = (href) => {
127
+ let fragment = href.slice(1);
128
+ try {
129
+ fragment = decodeURIComponent(fragment);
130
+ }
131
+ catch {
132
+ // A malformed escape is compared as written.
133
+ }
134
+ // "#top" scrolls to the top without an element; "#!" and "#/" are client-side routes,
135
+ // "#:~:text=" is a text fragment, and a fragment with "=" carries state for a script (a tab, a
136
+ // popup such as Elementor's "#elementor-action:action=popup:open"), not an element id.
137
+ if (!fragment ||
138
+ fragment.toLowerCase() === "top" ||
139
+ /^(?:!|\/|:~:)/u.test(fragment) ||
140
+ fragment.includes("=")) {
141
+ return null;
142
+ }
143
+ return fragment;
144
+ };
145
+ /** `href="#name"` links on the page whose target has no element with that id (or `<a name>`). */
146
+ export const extractMissingFragments = (doc) => {
147
+ const links = queryAll(doc, 'a[href^="#"]');
148
+ if (links.length === 0) {
149
+ return [];
150
+ }
151
+ const targets = new Set();
152
+ for (const el of queryAll(doc, "[id], a[name]")) {
153
+ for (const id of [
154
+ attr(el, "id"),
155
+ el.name === "a" ? attr(el, "name") : undefined,
156
+ ]) {
157
+ if (id) {
158
+ targets.add(id);
159
+ }
160
+ }
161
+ }
162
+ const missing = new Set();
163
+ for (const el of links) {
164
+ const href = (attr(el, "href") ?? "").trim();
165
+ const target = fragmentTarget(href);
166
+ if (target !== null && !targets.has(target)) {
167
+ missing.add(href);
168
+ }
169
+ }
170
+ return [...missing];
171
+ };
172
+ //# sourceMappingURL=resources.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"resources.js","sourceRoot":"","sources":["../../../src/core/html/resources.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,MAAM,YAAY,CAAC;AAGnC,OAAO,EAAE,UAAU,EAAE,MAAM,WAAW,CAAC;AACvC,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,MAAM,UAAU,CAAC;AAG1C,oFAAoF;AACpF,MAAM,OAAO,GAAG,CAAC,CAAS,EAAW,EAAE,CACrC,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,KAAK,IAAI,CAAC;AAEpE;;;;GAIG;AACH,MAAM,CAAC,MAAM,UAAU,GAAG,CAAC,MAAc,EAAY,EAAE;IACrD,MAAM,IAAI,GAAa,EAAE,CAAC;IAC1B,IAAI,CAAC,GAAG,CAAC,CAAC;IAEV,OAAO,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC;QACzB,OACE,CAAC,GAAG,MAAM,CAAC,MAAM;YACjB,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,IAAI,MAAM,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,EAC/C,CAAC;YACD,CAAC,IAAI,CAAC,CAAC;QACT,CAAC;QAED,MAAM,KAAK,GAAG,CAAC,CAAC;QAEhB,OAAO,CAAC,GAAG,MAAM,CAAC,MAAM,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,EAAE,CAAC;YACtD,CAAC,IAAI,CAAC,CAAC;QACT,CAAC;QAED,IAAI,GAAG,GAAG,MAAM,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;QAEjC,IAAI,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;YACtB,GAAG,GAAG,GAAG,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;QAChC,CAAC;aAAM,CAAC;YACN,gFAAgF;YAChF,IAAI,KAAK,GAAG,CAAC,CAAC;YAEd,OAAO,CAAC,GAAG,MAAM,CAAC,MAAM,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,GAAG,IAAI,KAAK,GAAG,CAAC,CAAC,EAAE,CAAC;gBAC7D,IAAI,MAAM,CAAC,CAAC,CAAC,KAAK,GAAG,EAAE,CAAC;oBACtB,KAAK,IAAI,CAAC,CAAC;gBACb,CAAC;qBAAM,IAAI,MAAM,CAAC,CAAC,CAAC,KAAK,GAAG,EAAE,CAAC;oBAC7B,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,GAAG,CAAC,CAAC,CAAC;gBACjC,CAAC;gBAED,CAAC,IAAI,CAAC,CAAC;YACT,CAAC;QACH,CAAC;QAED,IAAI,GAAG,EAAE,CAAC;YACR,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACjB,CAAC;IACH,CAAC;IAED,OAAO,IAAI,CAAC;AACd,CAAC,CAAC;AAEF,MAAM,WAAW,GAAG,CAAC,EAAW,EAAW,EAAE,CAC3C,EAAE,CAAC,MAAM,KAAK,IAAI,IAAI,KAAK,CAAC,EAAE,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC,MAAM,CAAC,IAAI,KAAK,SAAS,CAAC;AAEzE;;;GAGG;AACH,MAAM,MAAM,GAAG,CAAC,EAAW,EAA+B,EAAE;IAC1D,QAAQ,EAAE,CAAC,IAAI,EAAE,CAAC;QAChB,KAAK,QAAQ,EAAE,CAAC;YACd,OAAO,QAAQ,CAAC;QAClB,CAAC;QAED,KAAK,MAAM,EAAE,CAAC;YACZ,OAAO,YAAY,CAAC;QACtB,CAAC;QAED,KAAK,QAAQ,EAAE,CAAC;YACd,OAAO,OAAO,CAAC;QACjB,CAAC;QAED,KAAK,OAAO,CAAC;QACb,KAAK,QAAQ,EAAE,CAAC;YACd,OAAO,QAAQ,CAAC;QAClB,CAAC;QAED,KAAK,KAAK,EAAE,CAAC;YACX,OAAO,IAAI,CAAC,EAAE,EAAE,QAAQ,CAAC,IAAI,WAAW,CAAC,EAAE,CAAC;gBAC1C,CAAC,CAAC,kBAAkB;gBACpB,CAAC,CAAC,OAAO,CAAC;QACd,CAAC;QAED,KAAK,QAAQ,EAAE,CAAC;YACd,OAAO,WAAW,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,kBAAkB,CAAC,CAAC,CAAC,OAAO,CAAC;QACxD,CAAC;QAED,KAAK,OAAO,CAAC;QACb,KAAK,OAAO,EAAE,CAAC;YACb,OAAO,OAAO,CAAC;QACjB,CAAC;QAED,mFAAmF;QACnF,KAAK,OAAO,EAAE,CAAC;YACb,OAAO,OAAO,CAAC;QACjB,CAAC;QAED,SAAS,CAAC;YACR,OAAO,IAAI,CAAC;QACd,CAAC;IACH,CAAC;AACH,CAAC,CAAC;AAEF,MAAM,MAAM,GAAG,CAAC,EAAW,EAAY,EAAE;IACvC,IAAI,EAAE,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;QACvB,OAAO,CAAC,IAAI,CAAC,EAAE,EAAE,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC;IAClC,CAAC;IAED,IAAI,EAAE,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;QACzB,OAAO,CAAC,IAAI,CAAC,EAAE,EAAE,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC;IAClC,CAAC;IAED,OAAO;QACL,IAAI,CAAC,EAAE,EAAE,KAAK,CAAC,IAAI,EAAE;QACrB,GAAG,UAAU,CAAC,IAAI,CAAC,EAAE,EAAE,QAAQ,CAAC,IAAI,EAAE,CAAC;QACvC,GAAG,CAAC,EAAE,CAAC,IAAI,KAAK,OAAO,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,EAAE,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;KAC3D,CAAC;AACJ,CAAC,CAAC;AAEF;;;GAGG;AACH,MAAM,CAAC,MAAM,wBAAwB,GAAG,CAAC,GAAa,EAAW,EAAE,CACjE,QAAQ,CAAC,GAAG,EAAE,kBAAkB,CAAC,CAAC,IAAI,CACpC,CAAC,EAAE,EAAE,EAAE,CACL,CAAC,IAAI,CAAC,EAAE,EAAE,YAAY,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE;IACjD,yBAAyB;IAC3B,iDAAiD,CAAC,IAAI,CACpD,IAAI,CAAC,EAAE,EAAE,SAAS,CAAC,IAAI,EAAE,CAC1B,CACJ,CAAC;AAEJ,yFAAyF;AACzF,MAAM,CAAC,MAAM,gBAAgB,GAAG,CAC9B,GAAa,EACb,IAAY,EACI,EAAE;IAClB,MAAM,QAAQ,GAAG,QAAQ,CACvB,GAAG,EACH,iIAAiI,CAClI,CAAC;IAEF,MAAM,GAAG,GAAmB,EAAE,CAAC;IAE/B,KAAK,MAAM,EAAE,IAAI,QAAQ,EAAE,CAAC;QAC1B,MAAM,IAAI,GAAG,MAAM,CAAC,EAAE,CAAC,CAAC;QAExB,IAAI,CAAC,IAAI,EAAE,CAAC;YACV,SAAS;QACX,CAAC;QAED,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,EAAE,CAAC,EAAE,CAAC;YAC7B,MAAM,GAAG,GAAG,GAAG,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,UAAU,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;YAEtD,IAAI,GAAG,EAAE,CAAC;gBACR,GAAG,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,GAAG,EAAE,CAAC,CAAC;YAC1B,CAAC;QACH,CAAC;IACH,CAAC;IAED,OAAO,GAAG,CAAC;AACb,CAAC,CAAC;AAEF,yFAAyF;AACzF,MAAM,cAAc,GAAG,CAAC,IAAY,EAAiB,EAAE;IACrD,IAAI,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;IAE7B,IAAI,CAAC;QACH,QAAQ,GAAG,kBAAkB,CAAC,QAAQ,CAAC,CAAC;IAC1C,CAAC;IAAC,MAAM,CAAC;QACP,6CAA6C;IAC/C,CAAC;IAED,sFAAsF;IACtF,+FAA+F;IAC/F,uFAAuF;IACvF,IACE,CAAC,QAAQ;QACT,QAAQ,CAAC,WAAW,EAAE,KAAK,KAAK;QAChC,gBAAgB,CAAC,IAAI,CAAC,QAAQ,CAAC;QAC/B,QAAQ,CAAC,QAAQ,CAAC,GAAG,CAAC,EACtB,CAAC;QACD,OAAO,IAAI,CAAC;IACd,CAAC;IAED,OAAO,QAAQ,CAAC;AAClB,CAAC,CAAC;AAEF,iGAAiG;AACjG,MAAM,CAAC,MAAM,uBAAuB,GAAG,CAAC,GAAa,EAAY,EAAE;IACjE,MAAM,KAAK,GAAG,QAAQ,CAAC,GAAG,EAAE,cAAc,CAAC,CAAC;IAE5C,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACvB,OAAO,EAAE,CAAC;IACZ,CAAC;IAED,MAAM,OAAO,GAAG,IAAI,GAAG,EAAU,CAAC;IAElC,KAAK,MAAM,EAAE,IAAI,QAAQ,CAAC,GAAG,EAAE,eAAe,CAAC,EAAE,CAAC;QAChD,KAAK,MAAM,EAAE,IAAI;YACf,IAAI,CAAC,EAAE,EAAE,IAAI,CAAC;YACd,EAAE,CAAC,IAAI,KAAK,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,SAAS;SAC/C,EAAE,CAAC;YACF,IAAI,EAAE,EAAE,CAAC;gBACP,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;YAClB,CAAC;QACH,CAAC;IACH,CAAC;IAED,MAAM,OAAO,GAAG,IAAI,GAAG,EAAU,CAAC;IAElC,KAAK,MAAM,EAAE,IAAI,KAAK,EAAE,CAAC;QACvB,MAAM,IAAI,GAAG,CAAC,IAAI,CAAC,EAAE,EAAE,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;QAC7C,MAAM,MAAM,GAAG,cAAc,CAAC,IAAI,CAAC,CAAC;QAEpC,IAAI,MAAM,KAAK,IAAI,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC;YAC5C,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QACpB,CAAC;IACH,CAAC;IAED,OAAO,CAAC,GAAG,OAAO,CAAC,CAAC;AACtB,CAAC,CAAC"}
@@ -8,6 +8,8 @@ export interface JsonObject {
8
8
  }
9
9
  /** Whether a structured-data value is a string. */
10
10
  export declare const isString: (value: JsonValue | undefined) => value is string;
11
+ /** Whether a structured-data value is a JSON number. */
12
+ export declare const isNumber: (value: JsonValue | undefined) => value is number;
11
13
  /** Whether a structured-data value is an object with properties, rather than a list or a scalar. */
12
14
  export declare const isJsonObject: (value: JsonValue | undefined) => value is JsonObject;
13
15
  export interface StructuredDataExtraction {
@@ -3,6 +3,8 @@ import { attr, childElements, hasAttr, queryAll, text } from "./dom.js";
3
3
  import { collapseWhitespace } from "./text.js";
4
4
  /** Whether a structured-data value is a string. */
5
5
  export const isString = (value) => typeof value === "string";
6
+ /** Whether a structured-data value is a JSON number. */
7
+ export const isNumber = (value) => typeof value === "number";
6
8
  /** Whether a structured-data value is an object with properties, rather than a list or a scalar. */
7
9
  export const isJsonObject = (value) => value !== null && typeof value === "object" && !Array.isArray(value);
8
10
  const stripCdata = (source) => source