@crawlee/utils 4.0.0-beta.283 → 4.0.0-beta.285

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,6 +1,6 @@
1
1
  import { tryAbsoluteURL } from './extract-urls.js';
2
2
  // NOTE: We are skipping 'noscript' since it's content is evaluated as text, instead of HTML elements. That damages the results.
3
- const SKIP_TAGS_REGEX = /^(script|style|canvas|svg|noscript)$/i;
3
+ const SKIP_TAGS_REGEX = /^(script|style|canvas|svg|noscript|template)$/i;
4
4
  const BLOCK_TAGS_REGEX = /^(p|h1|h2|h3|h4|h5|h6|ol|ul|li|pre|address|blockquote|dl|dt|dd|div|fieldset|legend|form|table|caption|tr|select|option|hr|article|aside|section|nav|header|footer|main|hgroup|figure|figcaption|details|summary|dialog|menu)$/i;
5
5
  /**
6
6
  * The function converts a HTML document to a plain text.
@@ -6,7 +6,7 @@ export async function extractMicrodata(htmlOrCheerioElement) {
6
6
  const context = { $ };
7
7
  return $('[itemscope]')
8
8
  .toArray()
9
- .filter((element) => !('itemprop' in element.attribs))
9
+ .filter((element) => !('itemprop' in element.attribs) && !isInsideTemplate(element))
10
10
  .map((element) => parseItem(context, element, new Set()));
11
11
  }
12
12
  function parseItem(context, element, ancestors) {
@@ -53,7 +53,8 @@ function collectFromNodes(nodes, elements, visited) {
53
53
  elements.push(node);
54
54
  }
55
55
  // A nested item owns everything below it, so its subtree is not part of the enclosing item.
56
- if (!('itemscope' in node.attribs)) {
56
+ // A `<template>`'s content is an inert fragment, not part of the document.
57
+ if (!('itemscope' in node.attribs) && node.tagName.toLowerCase() !== 'template') {
57
58
  collectFromNodes(node.children, elements, visited);
58
59
  }
59
60
  }
@@ -62,12 +63,25 @@ function indexIds($) {
62
63
  const index = new Map();
63
64
  for (const element of $('[id]').toArray()) {
64
65
  // Duplicate ids are invalid HTML; `getElementById` resolves them to the first element.
65
- if (!index.has(element.attribs.id)) {
66
+ if (!index.has(element.attribs.id) && !isInsideTemplate(element)) {
66
67
  index.set(element.attribs.id, element);
67
68
  }
68
69
  }
69
70
  return index;
70
71
  }
72
+ /**
73
+ * Cheerio keeps a `<template>`'s content as its children, but in the DOM that content is a separate
74
+ * inert fragment, so its items and ids are not part of the document (framework placeholders like
75
+ * `<template x-for>` would otherwise show up as items with empty or `{{ mustache }}` values).
76
+ */
77
+ function isInsideTemplate(element) {
78
+ for (let parent = element.parent; parent; parent = parent.parent) {
79
+ if (isTag(parent) && parent.tagName.toLowerCase() === 'template') {
80
+ return true;
81
+ }
82
+ }
83
+ return false;
84
+ }
71
85
  function getPropertyValue(context, element, ancestors) {
72
86
  if ('itemscope' in element.attribs) {
73
87
  // `itemref` can point back at an enclosing item, which the spec treats as an error.
@@ -35,12 +35,21 @@ class SitemapTxtParser extends Transform {
35
35
  }
36
36
  }
37
37
  }
38
+ /** Namespaces of the sitemap protocol (https://www.sitemaps.org/protocol.html), including Google's legacy ones. */
39
+ const SITEMAP_NAMESPACES = new Set([
40
+ 'http://www.sitemaps.org/schemas/sitemap/0.9',
41
+ 'https://www.sitemaps.org/schemas/sitemap/0.9',
42
+ 'http://www.google.com/schemas/sitemap/0.9',
43
+ 'http://www.google.com/schemas/sitemap/0.84',
44
+ ]);
38
45
  class SitemapXmlParser extends Transform {
39
46
  #decoder = new StringDecoder('utf8');
40
47
  #parser;
41
48
  #rootTagName;
42
49
  #currentTag = undefined;
43
50
  #url = {};
51
+ /** Prefixes bound to the sitemap namespace, e.g. `ns0` for `<ns0:urlset xmlns:ns0="...">`. */
52
+ #sitemapPrefixes = new Set();
44
53
  static async create() {
45
54
  const { SAXParser } = (await import('sax')).default;
46
55
  return new SitemapXmlParser(new SAXParser(true));
@@ -68,23 +77,37 @@ class SitemapXmlParser extends Transform {
68
77
  this.#parser.oncdata = this.#onText.bind(this);
69
78
  this.#parser.onerror = this.destroy.bind(this);
70
79
  }
80
+ /**
81
+ * Returns the local name of a sitemap protocol element, or `undefined` for an element of another
82
+ * namespace (such as `image:loc`). Unprefixed elements are sitemap elements, as before.
83
+ */
84
+ #sitemapTagName(qualifiedName) {
85
+ const colon = qualifiedName.indexOf(':');
86
+ if (colon === -1)
87
+ return qualifiedName;
88
+ return this.#sitemapPrefixes.has(qualifiedName.slice(0, colon)) ? qualifiedName.slice(colon + 1) : undefined;
89
+ }
71
90
  #onOpenTag(node) {
91
+ for (const [attribute, value] of Object.entries(node.attributes)) {
92
+ if (attribute.startsWith('xmlns:') && SITEMAP_NAMESPACES.has(String(value).trim())) {
93
+ this.#sitemapPrefixes.add(attribute.slice('xmlns:'.length));
94
+ }
95
+ }
96
+ const name = this.#sitemapTagName(node.name);
72
97
  if (this.#rootTagName !== undefined) {
73
- if (node.name === 'loc' ||
74
- node.name === 'lastmod' ||
75
- node.name === 'priority' ||
76
- node.name === 'changefreq') {
77
- this.#currentTag = node.name;
98
+ if (name === 'loc' || name === 'lastmod' || name === 'priority' || name === 'changefreq') {
99
+ this.#currentTag = name;
78
100
  }
79
101
  }
80
- if (node.name === 'urlset') {
102
+ if (name === 'urlset') {
81
103
  this.#rootTagName = 'urlset';
82
104
  }
83
- if (node.name === 'sitemapindex') {
105
+ if (name === 'sitemapindex') {
84
106
  this.#rootTagName = 'sitemapindex';
85
107
  }
86
108
  }
87
- #onCloseTag(name) {
109
+ #onCloseTag(qualifiedName) {
110
+ const name = this.#sitemapTagName(qualifiedName);
88
111
  if (name === 'loc' || name === 'lastmod' || name === 'priority' || name === 'changefreq') {
89
112
  this.#currentTag = undefined;
90
113
  }
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@crawlee/utils",
3
- "version": "4.0.0-beta.283",
3
+ "version": "4.0.0-beta.285",
4
4
  "description": "A set of shared utilities that can be used by crawlers",
5
5
  "engines": {
6
6
  "node": ">=22.13.0"
@@ -43,15 +43,15 @@
43
43
  },
44
44
  "dependencies": {
45
45
  "@apify/validations": "^1.1.4",
46
- "@crawlee/http-client": "4.0.0-beta.283",
47
- "@crawlee/types": "4.0.0-beta.283",
46
+ "@crawlee/http-client": "4.0.0-beta.285",
47
+ "@crawlee/types": "4.0.0-beta.285",
48
48
  "@types/sax": "^1.2.7",
49
49
  "cheerio": "^1.2.0",
50
50
  "domhandler": "^6.0.1",
51
51
  "file-type": "^21.3.4",
52
52
  "robots-parser": "^3.0.1",
53
53
  "sax": "^1.6.1",
54
- "tldts": "^7.4.19",
54
+ "tldts": "^7.4.22",
55
55
  "tslib": "^2.9.1",
56
56
  "whatwg-mimetype": "^5.0.0",
57
57
  "zod": "^4.6.7"
@@ -63,5 +63,5 @@
63
63
  }
64
64
  }
65
65
  },
66
- "gitHead": "204d05fc6843b3e220598ad342b6547a7f583eb1"
66
+ "gitHead": "cab9875edf1eb61e9e1a1ae1245db441036915d9"
67
67
  }