@crawlee/utils 4.0.0-beta.260 → 4.0.0-beta.261
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -31,26 +31,30 @@ function parseItem(context, element, ancestors) {
|
|
|
31
31
|
}
|
|
32
32
|
function collectPropertyElements(context, scope) {
|
|
33
33
|
const elements = [];
|
|
34
|
-
|
|
34
|
+
// An `itemref` can point at an element already reached from the item's own subtree, or at one that
|
|
35
|
+
// contains another referenced element. The spec visits every element once, so each property counts once.
|
|
36
|
+
const visited = new Set();
|
|
37
|
+
collectFromNodes(scope.children, elements, visited);
|
|
35
38
|
for (const id of uniqueTokens(scope.attribs.itemref)) {
|
|
36
39
|
const referenced = (context.idIndex ??= indexIds(context.$)).get(id);
|
|
37
40
|
if (referenced) {
|
|
38
|
-
collectFromNodes([referenced], elements);
|
|
41
|
+
collectFromNodes([referenced], elements, visited);
|
|
39
42
|
}
|
|
40
43
|
}
|
|
41
44
|
return elements;
|
|
42
45
|
}
|
|
43
|
-
function collectFromNodes(nodes, elements) {
|
|
46
|
+
function collectFromNodes(nodes, elements, visited) {
|
|
44
47
|
for (const node of nodes) {
|
|
45
|
-
if (!isTag(node)) {
|
|
48
|
+
if (!isTag(node) || visited.has(node)) {
|
|
46
49
|
continue;
|
|
47
50
|
}
|
|
51
|
+
visited.add(node);
|
|
48
52
|
if ('itemprop' in node.attribs) {
|
|
49
53
|
elements.push(node);
|
|
50
54
|
}
|
|
51
55
|
// A nested item owns everything below it, so its subtree is not part of the enclosing item.
|
|
52
56
|
if (!('itemscope' in node.attribs)) {
|
|
53
|
-
collectFromNodes(node.children, elements);
|
|
57
|
+
collectFromNodes(node.children, elements, visited);
|
|
54
58
|
}
|
|
55
59
|
}
|
|
56
60
|
}
|
package/internals/sitemap.js
CHANGED
|
@@ -144,6 +144,10 @@ export async function* parseSitemap(initialSources, proxyUrl, options) {
|
|
|
144
144
|
if (mimeType?.essence === 'text/plain' || url?.pathname.endsWith('.txt')) {
|
|
145
145
|
return new SitemapTxtParser();
|
|
146
146
|
}
|
|
147
|
+
// A name like `sitemap.gz` says nothing about what was compressed. Sitemaps are XML unless named otherwise.
|
|
148
|
+
if (mimeType?.essence === 'application/gzip') {
|
|
149
|
+
return SitemapXmlParser.create();
|
|
150
|
+
}
|
|
147
151
|
throw new Error(`Unsupported sitemap content type (contentType = ${contentType}, url = ${url?.toString()})`);
|
|
148
152
|
};
|
|
149
153
|
while (sources.length > 0) {
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@crawlee/utils",
|
|
3
|
-
"version": "4.0.0-beta.
|
|
3
|
+
"version": "4.0.0-beta.261",
|
|
4
4
|
"description": "A set of shared utilities that can be used by crawlers",
|
|
5
5
|
"engines": {
|
|
6
6
|
"node": ">=22.13.0"
|
|
@@ -43,8 +43,8 @@
|
|
|
43
43
|
},
|
|
44
44
|
"dependencies": {
|
|
45
45
|
"@apify/validations": "^1.1.2",
|
|
46
|
-
"@crawlee/http-client": "4.0.0-beta.
|
|
47
|
-
"@crawlee/types": "4.0.0-beta.
|
|
46
|
+
"@crawlee/http-client": "4.0.0-beta.261",
|
|
47
|
+
"@crawlee/types": "4.0.0-beta.261",
|
|
48
48
|
"@types/sax": "^1.2.7",
|
|
49
49
|
"cheerio": "^1.2.0",
|
|
50
50
|
"domhandler": "^6.0.1",
|
|
@@ -63,5 +63,5 @@
|
|
|
63
63
|
}
|
|
64
64
|
}
|
|
65
65
|
},
|
|
66
|
-
"gitHead": "
|
|
66
|
+
"gitHead": "24e897c21c5f7ba19846bbca3eba4def5331a081"
|
|
67
67
|
}
|