@crawlee/utils 4.0.0-beta.249 → 4.0.0-beta.250
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/internals/cheerio.js +18 -10
- package/package.json +4 -4
package/internals/cheerio.js
CHANGED
|
@@ -34,7 +34,15 @@ export async function htmlToText(htmlOrCheerioElement) {
|
|
|
34
34
|
if (!htmlOrCheerioElement)
|
|
35
35
|
return '';
|
|
36
36
|
const $ = typeof htmlOrCheerioElement === 'function' ? htmlOrCheerioElement : load(htmlOrCheerioElement);
|
|
37
|
-
|
|
37
|
+
// Never read the accumulated text mid-loop: each read flattens V8's cons-string, making the traversal quadratic
|
|
38
|
+
const parts = [];
|
|
39
|
+
let last = '';
|
|
40
|
+
const push = (part) => {
|
|
41
|
+
if (!part)
|
|
42
|
+
return;
|
|
43
|
+
parts.push(part);
|
|
44
|
+
last = part[part.length - 1];
|
|
45
|
+
};
|
|
38
46
|
const process = (elems) => {
|
|
39
47
|
const len = elems ? elems.length : 0;
|
|
40
48
|
for (let i = 0; i < len; i++) {
|
|
@@ -49,35 +57,35 @@ export async function htmlToText(htmlOrCheerioElement) {
|
|
|
49
57
|
else
|
|
50
58
|
compr = elem.data.replace(/\s+/g, ' ');
|
|
51
59
|
// If text is empty or ends with a whitespace, don't add the leading whitespace
|
|
52
|
-
if (compr.startsWith(' ') &&
|
|
60
|
+
if (compr.startsWith(' ') && (last === '' || /\s/.test(last)))
|
|
53
61
|
compr = compr.substring(1);
|
|
54
|
-
|
|
62
|
+
push(compr);
|
|
55
63
|
}
|
|
56
64
|
else if (elem.type === 'comment' || SKIP_TAGS_REGEX.test(elem.tagName)) {
|
|
57
65
|
// Skip comments and special elements
|
|
58
66
|
}
|
|
59
67
|
else if (elem.tagName === 'br') {
|
|
60
|
-
|
|
68
|
+
push('\n');
|
|
61
69
|
}
|
|
62
70
|
else if (elem.tagName === 'td') {
|
|
63
71
|
process(elem.children);
|
|
64
|
-
|
|
72
|
+
push('\t');
|
|
65
73
|
}
|
|
66
74
|
else {
|
|
67
75
|
// Block elements must be surrounded by newlines (unless beginning of text)
|
|
68
76
|
const isBlockTag = BLOCK_TAGS_REGEX.test(elem.tagName);
|
|
69
|
-
if (isBlockTag &&
|
|
70
|
-
|
|
77
|
+
if (isBlockTag && last !== '' && last !== '\n')
|
|
78
|
+
push('\n');
|
|
71
79
|
process(elem.children);
|
|
72
|
-
if (isBlockTag &&
|
|
73
|
-
|
|
80
|
+
if (isBlockTag && last !== '\n')
|
|
81
|
+
push('\n');
|
|
74
82
|
}
|
|
75
83
|
}
|
|
76
84
|
};
|
|
77
85
|
// If HTML document has body, only convert that, otherwise convert the entire HTML
|
|
78
86
|
const $body = $('body');
|
|
79
87
|
process($body.length > 0 ? $body : $.root());
|
|
80
|
-
return
|
|
88
|
+
return parts.join('').trim();
|
|
81
89
|
}
|
|
82
90
|
/**
|
|
83
91
|
* Extracts URLs from a given Cheerio object.
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@crawlee/utils",
|
|
3
|
-
"version": "4.0.0-beta.
|
|
3
|
+
"version": "4.0.0-beta.250",
|
|
4
4
|
"description": "A set of shared utilities that can be used by crawlers",
|
|
5
5
|
"engines": {
|
|
6
6
|
"node": ">=22.13.0"
|
|
@@ -43,8 +43,8 @@
|
|
|
43
43
|
},
|
|
44
44
|
"dependencies": {
|
|
45
45
|
"@apify/validations": "^1.1.2",
|
|
46
|
-
"@crawlee/http-client": "4.0.0-beta.
|
|
47
|
-
"@crawlee/types": "4.0.0-beta.
|
|
46
|
+
"@crawlee/http-client": "4.0.0-beta.250",
|
|
47
|
+
"@crawlee/types": "4.0.0-beta.250",
|
|
48
48
|
"@types/sax": "^1.2.7",
|
|
49
49
|
"cheerio": "^1.2.0",
|
|
50
50
|
"domhandler": "^6.0.1",
|
|
@@ -63,5 +63,5 @@
|
|
|
63
63
|
}
|
|
64
64
|
}
|
|
65
65
|
},
|
|
66
|
-
"gitHead": "
|
|
66
|
+
"gitHead": "894ea551fedd04fbf2394731fb85010468ccc31b"
|
|
67
67
|
}
|