@crawlee/utils 4.0.0-beta.266 → 4.0.0-beta.268
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/internals/cheerio.js +8 -7
- package/package.json +4 -4
package/internals/cheerio.js
CHANGED
|
@@ -43,21 +43,22 @@ export async function htmlToText(htmlOrCheerioElement) {
|
|
|
43
43
|
parts.push(part);
|
|
44
44
|
last = part[part.length - 1];
|
|
45
45
|
};
|
|
46
|
-
const process = (elems) => {
|
|
46
|
+
const process = (elems, inPre = false) => {
|
|
47
47
|
const len = elems ? elems.length : 0;
|
|
48
48
|
for (let i = 0; i < len; i++) {
|
|
49
49
|
const elem = elems[i];
|
|
50
50
|
if (elem.type === 'text') {
|
|
51
|
-
// Compress spaces, unless we're inside <pre> element
|
|
51
|
+
// Compress spaces, unless we're inside <pre> element (including nested elements like <pre><code>)
|
|
52
52
|
let compr;
|
|
53
|
-
if (
|
|
53
|
+
if (inPre) {
|
|
54
54
|
// Strip single newline right after `<pre>` that htmlparser2 keeps
|
|
55
|
-
|
|
55
|
+
const isFirstInPre = elem.parent?.tagName === 'pre' && elem.parent.children?.[0] === elem;
|
|
56
|
+
compr = isFirstInPre ? elem.data.replace(/^\r?\n/, '') : elem.data;
|
|
56
57
|
}
|
|
57
58
|
else
|
|
58
59
|
compr = elem.data.replace(/\s+/g, ' ');
|
|
59
60
|
// If text is empty or ends with a whitespace, don't add the leading whitespace
|
|
60
|
-
if (compr.startsWith(' ') && (last === '' || /\s/.test(last)))
|
|
61
|
+
if (!inPre && compr.startsWith(' ') && (last === '' || /\s/.test(last)))
|
|
61
62
|
compr = compr.substring(1);
|
|
62
63
|
push(compr);
|
|
63
64
|
}
|
|
@@ -68,7 +69,7 @@ export async function htmlToText(htmlOrCheerioElement) {
|
|
|
68
69
|
push('\n');
|
|
69
70
|
}
|
|
70
71
|
else if (elem.tagName === 'td') {
|
|
71
|
-
process(elem.children);
|
|
72
|
+
process(elem.children, inPre);
|
|
72
73
|
push('\t');
|
|
73
74
|
}
|
|
74
75
|
else {
|
|
@@ -76,7 +77,7 @@ export async function htmlToText(htmlOrCheerioElement) {
|
|
|
76
77
|
const isBlockTag = BLOCK_TAGS_REGEX.test(elem.tagName);
|
|
77
78
|
if (isBlockTag && last !== '' && last !== '\n')
|
|
78
79
|
push('\n');
|
|
79
|
-
process(elem.children);
|
|
80
|
+
process(elem.children, inPre || elem.tagName === 'pre');
|
|
80
81
|
if (isBlockTag && last !== '\n')
|
|
81
82
|
push('\n');
|
|
82
83
|
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@crawlee/utils",
|
|
3
|
-
"version": "4.0.0-beta.
|
|
3
|
+
"version": "4.0.0-beta.268",
|
|
4
4
|
"description": "A set of shared utilities that can be used by crawlers",
|
|
5
5
|
"engines": {
|
|
6
6
|
"node": ">=22.13.0"
|
|
@@ -43,8 +43,8 @@
|
|
|
43
43
|
},
|
|
44
44
|
"dependencies": {
|
|
45
45
|
"@apify/validations": "^1.1.2",
|
|
46
|
-
"@crawlee/http-client": "4.0.0-beta.
|
|
47
|
-
"@crawlee/types": "4.0.0-beta.
|
|
46
|
+
"@crawlee/http-client": "4.0.0-beta.268",
|
|
47
|
+
"@crawlee/types": "4.0.0-beta.268",
|
|
48
48
|
"@types/sax": "^1.2.7",
|
|
49
49
|
"cheerio": "^1.2.0",
|
|
50
50
|
"domhandler": "^6.0.1",
|
|
@@ -63,5 +63,5 @@
|
|
|
63
63
|
}
|
|
64
64
|
}
|
|
65
65
|
},
|
|
66
|
-
"gitHead": "
|
|
66
|
+
"gitHead": "9462257a02015e7ef40ebc16d8fdd0665c9554fe"
|
|
67
67
|
}
|