@crawlee/utils 4.0.0-beta.249 → 4.0.0-beta.250

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -34,7 +34,15 @@ export async function htmlToText(htmlOrCheerioElement) {
34
34
  if (!htmlOrCheerioElement)
35
35
  return '';
36
36
  const $ = typeof htmlOrCheerioElement === 'function' ? htmlOrCheerioElement : load(htmlOrCheerioElement);
37
- let text = '';
37
+ // Never read the accumulated text mid-loop: each read flattens V8's cons-string, making the traversal quadratic
38
+ const parts = [];
39
+ let last = '';
40
+ const push = (part) => {
41
+ if (!part)
42
+ return;
43
+ parts.push(part);
44
+ last = part[part.length - 1];
45
+ };
38
46
  const process = (elems) => {
39
47
  const len = elems ? elems.length : 0;
40
48
  for (let i = 0; i < len; i++) {
@@ -49,35 +57,35 @@ export async function htmlToText(htmlOrCheerioElement) {
49
57
  else
50
58
  compr = elem.data.replace(/\s+/g, ' ');
51
59
  // If text is empty or ends with a whitespace, don't add the leading whitespace
52
- if (compr.startsWith(' ') && /(^|\s)$/.test(text))
60
+ if (compr.startsWith(' ') && (last === '' || /\s/.test(last)))
53
61
  compr = compr.substring(1);
54
- text += compr;
62
+ push(compr);
55
63
  }
56
64
  else if (elem.type === 'comment' || SKIP_TAGS_REGEX.test(elem.tagName)) {
57
65
  // Skip comments and special elements
58
66
  }
59
67
  else if (elem.tagName === 'br') {
60
- text += '\n';
68
+ push('\n');
61
69
  }
62
70
  else if (elem.tagName === 'td') {
63
71
  process(elem.children);
64
- text += '\t';
72
+ push('\t');
65
73
  }
66
74
  else {
67
75
  // Block elements must be surrounded by newlines (unless beginning of text)
68
76
  const isBlockTag = BLOCK_TAGS_REGEX.test(elem.tagName);
69
- if (isBlockTag && !/(^|\n)$/.test(text))
70
- text += '\n';
77
+ if (isBlockTag && last !== '' && last !== '\n')
78
+ push('\n');
71
79
  process(elem.children);
72
- if (isBlockTag && !text.endsWith('\n'))
73
- text += '\n';
80
+ if (isBlockTag && last !== '\n')
81
+ push('\n');
74
82
  }
75
83
  }
76
84
  };
77
85
  // If HTML document has body, only convert that, otherwise convert the entire HTML
78
86
  const $body = $('body');
79
87
  process($body.length > 0 ? $body : $.root());
80
- return text.trim();
88
+ return parts.join('').trim();
81
89
  }
82
90
  /**
83
91
  * Extracts URLs from a given Cheerio object.
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@crawlee/utils",
3
- "version": "4.0.0-beta.249",
3
+ "version": "4.0.0-beta.250",
4
4
  "description": "A set of shared utilities that can be used by crawlers",
5
5
  "engines": {
6
6
  "node": ">=22.13.0"
@@ -43,8 +43,8 @@
43
43
  },
44
44
  "dependencies": {
45
45
  "@apify/validations": "^1.1.2",
46
- "@crawlee/http-client": "4.0.0-beta.249",
47
- "@crawlee/types": "4.0.0-beta.249",
46
+ "@crawlee/http-client": "4.0.0-beta.250",
47
+ "@crawlee/types": "4.0.0-beta.250",
48
48
  "@types/sax": "^1.2.7",
49
49
  "cheerio": "^1.2.0",
50
50
  "domhandler": "^6.0.1",
@@ -63,5 +63,5 @@
63
63
  }
64
64
  }
65
65
  },
66
- "gitHead": "968ff26a9271601c30f66b82ffd8b9833e44b8ba"
66
+ "gitHead": "894ea551fedd04fbf2394731fb85010468ccc31b"
67
67
  }