@fffangchenghao/dsh-web-search 0.11.0 → 0.12.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +28 -0
- package/README.md +132 -34
- package/README.zh-CN.md +132 -34
- package/client.js +28 -3
- package/lib/cache-persist.d.ts +96 -0
- package/lib/cache-persist.js +249 -0
- package/lib/cache.d.ts +48 -4
- package/lib/cache.js +134 -9
- package/lib/config.d.ts +16 -0
- package/lib/config.js +28 -0
- package/lib/fetch.d.ts +7 -0
- package/lib/fetch.js +36 -3
- package/lib/index.d.ts +20 -0
- package/lib/index.js +91 -27
- package/lib/markdown.d.ts +84 -0
- package/lib/markdown.js +847 -0
- package/lib/search-many.d.ts +117 -0
- package/lib/search-many.js +286 -0
- package/lib/types.d.ts +7 -4
- package/package.json +1 -1
package/lib/markdown.js
ADDED
|
@@ -0,0 +1,847 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Dependency-free HTML → readable Markdown for `web_fetch`.
|
|
3
|
+
*
|
|
4
|
+
* A fetched page is mostly furniture: site chrome, a table of contents, ad
|
|
5
|
+
* slots, scripts. Handing that markup to a model wastes its budget on
|
|
6
|
+
* navigation and makes it parse tags instead of reading prose. This module
|
|
7
|
+
* keeps the parts a reader would keep and renders them as Markdown:
|
|
8
|
+
*
|
|
9
|
+
* 1. a tolerant tokenizer and tree builder, because real pages are not XML;
|
|
10
|
+
* 2. a pruning pass that drops chrome tags, comments and hidden nodes;
|
|
11
|
+
* 3. a body selection pass that prefers `<article>`, then `<main>`/`role=main`,
|
|
12
|
+
* then the densest text container;
|
|
13
|
+
* 4. a renderer for headings, paragraphs, lists, code, quotes, links, images,
|
|
14
|
+
* rules and tables, with entities decoded and URLs made absolute.
|
|
15
|
+
*
|
|
16
|
+
* It deliberately has no runtime dependency: the plugin ships as plain ESM and
|
|
17
|
+
* a parser package would be the first thing to break on a Node upgrade.
|
|
18
|
+
*
|
|
19
|
+
* @module @fangchenghao/dsh-web-search/markdown
|
|
20
|
+
*/
|
|
21
|
+
/** Default `fetchMaxLength`: about 20k characters of prose. */
|
|
22
|
+
export const DEFAULT_FETCH_MAX_LENGTH = 20_000;
|
|
23
|
+
/** Named entities that appear in real prose; unknown names are left alone. */
|
|
24
|
+
const ENTITIES = {
|
|
25
|
+
amp: '&',
|
|
26
|
+
lt: '<',
|
|
27
|
+
gt: '>',
|
|
28
|
+
quot: '"',
|
|
29
|
+
apos: "'",
|
|
30
|
+
nbsp: ' ',
|
|
31
|
+
ensp: ' ',
|
|
32
|
+
emsp: ' ',
|
|
33
|
+
thinsp: ' ',
|
|
34
|
+
hairsp: ' ',
|
|
35
|
+
ndash: '–',
|
|
36
|
+
mdash: '—',
|
|
37
|
+
hellip: '…',
|
|
38
|
+
lsquo: '\u2018',
|
|
39
|
+
rsquo: '\u2019',
|
|
40
|
+
ldquo: '\u201c',
|
|
41
|
+
rdquo: '\u201d',
|
|
42
|
+
laquo: '«',
|
|
43
|
+
raquo: '»',
|
|
44
|
+
bull: '•',
|
|
45
|
+
middot: '·',
|
|
46
|
+
copy: '©',
|
|
47
|
+
reg: '®',
|
|
48
|
+
trade: '™',
|
|
49
|
+
deg: '°',
|
|
50
|
+
times: '×',
|
|
51
|
+
divide: '÷',
|
|
52
|
+
plusmn: '±',
|
|
53
|
+
frac12: '½',
|
|
54
|
+
sup2: '²',
|
|
55
|
+
sup3: '³',
|
|
56
|
+
euro: '€',
|
|
57
|
+
pound: '£',
|
|
58
|
+
yen: '¥',
|
|
59
|
+
cent: '¢',
|
|
60
|
+
sect: '§',
|
|
61
|
+
para: '¶',
|
|
62
|
+
dagger: '†',
|
|
63
|
+
prime: '′',
|
|
64
|
+
permil: '‰',
|
|
65
|
+
micro: 'µ',
|
|
66
|
+
larr: '←',
|
|
67
|
+
rarr: '→',
|
|
68
|
+
uarr: '↑',
|
|
69
|
+
darr: '↓',
|
|
70
|
+
harr: '↔',
|
|
71
|
+
le: '≤',
|
|
72
|
+
ge: '≥',
|
|
73
|
+
ne: '≠',
|
|
74
|
+
asymp: '≈',
|
|
75
|
+
infin: '∞',
|
|
76
|
+
sum: '∑',
|
|
77
|
+
prod: '∏',
|
|
78
|
+
radic: '√',
|
|
79
|
+
minus: '−',
|
|
80
|
+
};
|
|
81
|
+
/**
|
|
82
|
+
* Decode named and numeric HTML entities.
|
|
83
|
+
*
|
|
84
|
+
* Unknown named entities are preserved verbatim: dropping them would silently
|
|
85
|
+
* delete text, and guessing a codepoint would corrupt it.
|
|
86
|
+
*
|
|
87
|
+
* @param value - text or an attribute value.
|
|
88
|
+
* @returns the decoded text.
|
|
89
|
+
*/
|
|
90
|
+
export function decodeEntities(value) {
|
|
91
|
+
if (typeof value !== 'string' || value.indexOf('&') === -1)
|
|
92
|
+
return value;
|
|
93
|
+
return value.replace(/&(#x[0-9a-fA-F]+|#X[0-9a-fA-F]+|#[0-9]+|[a-zA-Z][a-zA-Z0-9]+);/g, (whole, body) => {
|
|
94
|
+
if (body.charAt(0) === '#') {
|
|
95
|
+
const hexadecimal = body.charAt(1) === 'x' || body.charAt(1) === 'X';
|
|
96
|
+
const code = Number.parseInt(body.slice(hexadecimal ? 2 : 1), hexadecimal ? 16 : 10);
|
|
97
|
+
if (!Number.isFinite(code) || code <= 0 || code > 0x10ffff)
|
|
98
|
+
return whole;
|
|
99
|
+
return String.fromCodePoint(code);
|
|
100
|
+
}
|
|
101
|
+
return ENTITIES[body.toLowerCase()] ?? whole;
|
|
102
|
+
});
|
|
103
|
+
}
|
|
104
|
+
/** Tags whose entire subtree is never page content. */
|
|
105
|
+
const DROP_TAGS = new Set([
|
|
106
|
+
'script', 'style', 'nav', 'header', 'footer', 'aside', 'form', 'iframe',
|
|
107
|
+
'noscript', 'template', 'svg', 'canvas', 'object', 'embed', 'button',
|
|
108
|
+
'select', 'option', 'optgroup', 'input', 'textarea', 'head', 'link',
|
|
109
|
+
'meta', 'base', 'title',
|
|
110
|
+
]);
|
|
111
|
+
/** Structural roots whose class list may mention chrome without being chrome. */
|
|
112
|
+
const NOISE_EXEMPT_TAGS = new Set(['#root', 'html', 'body', 'main', 'article']);
|
|
113
|
+
/** ARIA landmark roles that mark chrome rather than content. */
|
|
114
|
+
const DROP_ROLES = new Set([
|
|
115
|
+
'navigation', 'banner', 'contentinfo', 'complementary', 'search', 'menu',
|
|
116
|
+
'menubar', 'toolbar', 'dialog', 'alertdialog',
|
|
117
|
+
]);
|
|
118
|
+
/** Class/id fragments that mark site furniture on pages without landmarks. */
|
|
119
|
+
const NOISE_PATTERN = /(?:^|[\s_-])(?:nav|navbar|navigation|menu|sidebar|side-bar|breadcrumb|footer|header|advert|advertisement|ads?|cookie|consent|banner|social|share|comment|comments|related|recommend|popup|modal|editsection)(?:$|[\s_-])/i;
|
|
120
|
+
/** Elements that render inline; everything else is treated as a block. */
|
|
121
|
+
const INLINE_TAGS = new Set([
|
|
122
|
+
'a', 'abbr', 'b', 'bdi', 'bdo', 'br', 'cite', 'code', 'data', 'del', 'dfn',
|
|
123
|
+
'em', 'i', 'img', 'ins', 'kbd', 'label', 'mark', 'q', 's', 'samp', 'small',
|
|
124
|
+
'span', 'strong', 'sub', 'sup', 'time', 'u', 'var', 'wbr', 'strike',
|
|
125
|
+
]);
|
|
126
|
+
/** Containers the density fallback may pick from. */
|
|
127
|
+
const CONTAINER_TAGS = new Set(['body', 'div', 'section', 'article', 'main', 'td', 'center']);
|
|
128
|
+
/** HTML void elements, which never take a closing tag. */
|
|
129
|
+
const VOID_TAGS = new Set([
|
|
130
|
+
'area', 'base', 'br', 'col', 'embed', 'hr', 'img', 'input', 'link', 'meta',
|
|
131
|
+
'param', 'source', 'track', 'wbr',
|
|
132
|
+
]);
|
|
133
|
+
/** Elements whose contents are raw text, not markup. */
|
|
134
|
+
const RAW_TEXT_TAGS = new Set(['script', 'style', 'textarea', 'title', 'noscript', 'template']);
|
|
135
|
+
/** Tags that implicitly close an open paragraph. */
|
|
136
|
+
const BLOCK_TAGS = new Set([
|
|
137
|
+
'address', 'article', 'aside', 'blockquote', 'details', 'div', 'dl', 'dt',
|
|
138
|
+
'dd', 'fieldset', 'figcaption', 'figure', 'footer', 'form', 'h1', 'h2',
|
|
139
|
+
'h3', 'h4', 'h5', 'h6', 'header', 'hr', 'li', 'main', 'nav', 'ol', 'p',
|
|
140
|
+
'pre', 'section', 'table', 'ul',
|
|
141
|
+
]);
|
|
142
|
+
/** Build an element node. */
|
|
143
|
+
function element(tag, attrs) {
|
|
144
|
+
return { kind: 'element', tag, attrs, children: [] };
|
|
145
|
+
}
|
|
146
|
+
/** Find the `>` that ends an opening tag, ignoring quoted attribute values. */
|
|
147
|
+
function findTagEnd(html, from) {
|
|
148
|
+
let quote = '';
|
|
149
|
+
for (let index = from; index < html.length; index += 1) {
|
|
150
|
+
const char = html.charAt(index);
|
|
151
|
+
if (quote !== '') {
|
|
152
|
+
if (char === quote)
|
|
153
|
+
quote = '';
|
|
154
|
+
}
|
|
155
|
+
else if (char === '"' || char === "'") {
|
|
156
|
+
quote = char;
|
|
157
|
+
}
|
|
158
|
+
else if (char === '>') {
|
|
159
|
+
return index;
|
|
160
|
+
}
|
|
161
|
+
}
|
|
162
|
+
return -1;
|
|
163
|
+
}
|
|
164
|
+
/** Parse attributes, dropping inline event handlers. */
|
|
165
|
+
function parseAttributes(raw) {
|
|
166
|
+
const attrs = {};
|
|
167
|
+
const pattern = /([^\s=/>]+)(?:\s*=\s*(?:"([^"]*)"|'([^']*)'|([^\s"'=<>`]+)))?/g;
|
|
168
|
+
let match = pattern.exec(raw);
|
|
169
|
+
while (match !== null) {
|
|
170
|
+
const name = (match[1] ?? '').toLowerCase();
|
|
171
|
+
if (name.length > 0 && !name.startsWith('on')) {
|
|
172
|
+
attrs[name] = decodeEntities(match[2] ?? match[3] ?? match[4] ?? '');
|
|
173
|
+
}
|
|
174
|
+
match = pattern.exec(raw);
|
|
175
|
+
}
|
|
176
|
+
return attrs;
|
|
177
|
+
}
|
|
178
|
+
/**
|
|
179
|
+
* Parse a document into a tolerant tree.
|
|
180
|
+
*
|
|
181
|
+
* The builder is deliberately forgiving: unknown tags become containers, stray
|
|
182
|
+
* closing tags are ignored, and implicit closes for `p`, `li`, table parts and
|
|
183
|
+
* `option` keep real-world markup in the shape a browser would produce.
|
|
184
|
+
*
|
|
185
|
+
* @param html - the document or fragment.
|
|
186
|
+
* @returns the synthetic `#root` element.
|
|
187
|
+
*/
|
|
188
|
+
function parseHtml(html) {
|
|
189
|
+
const root = element('#root', {});
|
|
190
|
+
const stack = [root];
|
|
191
|
+
const pushText = (value) => {
|
|
192
|
+
if (value.length === 0)
|
|
193
|
+
return;
|
|
194
|
+
const top = stack[stack.length - 1];
|
|
195
|
+
if (top === undefined)
|
|
196
|
+
return;
|
|
197
|
+
const previous = top.children[top.children.length - 1];
|
|
198
|
+
if (previous !== undefined && previous.kind === 'text')
|
|
199
|
+
previous.value += value;
|
|
200
|
+
else
|
|
201
|
+
top.children.push({ kind: 'text', value });
|
|
202
|
+
};
|
|
203
|
+
const closeNearest = (names, boundaries) => {
|
|
204
|
+
for (let index = stack.length - 1; index >= 1; index -= 1) {
|
|
205
|
+
const open = stack[index];
|
|
206
|
+
if (open === undefined)
|
|
207
|
+
continue;
|
|
208
|
+
if (names.includes(open.tag)) {
|
|
209
|
+
stack.length = index;
|
|
210
|
+
return;
|
|
211
|
+
}
|
|
212
|
+
if (boundaries.includes(open.tag))
|
|
213
|
+
return;
|
|
214
|
+
}
|
|
215
|
+
};
|
|
216
|
+
const closeTag = (name) => {
|
|
217
|
+
for (let index = stack.length - 1; index >= 1; index -= 1) {
|
|
218
|
+
const open = stack[index];
|
|
219
|
+
if (open === undefined)
|
|
220
|
+
continue;
|
|
221
|
+
if (open.tag === name) {
|
|
222
|
+
stack.length = index;
|
|
223
|
+
return;
|
|
224
|
+
}
|
|
225
|
+
if (open.tag === 'body' || open.tag === 'html')
|
|
226
|
+
return;
|
|
227
|
+
}
|
|
228
|
+
};
|
|
229
|
+
const openTag = (name, attrs, selfClosing) => {
|
|
230
|
+
const top = stack[stack.length - 1];
|
|
231
|
+
if (top === undefined)
|
|
232
|
+
return;
|
|
233
|
+
if (top.tag === 'p' && BLOCK_TAGS.has(name))
|
|
234
|
+
stack.pop();
|
|
235
|
+
if (name === 'li')
|
|
236
|
+
closeNearest(['li'], ['ul', 'ol']);
|
|
237
|
+
else if (name === 'dt' || name === 'dd')
|
|
238
|
+
closeNearest(['dt', 'dd'], ['dl']);
|
|
239
|
+
else if (name === 'tr')
|
|
240
|
+
closeNearest(['tr'], ['table', 'tbody', 'thead', 'tfoot']);
|
|
241
|
+
else if (name === 'td' || name === 'th')
|
|
242
|
+
closeNearest(['td', 'th'], ['tr', 'table']);
|
|
243
|
+
else if (name === 'thead' || name === 'tbody' || name === 'tfoot')
|
|
244
|
+
closeNearest(['thead', 'tbody', 'tfoot'], ['table']);
|
|
245
|
+
else if (name === 'option')
|
|
246
|
+
closeNearest(['option'], ['select', 'datalist', 'optgroup']);
|
|
247
|
+
const parent = stack[stack.length - 1];
|
|
248
|
+
if (parent === undefined)
|
|
249
|
+
return;
|
|
250
|
+
const node = element(name, attrs);
|
|
251
|
+
parent.children.push(node);
|
|
252
|
+
if (!selfClosing && !VOID_TAGS.has(name))
|
|
253
|
+
stack.push(node);
|
|
254
|
+
};
|
|
255
|
+
let index = 0;
|
|
256
|
+
while (index < html.length) {
|
|
257
|
+
const lt = html.indexOf('<', index);
|
|
258
|
+
if (lt === -1) {
|
|
259
|
+
pushText(html.slice(index));
|
|
260
|
+
break;
|
|
261
|
+
}
|
|
262
|
+
if (lt > index)
|
|
263
|
+
pushText(html.slice(index, lt));
|
|
264
|
+
if (html.startsWith('<!--', lt)) {
|
|
265
|
+
const end = html.indexOf('-->', lt + 4);
|
|
266
|
+
index = end === -1 ? html.length : end + 3;
|
|
267
|
+
continue;
|
|
268
|
+
}
|
|
269
|
+
if (html.startsWith('<![CDATA[', lt)) {
|
|
270
|
+
const end = html.indexOf(']]>', lt + 9);
|
|
271
|
+
// CDATA is text in XML feeds, not markup to discard.
|
|
272
|
+
pushText(html.slice(lt + 9, end === -1 ? html.length : end));
|
|
273
|
+
index = end === -1 ? html.length : end + 3;
|
|
274
|
+
continue;
|
|
275
|
+
}
|
|
276
|
+
if (html.startsWith('<!', lt) || html.startsWith('<?', lt)) {
|
|
277
|
+
const end = html.indexOf('>', lt + 2);
|
|
278
|
+
index = end === -1 ? html.length : end + 1;
|
|
279
|
+
continue;
|
|
280
|
+
}
|
|
281
|
+
if (html.startsWith('</', lt)) {
|
|
282
|
+
const end = html.indexOf('>', lt + 2);
|
|
283
|
+
const name = html
|
|
284
|
+
.slice(lt + 2, end === -1 ? html.length : end)
|
|
285
|
+
.trim()
|
|
286
|
+
.toLowerCase();
|
|
287
|
+
if (name.length > 0)
|
|
288
|
+
closeTag(name);
|
|
289
|
+
index = end === -1 ? html.length : end + 1;
|
|
290
|
+
continue;
|
|
291
|
+
}
|
|
292
|
+
const end = findTagEnd(html, lt + 1);
|
|
293
|
+
if (end === -1) {
|
|
294
|
+
pushText(html.slice(lt));
|
|
295
|
+
break;
|
|
296
|
+
}
|
|
297
|
+
const raw = html.slice(lt + 1, end);
|
|
298
|
+
const selfClosing = raw.endsWith('/');
|
|
299
|
+
const body = selfClosing ? raw.slice(0, -1) : raw;
|
|
300
|
+
const match = /^[a-zA-Z][a-zA-Z0-9:_-]*/.exec(body);
|
|
301
|
+
if (match === null) {
|
|
302
|
+
pushText(html.slice(lt, end + 1));
|
|
303
|
+
index = end + 1;
|
|
304
|
+
continue;
|
|
305
|
+
}
|
|
306
|
+
const name = (match[0] ?? '').toLowerCase();
|
|
307
|
+
index = end + 1;
|
|
308
|
+
if (RAW_TEXT_TAGS.has(name)) {
|
|
309
|
+
const closeAt = html.toLowerCase().indexOf(`</${name}`, index);
|
|
310
|
+
if (closeAt === -1) {
|
|
311
|
+
index = html.length;
|
|
312
|
+
continue;
|
|
313
|
+
}
|
|
314
|
+
const closeEnd = html.indexOf('>', closeAt);
|
|
315
|
+
index = closeEnd === -1 ? html.length : closeEnd + 1;
|
|
316
|
+
continue;
|
|
317
|
+
}
|
|
318
|
+
openTag(name, parseAttributes(body.slice(match[0].length)), selfClosing);
|
|
319
|
+
}
|
|
320
|
+
return root;
|
|
321
|
+
}
|
|
322
|
+
/** Decode all visible text below a node, without Markdown syntax. */
|
|
323
|
+
function textOf(node) {
|
|
324
|
+
if (node.kind === 'text')
|
|
325
|
+
return decodeEntities(node.value);
|
|
326
|
+
if (node.tag === 'br')
|
|
327
|
+
return '\n';
|
|
328
|
+
if (node.tag === 'img')
|
|
329
|
+
return node.attrs['alt'] ?? '';
|
|
330
|
+
if (DROP_TAGS.has(node.tag))
|
|
331
|
+
return '';
|
|
332
|
+
let text = '';
|
|
333
|
+
for (const child of node.children)
|
|
334
|
+
text += textOf(child);
|
|
335
|
+
return text;
|
|
336
|
+
}
|
|
337
|
+
/** Count element descendants, used by the density fallback. */
|
|
338
|
+
function elementCount(node) {
|
|
339
|
+
let count = 0;
|
|
340
|
+
for (const child of node.children) {
|
|
341
|
+
if (child.kind !== 'element')
|
|
342
|
+
continue;
|
|
343
|
+
count += 1 + elementCount(child);
|
|
344
|
+
}
|
|
345
|
+
return count;
|
|
346
|
+
}
|
|
347
|
+
/** Visit every descendant element, children first order. */
|
|
348
|
+
function walkElements(node, visit) {
|
|
349
|
+
for (const child of node.children) {
|
|
350
|
+
if (child.kind !== 'element')
|
|
351
|
+
continue;
|
|
352
|
+
visit(child);
|
|
353
|
+
walkElements(child, visit);
|
|
354
|
+
}
|
|
355
|
+
}
|
|
356
|
+
/** First descendant matching a predicate. */
|
|
357
|
+
function findElement(node, predicate) {
|
|
358
|
+
for (const child of node.children) {
|
|
359
|
+
if (child.kind !== 'element')
|
|
360
|
+
continue;
|
|
361
|
+
if (predicate(child))
|
|
362
|
+
return child;
|
|
363
|
+
const found = findElement(child, predicate);
|
|
364
|
+
if (found !== undefined)
|
|
365
|
+
return found;
|
|
366
|
+
}
|
|
367
|
+
return undefined;
|
|
368
|
+
}
|
|
369
|
+
/** Prune chrome, hidden nodes and noise class/id containers in place. */
|
|
370
|
+
function prune(node) {
|
|
371
|
+
const kept = [];
|
|
372
|
+
for (const child of node.children) {
|
|
373
|
+
if (child.kind === 'text') {
|
|
374
|
+
kept.push(child);
|
|
375
|
+
continue;
|
|
376
|
+
}
|
|
377
|
+
if (DROP_TAGS.has(child.tag) || (!NOISE_EXEMPT_TAGS.has(child.tag) && isNoise(child)))
|
|
378
|
+
continue;
|
|
379
|
+
prune(child);
|
|
380
|
+
kept.push(child);
|
|
381
|
+
}
|
|
382
|
+
node.children = kept;
|
|
383
|
+
}
|
|
384
|
+
/** Whether an element is hidden or named like site furniture. */
|
|
385
|
+
function isNoise(node) {
|
|
386
|
+
const role = (node.attrs['role'] ?? '').toLowerCase();
|
|
387
|
+
if (DROP_ROLES.has(role))
|
|
388
|
+
return true;
|
|
389
|
+
if (node.attrs['hidden'] !== undefined)
|
|
390
|
+
return true;
|
|
391
|
+
const style = node.attrs['style'];
|
|
392
|
+
if (style !== undefined && /display\s*:\s*none|visibility\s*:\s*hidden/i.test(style))
|
|
393
|
+
return true;
|
|
394
|
+
const marker = `${node.attrs['class'] ?? ''} ${node.attrs['id'] ?? ''}`;
|
|
395
|
+
return marker.trim().length > 0 && NOISE_PATTERN.test(marker);
|
|
396
|
+
}
|
|
397
|
+
/** Visible character count with whitespace collapsed. */
|
|
398
|
+
function characterCount(node) {
|
|
399
|
+
return textOf(node).replace(/\s+/g, ' ').trim().length;
|
|
400
|
+
}
|
|
401
|
+
/** Score a container by how much prose it holds per unit of markup. */
|
|
402
|
+
function density(node) {
|
|
403
|
+
const characters = characterCount(node);
|
|
404
|
+
return characters === 0 ? 0 : characters / (1 + elementCount(node));
|
|
405
|
+
}
|
|
406
|
+
/** Highest-density container, favouring the one that carries the most text. */
|
|
407
|
+
function densestContainer(body) {
|
|
408
|
+
let best = body;
|
|
409
|
+
let bestScore = density(body);
|
|
410
|
+
let bestCharacters = characterCount(body);
|
|
411
|
+
walkElements(body, (node) => {
|
|
412
|
+
if (!CONTAINER_TAGS.has(node.tag))
|
|
413
|
+
return;
|
|
414
|
+
const characters = characterCount(node);
|
|
415
|
+
if (characters < 120)
|
|
416
|
+
return;
|
|
417
|
+
const score = density(node);
|
|
418
|
+
if (score > bestScore || (score === bestScore && characters > bestCharacters)) {
|
|
419
|
+
best = node;
|
|
420
|
+
bestScore = score;
|
|
421
|
+
bestCharacters = characters;
|
|
422
|
+
}
|
|
423
|
+
});
|
|
424
|
+
return best;
|
|
425
|
+
}
|
|
426
|
+
/** Pick the reading root: article, main, then the densest container. */
|
|
427
|
+
function pickMain(root) {
|
|
428
|
+
const body = findElement(root, (node) => node.tag === 'body') ?? root;
|
|
429
|
+
let article;
|
|
430
|
+
let articleCharacters = -1;
|
|
431
|
+
walkElements(body, (node) => {
|
|
432
|
+
if (node.tag !== 'article')
|
|
433
|
+
return;
|
|
434
|
+
const characters = characterCount(node);
|
|
435
|
+
if (characters > articleCharacters) {
|
|
436
|
+
article = node;
|
|
437
|
+
articleCharacters = characters;
|
|
438
|
+
}
|
|
439
|
+
});
|
|
440
|
+
if (article !== undefined && articleCharacters > 0)
|
|
441
|
+
return article;
|
|
442
|
+
const main = findElement(body, (node) => node.tag === 'main' || (node.attrs['role'] ?? '').toLowerCase() === 'main');
|
|
443
|
+
if (main !== undefined && characterCount(main) > 0)
|
|
444
|
+
return main;
|
|
445
|
+
return densestContainer(body);
|
|
446
|
+
}
|
|
447
|
+
/** Replace a code fragment with a placeholder so clean-up cannot touch it. */
|
|
448
|
+
function stashCode(ctx, text) {
|
|
449
|
+
ctx.stash.push(text);
|
|
450
|
+
return `\uE000${ctx.stash.length - 1}\uE001`;
|
|
451
|
+
}
|
|
452
|
+
/** Put stashed code fragments back after whitespace clean-up. */
|
|
453
|
+
function restoreCode(text, ctx) {
|
|
454
|
+
let out = text;
|
|
455
|
+
for (let index = 0; index < ctx.stash.length; index += 1) {
|
|
456
|
+
out = out.split(`\uE000${index}\uE001`).join(ctx.stash[index] ?? '');
|
|
457
|
+
}
|
|
458
|
+
return out;
|
|
459
|
+
}
|
|
460
|
+
/** Collapse source whitespace inside prose, leaving block separators alone. */
|
|
461
|
+
function collapseInline(value) {
|
|
462
|
+
return value.replace(/[\t\n\r\f ]+/g, ' ');
|
|
463
|
+
}
|
|
464
|
+
/** Normalise block output: line endings, trailing spaces, blank runs. */
|
|
465
|
+
function cleanup(value) {
|
|
466
|
+
return value
|
|
467
|
+
.replace(/\r\n?/g, '\n')
|
|
468
|
+
.replace(/[ \t]+\n/g, '\n')
|
|
469
|
+
.replace(/\n{3,}/g, '\n\n')
|
|
470
|
+
.replace(/^\n+/, '')
|
|
471
|
+
.replace(/\n+$/, '');
|
|
472
|
+
}
|
|
473
|
+
/** Longest run of one character, used to pick a safe code fence. */
|
|
474
|
+
function longestRun(value, character) {
|
|
475
|
+
let longest = 0;
|
|
476
|
+
let current = 0;
|
|
477
|
+
for (const char of value) {
|
|
478
|
+
if (char === character) {
|
|
479
|
+
current += 1;
|
|
480
|
+
if (current > longest)
|
|
481
|
+
longest = current;
|
|
482
|
+
}
|
|
483
|
+
else {
|
|
484
|
+
current = 0;
|
|
485
|
+
}
|
|
486
|
+
}
|
|
487
|
+
return longest;
|
|
488
|
+
}
|
|
489
|
+
/** Make a link target absolute and reject script-bearing schemes. */
|
|
490
|
+
function absoluteUrl(raw, baseUrl) {
|
|
491
|
+
const value = raw.trim();
|
|
492
|
+
if (value.length === 0)
|
|
493
|
+
return '';
|
|
494
|
+
if (/^(?:javascript|data|vbscript):/i.test(value))
|
|
495
|
+
return '';
|
|
496
|
+
if (/^[a-zA-Z][a-zA-Z0-9+.-]*:/.test(value) && !/^(?:https?|mailto|ftp):/i.test(value))
|
|
497
|
+
return '';
|
|
498
|
+
if (baseUrl === undefined)
|
|
499
|
+
return value;
|
|
500
|
+
try {
|
|
501
|
+
return new URL(value, baseUrl).toString();
|
|
502
|
+
}
|
|
503
|
+
catch {
|
|
504
|
+
return value;
|
|
505
|
+
}
|
|
506
|
+
}
|
|
507
|
+
/** Wrap inline content in a Markdown marker, keeping surrounding spaces. */
|
|
508
|
+
function wrapInline(inner, marker, format) {
|
|
509
|
+
if (format === 'text')
|
|
510
|
+
return inner;
|
|
511
|
+
const match = /^(\s*)([\s\S]*?)(\s*)$/.exec(inner);
|
|
512
|
+
if (match === null)
|
|
513
|
+
return inner;
|
|
514
|
+
const leading = match[1] ?? '';
|
|
515
|
+
const core = match[2] ?? '';
|
|
516
|
+
const trailing = match[3] ?? '';
|
|
517
|
+
return core.length === 0 ? inner : `${leading}${marker}${core}${marker}${trailing}`;
|
|
518
|
+
}
|
|
519
|
+
/** Render every child as inline content, separating adjacent blocks. */
|
|
520
|
+
function inlineChildren(node, ctx) {
|
|
521
|
+
let out = '';
|
|
522
|
+
for (const child of node.children) {
|
|
523
|
+
const part = inlineNode(child, ctx);
|
|
524
|
+
if (part.length === 0)
|
|
525
|
+
continue;
|
|
526
|
+
const block = child.kind === 'element' && child.tag !== 'br' && !INLINE_TAGS.has(child.tag);
|
|
527
|
+
if (block && out.length > 0 && !/\s$/.test(out) && !/^\s/.test(part))
|
|
528
|
+
out += ' ';
|
|
529
|
+
out += part;
|
|
530
|
+
}
|
|
531
|
+
return out;
|
|
532
|
+
}
|
|
533
|
+
/** Render one node as inline content. */
|
|
534
|
+
function inlineNode(node, ctx) {
|
|
535
|
+
if (node.kind === 'text')
|
|
536
|
+
return collapseInline(decodeEntities(node.value));
|
|
537
|
+
if (DROP_TAGS.has(node.tag))
|
|
538
|
+
return '';
|
|
539
|
+
return inlineElement(node, ctx);
|
|
540
|
+
}
|
|
541
|
+
/** Render one element as inline content. */
|
|
542
|
+
function inlineElement(node, ctx) {
|
|
543
|
+
switch (node.tag) {
|
|
544
|
+
case 'br':
|
|
545
|
+
return '\n';
|
|
546
|
+
case 'img': {
|
|
547
|
+
const source = absoluteUrl(node.attrs['src'] ?? node.attrs['data-src'] ?? '', ctx.baseUrl);
|
|
548
|
+
const alt = node.attrs['alt'] ?? '';
|
|
549
|
+
if (source.length === 0 || ctx.format === 'text')
|
|
550
|
+
return alt;
|
|
551
|
+
return `![${alt.replace(/]/g, '\\]')}](${source})`;
|
|
552
|
+
}
|
|
553
|
+
case 'a': {
|
|
554
|
+
const href = absoluteUrl(node.attrs['href'] ?? '', ctx.baseUrl);
|
|
555
|
+
const inner = inlineChildren(node, ctx);
|
|
556
|
+
if (ctx.format === 'text')
|
|
557
|
+
return href.length === 0 ? inner : `${inner} (${href})`;
|
|
558
|
+
if (href.length === 0)
|
|
559
|
+
return inner;
|
|
560
|
+
return `[${inner.trim().length === 0 ? href : inner}](${href})`;
|
|
561
|
+
}
|
|
562
|
+
case 'code': {
|
|
563
|
+
const text = textOf(node).replace(/\s+/g, ' ').trim();
|
|
564
|
+
if (text.length === 0)
|
|
565
|
+
return '';
|
|
566
|
+
if (ctx.format === 'text')
|
|
567
|
+
return stashCode(ctx, text);
|
|
568
|
+
const fence = '`'.repeat(longestRun(text, '`') + 1);
|
|
569
|
+
const pad = /^\s|^`|`$|\s$/.test(text) ? ' ' : '';
|
|
570
|
+
return stashCode(ctx, `${fence}${pad}${text}${pad}${fence}`);
|
|
571
|
+
}
|
|
572
|
+
case 'strong':
|
|
573
|
+
case 'b':
|
|
574
|
+
return wrapInline(inlineChildren(node, ctx), '**', ctx.format);
|
|
575
|
+
case 'em':
|
|
576
|
+
case 'i':
|
|
577
|
+
return wrapInline(inlineChildren(node, ctx), '*', ctx.format);
|
|
578
|
+
case 'del':
|
|
579
|
+
case 's':
|
|
580
|
+
case 'strike':
|
|
581
|
+
return wrapInline(inlineChildren(node, ctx), '~~', ctx.format);
|
|
582
|
+
case 'q':
|
|
583
|
+
return ctx.format === 'markdown' ? `"${inlineChildren(node, ctx)}"` : inlineChildren(node, ctx);
|
|
584
|
+
default:
|
|
585
|
+
return inlineChildren(node, ctx);
|
|
586
|
+
}
|
|
587
|
+
}
|
|
588
|
+
/** Render every child as block content. */
|
|
589
|
+
function renderChildren(node, ctx) {
|
|
590
|
+
let out = '';
|
|
591
|
+
for (const child of node.children)
|
|
592
|
+
out += renderNode(child, ctx);
|
|
593
|
+
return out;
|
|
594
|
+
}
|
|
595
|
+
/** Render one node as block content. */
|
|
596
|
+
function renderNode(node, ctx) {
|
|
597
|
+
if (node.kind === 'text')
|
|
598
|
+
return collapseInline(decodeEntities(node.value));
|
|
599
|
+
if (DROP_TAGS.has(node.tag))
|
|
600
|
+
return '';
|
|
601
|
+
return renderElement(node, ctx);
|
|
602
|
+
}
|
|
603
|
+
/** Render one element as block content. */
|
|
604
|
+
function renderElement(node, ctx) {
|
|
605
|
+
if (/^h[1-6]$/.test(node.tag)) {
|
|
606
|
+
const level = Number.parseInt(node.tag.slice(1), 10);
|
|
607
|
+
const text = inlineChildren(node, ctx).trim();
|
|
608
|
+
if (text.length === 0)
|
|
609
|
+
return '';
|
|
610
|
+
const prefix = ctx.format === 'markdown' ? `${'#'.repeat(level)} ` : '';
|
|
611
|
+
return `\n\n${prefix}${text}\n\n`;
|
|
612
|
+
}
|
|
613
|
+
switch (node.tag) {
|
|
614
|
+
case 'p':
|
|
615
|
+
return `\n\n${inlineChildren(node, ctx).trim()}\n\n`;
|
|
616
|
+
case 'pre':
|
|
617
|
+
return `\n\n${renderPre(node, ctx)}\n\n`;
|
|
618
|
+
case 'ul':
|
|
619
|
+
case 'ol':
|
|
620
|
+
return `\n\n${renderList(node, ctx)}\n\n`;
|
|
621
|
+
case 'blockquote': {
|
|
622
|
+
const inner = cleanup(renderChildren(node, ctx));
|
|
623
|
+
if (inner.length === 0)
|
|
624
|
+
return '';
|
|
625
|
+
if (ctx.format === 'text')
|
|
626
|
+
return `\n\n${inner.replace(/^/gm, ' ')}\n\n`;
|
|
627
|
+
const quoted = inner
|
|
628
|
+
.split('\n')
|
|
629
|
+
.map((line) => (line.length === 0 ? '>' : `> ${line}`))
|
|
630
|
+
.join('\n');
|
|
631
|
+
return `\n\n${quoted}\n\n`;
|
|
632
|
+
}
|
|
633
|
+
case 'table':
|
|
634
|
+
return `\n\n${renderTable(node, ctx)}\n\n`;
|
|
635
|
+
case 'hr':
|
|
636
|
+
return ctx.format === 'markdown' ? '\n\n---\n\n' : '\n\n';
|
|
637
|
+
case 'br':
|
|
638
|
+
return '\n';
|
|
639
|
+
case 'img':
|
|
640
|
+
case 'a':
|
|
641
|
+
case 'code':
|
|
642
|
+
return inlineElement(node, ctx);
|
|
643
|
+
case 'dl':
|
|
644
|
+
return `\n\n${renderChildren(node, ctx)}\n\n`;
|
|
645
|
+
case 'dt': {
|
|
646
|
+
const text = inlineChildren(node, ctx).trim();
|
|
647
|
+
return text.length === 0 ? '' : `\n\n${ctx.format === 'markdown' ? `**${text}**` : text}\n\n`;
|
|
648
|
+
}
|
|
649
|
+
case 'dd': {
|
|
650
|
+
const text = inlineChildren(node, ctx).trim();
|
|
651
|
+
return text.length === 0 ? '' : `\n\n${text}\n\n`;
|
|
652
|
+
}
|
|
653
|
+
case 'summary':
|
|
654
|
+
return `\n\n${inlineChildren(node, ctx).trim()}\n\n`;
|
|
655
|
+
default:
|
|
656
|
+
if (INLINE_TAGS.has(node.tag))
|
|
657
|
+
return inlineElement(node, ctx);
|
|
658
|
+
return `\n\n${renderChildren(node, ctx)}\n\n`;
|
|
659
|
+
}
|
|
660
|
+
}
|
|
661
|
+
/** Language hint from `class="language-x"`, `lang-x` or `brush: x`. */
|
|
662
|
+
function codeLanguage(node) {
|
|
663
|
+
let marker = node.attrs['class'] ?? '';
|
|
664
|
+
const code = node.children.find((child) => child.kind === 'element' && child.tag === 'code');
|
|
665
|
+
if (code !== undefined)
|
|
666
|
+
marker += ` ${code.attrs['class'] ?? ''}`;
|
|
667
|
+
const match = /(?:language-|lang-|brush:\s*)([A-Za-z0-9_+#.-]+)/i.exec(marker);
|
|
668
|
+
return match === null ? '' : (match[1] ?? '').toLowerCase();
|
|
669
|
+
}
|
|
670
|
+
/** Render a `<pre>` block, fence included for Markdown output. */
|
|
671
|
+
function renderPre(node, ctx) {
|
|
672
|
+
const content = textOf(node)
|
|
673
|
+
.replace(/\r\n?/g, '\n')
|
|
674
|
+
.replace(/^\n+/, '')
|
|
675
|
+
.replace(/[ \t]+$/gm, '')
|
|
676
|
+
.replace(/\s+$/, '');
|
|
677
|
+
if (content.length === 0)
|
|
678
|
+
return '';
|
|
679
|
+
if (ctx.format === 'text')
|
|
680
|
+
return stashCode(ctx, content);
|
|
681
|
+
const fence = '`'.repeat(Math.max(3, longestRun(content, '`') + 1));
|
|
682
|
+
const language = codeLanguage(node);
|
|
683
|
+
return stashCode(ctx, `${fence}${language}\n${content}\n${fence}`);
|
|
684
|
+
}
|
|
685
|
+
/** Render one list item, including any nested lists. */
|
|
686
|
+
function renderListItem(item, ctx, marker) {
|
|
687
|
+
const nested = [];
|
|
688
|
+
const parts = [];
|
|
689
|
+
for (const child of item.children) {
|
|
690
|
+
if (child.kind === 'element' && (child.tag === 'ul' || child.tag === 'ol')) {
|
|
691
|
+
nested.push(child);
|
|
692
|
+
continue;
|
|
693
|
+
}
|
|
694
|
+
parts.push(renderNode(child, ctx));
|
|
695
|
+
}
|
|
696
|
+
const body = cleanup(parts.join('')).replace(/\n{2,}/g, '\n');
|
|
697
|
+
const indent = ' '.repeat(marker.length);
|
|
698
|
+
let out = '';
|
|
699
|
+
const lines = body.length === 0 ? [] : body.split('\n');
|
|
700
|
+
const first = lines[0] ?? '';
|
|
701
|
+
out = `${marker}${first}`.trimEnd();
|
|
702
|
+
for (let index = 1; index < lines.length; index += 1) {
|
|
703
|
+
const line = lines[index] ?? '';
|
|
704
|
+
out += line.length === 0 ? '\n' : `\n${indent}${line}`;
|
|
705
|
+
}
|
|
706
|
+
for (const list of nested) {
|
|
707
|
+
const rendered = cleanup(renderList(list, ctx));
|
|
708
|
+
for (const line of rendered.split('\n'))
|
|
709
|
+
out += `\n${indent}${line}`;
|
|
710
|
+
}
|
|
711
|
+
return out;
|
|
712
|
+
}
|
|
713
|
+
/** Render an ordered or unordered list. */
|
|
714
|
+
function renderList(node, ctx) {
|
|
715
|
+
const ordered = node.tag === 'ol';
|
|
716
|
+
const parsedStart = Number.parseInt(node.attrs['start'] ?? '1', 10);
|
|
717
|
+
const start = Number.isFinite(parsedStart) && parsedStart > 0 ? parsedStart : 1;
|
|
718
|
+
const items = node.children.filter((child) => child.kind === 'element' && child.tag === 'li');
|
|
719
|
+
const lines = [];
|
|
720
|
+
for (let index = 0; index < items.length; index += 1) {
|
|
721
|
+
const item = items[index];
|
|
722
|
+
if (item === undefined)
|
|
723
|
+
continue;
|
|
724
|
+
const marker = ordered ? `${start + index}. ` : ctx.format === 'markdown' ? '- ' : '• ';
|
|
725
|
+
lines.push(renderListItem(item, ctx, marker));
|
|
726
|
+
}
|
|
727
|
+
return lines.join('\n');
|
|
728
|
+
}
|
|
729
|
+
/** Collect table rows without descending into a nested table. */
|
|
730
|
+
function collectRows(node, rows) {
|
|
731
|
+
for (const child of node.children) {
|
|
732
|
+
if (child.kind !== 'element')
|
|
733
|
+
continue;
|
|
734
|
+
if (child.tag === 'tr') {
|
|
735
|
+
rows.push(child);
|
|
736
|
+
}
|
|
737
|
+
else if (child.tag === 'thead' || child.tag === 'tbody' || child.tag === 'tfoot') {
|
|
738
|
+
collectRows(child, rows);
|
|
739
|
+
}
|
|
740
|
+
}
|
|
741
|
+
}
|
|
742
|
+
/** Render a table as a Markdown table with a header separator row. */
|
|
743
|
+
function renderTable(node, ctx) {
|
|
744
|
+
const rows = [];
|
|
745
|
+
collectRows(node, rows);
|
|
746
|
+
const matrix = [];
|
|
747
|
+
for (const row of rows) {
|
|
748
|
+
const cells = [];
|
|
749
|
+
for (const child of row.children) {
|
|
750
|
+
if (child.kind !== 'element' || (child.tag !== 'td' && child.tag !== 'th'))
|
|
751
|
+
continue;
|
|
752
|
+
const cell = inlineChildren(child, ctx).replace(/\s+/g, ' ').trim().replace(/\|/g, '\\|');
|
|
753
|
+
cells.push(cell);
|
|
754
|
+
}
|
|
755
|
+
if (cells.length > 0)
|
|
756
|
+
matrix.push(cells);
|
|
757
|
+
}
|
|
758
|
+
if (matrix.length === 0)
|
|
759
|
+
return '';
|
|
760
|
+
let columns = 0;
|
|
761
|
+
for (const row of matrix)
|
|
762
|
+
columns = Math.max(columns, row.length);
|
|
763
|
+
const padded = matrix.map((row) => {
|
|
764
|
+
const copy = [...row];
|
|
765
|
+
while (copy.length < columns)
|
|
766
|
+
copy.push('');
|
|
767
|
+
return copy;
|
|
768
|
+
});
|
|
769
|
+
if (ctx.format === 'text')
|
|
770
|
+
return padded.map((row) => row.join(' | ')).join('\n');
|
|
771
|
+
const header = padded[0] ?? [];
|
|
772
|
+
const lines = [
|
|
773
|
+
`| ${header.join(' | ')} |`,
|
|
774
|
+
`| ${header.map(() => '---').join(' | ')} |`,
|
|
775
|
+
];
|
|
776
|
+
for (const row of padded.slice(1))
|
|
777
|
+
lines.push(`| ${row.join(' | ')} |`);
|
|
778
|
+
return lines.join('\n');
|
|
779
|
+
}
|
|
780
|
+
/**
|
|
781
|
+
* Convert an HTML document or fragment to readable Markdown or plain text.
|
|
782
|
+
*
|
|
783
|
+
* @param html - the raw markup.
|
|
784
|
+
* @param options - output format and the base URL for relative links.
|
|
785
|
+
* @returns the converted text; an empty or contentless document yields `''`.
|
|
786
|
+
*/
|
|
787
|
+
export function htmlToMarkdown(html, options = {}) {
|
|
788
|
+
if (typeof html !== 'string')
|
|
789
|
+
return '';
|
|
790
|
+
const trimmed = html.trim();
|
|
791
|
+
if (trimmed.length === 0)
|
|
792
|
+
return '';
|
|
793
|
+
const root = parseHtml(trimmed);
|
|
794
|
+
prune(root);
|
|
795
|
+
const body = findElement(root, (node) => node.tag === 'body') ?? root;
|
|
796
|
+
if (characterCount(body) === 0)
|
|
797
|
+
return '';
|
|
798
|
+
const main = pickMain(root);
|
|
799
|
+
const ctx = {
|
|
800
|
+
format: options.format ?? 'markdown',
|
|
801
|
+
baseUrl: options.baseUrl,
|
|
802
|
+
stash: [],
|
|
803
|
+
};
|
|
804
|
+
return restoreCode(cleanup(renderElement(main, ctx)), ctx).trim();
|
|
805
|
+
}
|
|
806
|
+
/**
|
|
807
|
+
* Apply the output character budget.
|
|
808
|
+
*
|
|
809
|
+
* The notice names both the configured limit and the original converted
|
|
810
|
+
* length, so a reader knows how much is missing and can ask for a narrower URL.
|
|
811
|
+
*
|
|
812
|
+
* @param content - converted content.
|
|
813
|
+
* @param maxLength - configured character budget.
|
|
814
|
+
* @returns the content, cut with a notice when it exceeded the budget.
|
|
815
|
+
*/
|
|
816
|
+
export function truncateFetchText(content, maxLength) {
|
|
817
|
+
const limit = Number.isFinite(maxLength) && maxLength >= 1 ? Math.floor(maxLength) : DEFAULT_FETCH_MAX_LENGTH;
|
|
818
|
+
if (content.length <= limit)
|
|
819
|
+
return { content, truncated: false };
|
|
820
|
+
let cut = content.slice(0, limit);
|
|
821
|
+
const last = cut.charCodeAt(cut.length - 1);
|
|
822
|
+
// Never leave half of a surrogate pair at the cut point.
|
|
823
|
+
if (last >= 0xd800 && last <= 0xdbff)
|
|
824
|
+
cut = cut.slice(0, -1);
|
|
825
|
+
cut = cut.replace(/\s+$/, '');
|
|
826
|
+
const notice = `\n\n[truncated at ${limit} chars; original length ${content.length} chars; the rest is not shown]`;
|
|
827
|
+
return { content: `${cut}${notice}`, truncated: true };
|
|
828
|
+
}
|
|
829
|
+
/**
|
|
830
|
+
* Convert a fetched HTML page and apply the configured budget in one step.
|
|
831
|
+
*
|
|
832
|
+
* @param html - the page markup.
|
|
833
|
+
* @param options - format, base URL and character budget.
|
|
834
|
+
* @returns converted content plus the pre-truncation length and cut flag.
|
|
835
|
+
*/
|
|
836
|
+
export function renderFetchText(html, options) {
|
|
837
|
+
const converted = htmlToMarkdown(html, {
|
|
838
|
+
format: options.format ?? 'markdown',
|
|
839
|
+
baseUrl: options.baseUrl,
|
|
840
|
+
});
|
|
841
|
+
const limited = truncateFetchText(converted, options.maxLength);
|
|
842
|
+
return {
|
|
843
|
+
content: limited.content,
|
|
844
|
+
originalLength: converted.length,
|
|
845
|
+
truncated: limited.truncated,
|
|
846
|
+
};
|
|
847
|
+
}
|