@j0hanz/fetch-url-mcp 1.4.0 → 1.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli.d.ts +2 -3
- package/dist/cli.js +1 -2
- package/dist/http/auth.d.ts +5 -3
- package/dist/http/auth.js +64 -15
- package/dist/http/health.d.ts +1 -2
- package/dist/http/health.js +7 -18
- package/dist/http/helpers.d.ts +3 -4
- package/dist/http/helpers.js +21 -21
- package/dist/http/native.d.ts +0 -1
- package/dist/http/native.js +34 -26
- package/dist/http/rate-limit.d.ts +0 -1
- package/dist/http/rate-limit.js +3 -4
- package/dist/index.d.ts +0 -1
- package/dist/index.js +17 -18
- package/dist/lib/{markdown-cleanup.d.ts → content.d.ts} +4 -2
- package/dist/lib/content.js +1356 -0
- package/dist/lib/core.d.ts +253 -0
- package/dist/lib/core.js +1228 -0
- package/dist/lib/{tool-pipeline.d.ts → fetch-pipeline.d.ts} +1 -2
- package/dist/lib/{tool-pipeline.js → fetch-pipeline.js} +10 -19
- package/dist/lib/{fetch.d.ts → http.d.ts} +7 -9
- package/dist/lib/{fetch.js → http.js} +706 -944
- package/dist/lib/mcp-tools.d.ts +28 -0
- package/dist/lib/mcp-tools.js +107 -0
- package/dist/lib/{tool-progress.d.ts → progress.d.ts} +0 -1
- package/dist/lib/{tool-progress.js → progress.js} +8 -13
- package/dist/lib/task-handlers.d.ts +5 -0
- package/dist/lib/{mcp.js → task-handlers.js} +56 -12
- package/dist/lib/url.d.ts +70 -0
- package/dist/lib/url.js +686 -0
- package/dist/lib/utils.d.ts +58 -0
- package/dist/lib/utils.js +304 -0
- package/dist/prompts/index.d.ts +0 -1
- package/dist/prompts/index.js +0 -1
- package/dist/resources/index.d.ts +0 -1
- package/dist/resources/index.js +74 -33
- package/dist/resources/instructions.d.ts +0 -1
- package/dist/resources/instructions.js +2 -2
- package/dist/schemas/inputs.d.ts +0 -1
- package/dist/schemas/inputs.js +2 -3
- package/dist/schemas/outputs.d.ts +0 -1
- package/dist/schemas/outputs.js +1 -2
- package/dist/server.d.ts +0 -1
- package/dist/server.js +16 -26
- package/dist/tasks/execution.d.ts +0 -1
- package/dist/tasks/execution.js +27 -24
- package/dist/tasks/manager.d.ts +7 -3
- package/dist/tasks/manager.js +53 -34
- package/dist/tasks/owner.d.ts +1 -2
- package/dist/tasks/owner.js +1 -2
- package/dist/tasks/tool-registry.d.ts +1 -2
- package/dist/tasks/tool-registry.js +0 -1
- package/dist/tools/fetch-url.d.ts +1 -2
- package/dist/tools/fetch-url.js +39 -31
- package/dist/tools/index.d.ts +0 -1
- package/dist/tools/index.js +0 -1
- package/dist/transform/html-translators.d.ts +1 -0
- package/dist/transform/html-translators.js +454 -0
- package/dist/transform/metadata.d.ts +4 -0
- package/dist/transform/metadata.js +183 -0
- package/dist/transform/transform.d.ts +0 -1
- package/dist/transform/transform.js +24 -641
- package/dist/transform/types.d.ts +9 -11
- package/dist/transform/types.js +0 -1
- package/dist/transform/worker-pool.d.ts +0 -1
- package/dist/transform/worker-pool.js +7 -16
- package/dist/transform/workers/shared.d.ts +0 -1
- package/dist/transform/workers/shared.js +1 -2
- package/dist/transform/workers/transform-child.d.ts +0 -1
- package/dist/transform/workers/transform-child.js +0 -1
- package/dist/transform/workers/transform-worker.d.ts +0 -1
- package/dist/transform/workers/transform-worker.js +0 -1
- package/package.json +6 -3
- package/dist/cli.d.ts.map +0 -1
- package/dist/cli.js.map +0 -1
- package/dist/http/auth.d.ts.map +0 -1
- package/dist/http/auth.js.map +0 -1
- package/dist/http/health.d.ts.map +0 -1
- package/dist/http/health.js.map +0 -1
- package/dist/http/helpers.d.ts.map +0 -1
- package/dist/http/helpers.js.map +0 -1
- package/dist/http/native.d.ts.map +0 -1
- package/dist/http/native.js.map +0 -1
- package/dist/http/rate-limit.d.ts.map +0 -1
- package/dist/http/rate-limit.js.map +0 -1
- package/dist/index.d.ts.map +0 -1
- package/dist/index.js.map +0 -1
- package/dist/lib/cache.d.ts +0 -54
- package/dist/lib/cache.d.ts.map +0 -1
- package/dist/lib/cache.js +0 -264
- package/dist/lib/cache.js.map +0 -1
- package/dist/lib/config.d.ts +0 -143
- package/dist/lib/config.d.ts.map +0 -1
- package/dist/lib/config.js +0 -476
- package/dist/lib/config.js.map +0 -1
- package/dist/lib/crypto.d.ts +0 -4
- package/dist/lib/crypto.d.ts.map +0 -1
- package/dist/lib/crypto.js +0 -56
- package/dist/lib/crypto.js.map +0 -1
- package/dist/lib/dom-noise-removal.d.ts +0 -2
- package/dist/lib/dom-noise-removal.d.ts.map +0 -1
- package/dist/lib/dom-noise-removal.js +0 -494
- package/dist/lib/dom-noise-removal.js.map +0 -1
- package/dist/lib/download.d.ts +0 -4
- package/dist/lib/download.d.ts.map +0 -1
- package/dist/lib/download.js +0 -106
- package/dist/lib/download.js.map +0 -1
- package/dist/lib/errors.d.ts +0 -14
- package/dist/lib/errors.d.ts.map +0 -1
- package/dist/lib/errors.js +0 -72
- package/dist/lib/errors.js.map +0 -1
- package/dist/lib/fetch-content.d.ts +0 -5
- package/dist/lib/fetch-content.d.ts.map +0 -1
- package/dist/lib/fetch-content.js +0 -164
- package/dist/lib/fetch-content.js.map +0 -1
- package/dist/lib/fetch-stream.d.ts +0 -5
- package/dist/lib/fetch-stream.d.ts.map +0 -1
- package/dist/lib/fetch-stream.js +0 -29
- package/dist/lib/fetch-stream.js.map +0 -1
- package/dist/lib/fetch.d.ts.map +0 -1
- package/dist/lib/fetch.js.map +0 -1
- package/dist/lib/host-normalization.d.ts +0 -2
- package/dist/lib/host-normalization.d.ts.map +0 -1
- package/dist/lib/host-normalization.js +0 -91
- package/dist/lib/host-normalization.js.map +0 -1
- package/dist/lib/ip-blocklist.d.ts +0 -9
- package/dist/lib/ip-blocklist.d.ts.map +0 -1
- package/dist/lib/ip-blocklist.js +0 -79
- package/dist/lib/ip-blocklist.js.map +0 -1
- package/dist/lib/json.d.ts +0 -2
- package/dist/lib/json.d.ts.map +0 -1
- package/dist/lib/json.js +0 -45
- package/dist/lib/json.js.map +0 -1
- package/dist/lib/language-detection.d.ts +0 -3
- package/dist/lib/language-detection.d.ts.map +0 -1
- package/dist/lib/language-detection.js +0 -355
- package/dist/lib/language-detection.js.map +0 -1
- package/dist/lib/markdown-cleanup.d.ts.map +0 -1
- package/dist/lib/markdown-cleanup.js +0 -532
- package/dist/lib/markdown-cleanup.js.map +0 -1
- package/dist/lib/mcp-lifecycle.d.ts +0 -5
- package/dist/lib/mcp-lifecycle.d.ts.map +0 -1
- package/dist/lib/mcp-lifecycle.js +0 -51
- package/dist/lib/mcp-lifecycle.js.map +0 -1
- package/dist/lib/mcp-validator.d.ts +0 -17
- package/dist/lib/mcp-validator.d.ts.map +0 -1
- package/dist/lib/mcp-validator.js +0 -45
- package/dist/lib/mcp-validator.js.map +0 -1
- package/dist/lib/mcp.d.ts +0 -4
- package/dist/lib/mcp.d.ts.map +0 -1
- package/dist/lib/mcp.js.map +0 -1
- package/dist/lib/observability.d.ts +0 -23
- package/dist/lib/observability.d.ts.map +0 -1
- package/dist/lib/observability.js +0 -238
- package/dist/lib/observability.js.map +0 -1
- package/dist/lib/server-tuning.d.ts +0 -15
- package/dist/lib/server-tuning.d.ts.map +0 -1
- package/dist/lib/server-tuning.js +0 -49
- package/dist/lib/server-tuning.js.map +0 -1
- package/dist/lib/session.d.ts +0 -45
- package/dist/lib/session.d.ts.map +0 -1
- package/dist/lib/session.js +0 -263
- package/dist/lib/session.js.map +0 -1
- package/dist/lib/timer-utils.d.ts +0 -13
- package/dist/lib/timer-utils.d.ts.map +0 -1
- package/dist/lib/timer-utils.js +0 -44
- package/dist/lib/timer-utils.js.map +0 -1
- package/dist/lib/tool-errors.d.ts +0 -12
- package/dist/lib/tool-errors.d.ts.map +0 -1
- package/dist/lib/tool-errors.js +0 -55
- package/dist/lib/tool-errors.js.map +0 -1
- package/dist/lib/tool-pipeline.d.ts.map +0 -1
- package/dist/lib/tool-pipeline.js.map +0 -1
- package/dist/lib/tool-progress.d.ts.map +0 -1
- package/dist/lib/tool-progress.js.map +0 -1
- package/dist/lib/type-guards.d.ts +0 -16
- package/dist/lib/type-guards.d.ts.map +0 -1
- package/dist/lib/type-guards.js +0 -13
- package/dist/lib/type-guards.js.map +0 -1
- package/dist/prompts/index.d.ts.map +0 -1
- package/dist/prompts/index.js.map +0 -1
- package/dist/resources/index.d.ts.map +0 -1
- package/dist/resources/index.js.map +0 -1
- package/dist/resources/instructions.d.ts.map +0 -1
- package/dist/resources/instructions.js.map +0 -1
- package/dist/schemas/inputs.d.ts.map +0 -1
- package/dist/schemas/inputs.js.map +0 -1
- package/dist/schemas/outputs.d.ts.map +0 -1
- package/dist/schemas/outputs.js.map +0 -1
- package/dist/server.d.ts.map +0 -1
- package/dist/server.js.map +0 -1
- package/dist/tasks/execution.d.ts.map +0 -1
- package/dist/tasks/execution.js.map +0 -1
- package/dist/tasks/manager.d.ts.map +0 -1
- package/dist/tasks/manager.js.map +0 -1
- package/dist/tasks/owner.d.ts.map +0 -1
- package/dist/tasks/owner.js.map +0 -1
- package/dist/tasks/tool-registry.d.ts.map +0 -1
- package/dist/tasks/tool-registry.js.map +0 -1
- package/dist/tools/fetch-url.d.ts.map +0 -1
- package/dist/tools/fetch-url.js.map +0 -1
- package/dist/tools/index.d.ts.map +0 -1
- package/dist/tools/index.js.map +0 -1
- package/dist/transform/transform.d.ts.map +0 -1
- package/dist/transform/transform.js.map +0 -1
- package/dist/transform/types.d.ts.map +0 -1
- package/dist/transform/types.js.map +0 -1
- package/dist/transform/worker-pool.d.ts.map +0 -1
- package/dist/transform/worker-pool.js.map +0 -1
- package/dist/transform/workers/shared.d.ts.map +0 -1
- package/dist/transform/workers/shared.js.map +0 -1
- package/dist/transform/workers/transform-child.d.ts.map +0 -1
- package/dist/transform/workers/transform-child.js.map +0 -1
- package/dist/transform/workers/transform-worker.d.ts.map +0 -1
- package/dist/transform/workers/transform-worker.js.map +0 -1
|
@@ -3,15 +3,16 @@ import diagnosticsChannel from 'node:diagnostics_channel';
|
|
|
3
3
|
import { performance } from 'node:perf_hooks';
|
|
4
4
|
import { isProbablyReaderable, Readability } from '@mozilla/readability';
|
|
5
5
|
import { parseHTML } from 'linkedom';
|
|
6
|
-
import {
|
|
7
|
-
import {
|
|
8
|
-
import {
|
|
9
|
-
import {
|
|
10
|
-
import { isRawTextContentUrl } from '../lib/
|
|
11
|
-
import {
|
|
12
|
-
import {
|
|
13
|
-
import {
|
|
14
|
-
import {
|
|
6
|
+
import { removeNoiseFromHtml } from '../lib/content.js';
|
|
7
|
+
import { addSourceToMarkdown, buildMetadataFooter, cleanupMarkdownArtifacts, extractTitleFromRawMarkdown, isRawTextContent, } from '../lib/content.js';
|
|
8
|
+
import { config } from '../lib/core.js';
|
|
9
|
+
import { getOperationId, getRequestId, logDebug, logError, logInfo, logWarn, redactUrl, } from '../lib/core.js';
|
|
10
|
+
import { isRawTextContentUrl } from '../lib/http.js';
|
|
11
|
+
import { createAbortError, throwIfAborted } from '../lib/utils.js';
|
|
12
|
+
import { FetchError, getErrorMessage } from '../lib/utils.js';
|
|
13
|
+
import { isObject } from '../lib/utils.js';
|
|
14
|
+
import { translateHtmlFragmentToMarkdown } from './html-translators.js';
|
|
15
|
+
import { extractMetadata, extractMetadataFromHead, mergeMetadata, } from './metadata.js';
|
|
15
16
|
import { getOrCreateWorkerPool, getWorkerPoolStats, shutdownWorkerPool, } from './worker-pool.js';
|
|
16
17
|
const utf8Decoder = new TextDecoder('utf-8');
|
|
17
18
|
function decodeInput(input, encoding) {
|
|
@@ -30,43 +31,9 @@ function decodeInput(input, encoding) {
|
|
|
30
31
|
return utf8Decoder.decode(input);
|
|
31
32
|
}
|
|
32
33
|
}
|
|
33
|
-
function getTagName(node) {
|
|
34
|
-
if (!isLikeNode(node))
|
|
35
|
-
return '';
|
|
36
|
-
const raw = node.tagName;
|
|
37
|
-
return typeof raw === 'string' ? raw.toUpperCase() : '';
|
|
38
|
-
}
|
|
39
34
|
function asError(value) {
|
|
40
35
|
return value instanceof Error ? value : undefined;
|
|
41
36
|
}
|
|
42
|
-
function getAbortReason(signal) {
|
|
43
|
-
const record = isObject(signal) ? signal : null;
|
|
44
|
-
return record && 'reason' in record ? record['reason'] : undefined;
|
|
45
|
-
}
|
|
46
|
-
function isTimeoutAbortReason(reason) {
|
|
47
|
-
return reason instanceof Error && reason.name === 'TimeoutError';
|
|
48
|
-
}
|
|
49
|
-
function throwIfAborted(signal, url, stage) {
|
|
50
|
-
if (!signal?.aborted)
|
|
51
|
-
return;
|
|
52
|
-
const reason = getAbortReason(signal);
|
|
53
|
-
if (isTimeoutAbortReason(reason)) {
|
|
54
|
-
throw new FetchError('Request timeout', url, 504, {
|
|
55
|
-
reason: 'timeout',
|
|
56
|
-
stage,
|
|
57
|
-
});
|
|
58
|
-
}
|
|
59
|
-
throw new FetchError('Request was canceled', url, 499, {
|
|
60
|
-
reason: 'aborted',
|
|
61
|
-
stage,
|
|
62
|
-
});
|
|
63
|
-
}
|
|
64
|
-
function createAbortError(url, stage) {
|
|
65
|
-
return new FetchError('Request was canceled', url, 499, {
|
|
66
|
-
reason: 'aborted',
|
|
67
|
-
stage,
|
|
68
|
-
});
|
|
69
|
-
}
|
|
70
37
|
const abortPolicy = { throwIfAborted, createAbortError };
|
|
71
38
|
function buildTransformSignal(signal) {
|
|
72
39
|
const { timeoutMs } = config.transform;
|
|
@@ -213,9 +180,20 @@ function trimUtf8Buffer(buffer, maxBytes) {
|
|
|
213
180
|
function trimDanglingTagFragment(content) {
|
|
214
181
|
const lastOpen = content.lastIndexOf('<');
|
|
215
182
|
const lastClose = content.lastIndexOf('>');
|
|
216
|
-
if (lastOpen > lastClose
|
|
217
|
-
|
|
218
|
-
|
|
183
|
+
if (lastOpen > lastClose) {
|
|
184
|
+
if (lastOpen === content.length - 1) {
|
|
185
|
+
return content.substring(0, lastOpen);
|
|
186
|
+
}
|
|
187
|
+
const code = content.codePointAt(lastOpen + 1);
|
|
188
|
+
if (code !== undefined &&
|
|
189
|
+
(code === 47 || // '/'
|
|
190
|
+
code === 33 || // '!'
|
|
191
|
+
code === 63 || // '?'
|
|
192
|
+
(code >= 65 && code <= 90) || // A-Z
|
|
193
|
+
(code >= 97 && code <= 122)) // a-z
|
|
194
|
+
) {
|
|
195
|
+
return content.substring(0, lastOpen);
|
|
196
|
+
}
|
|
219
197
|
}
|
|
220
198
|
return content;
|
|
221
199
|
}
|
|
@@ -245,179 +223,6 @@ function willTruncate(html) {
|
|
|
245
223
|
const maxSize = config.constants.maxHtmlSize;
|
|
246
224
|
return (maxSize > 0 && (html.length > maxSize || getUtf8ByteLength(html) > maxSize));
|
|
247
225
|
}
|
|
248
|
-
const HEAD_END_PATTERN = /<\/head\s*>|<body\b/i;
|
|
249
|
-
const MAX_HEAD_SCAN_LENGTH = 50_000;
|
|
250
|
-
function extractHeadSection(html) {
|
|
251
|
-
if (html.length <= MAX_HEAD_SCAN_LENGTH) {
|
|
252
|
-
const match = HEAD_END_PATTERN.exec(html);
|
|
253
|
-
return match ? html.substring(0, match.index) : null;
|
|
254
|
-
}
|
|
255
|
-
const searchText = html.substring(0, MAX_HEAD_SCAN_LENGTH);
|
|
256
|
-
const match = HEAD_END_PATTERN.exec(searchText);
|
|
257
|
-
if (!match)
|
|
258
|
-
return null;
|
|
259
|
-
return html.substring(0, match.index);
|
|
260
|
-
}
|
|
261
|
-
function extractMetadataFromHead(html, baseUrl) {
|
|
262
|
-
const headSection = extractHeadSection(html);
|
|
263
|
-
if (!headSection)
|
|
264
|
-
return null;
|
|
265
|
-
try {
|
|
266
|
-
const { document } = parseHTML(`<!DOCTYPE html><html>${headSection}</head><body></body></html>`);
|
|
267
|
-
return extractMetadata(document, baseUrl);
|
|
268
|
-
}
|
|
269
|
-
catch {
|
|
270
|
-
return null;
|
|
271
|
-
}
|
|
272
|
-
}
|
|
273
|
-
function mergeMetadata(early, late) {
|
|
274
|
-
if (!early)
|
|
275
|
-
return late;
|
|
276
|
-
const merged = {};
|
|
277
|
-
const keys = [
|
|
278
|
-
'title',
|
|
279
|
-
'description',
|
|
280
|
-
'author',
|
|
281
|
-
'image',
|
|
282
|
-
'publishedAt',
|
|
283
|
-
'modifiedAt',
|
|
284
|
-
];
|
|
285
|
-
for (const key of keys) {
|
|
286
|
-
const value = late[key] ?? early[key];
|
|
287
|
-
if (value !== undefined)
|
|
288
|
-
merged[key] = value;
|
|
289
|
-
}
|
|
290
|
-
return merged;
|
|
291
|
-
}
|
|
292
|
-
const META_PROPERTY_HANDLERS = new Map([
|
|
293
|
-
[
|
|
294
|
-
'og:title',
|
|
295
|
-
(ctx, c) => {
|
|
296
|
-
ctx.title.og = c;
|
|
297
|
-
},
|
|
298
|
-
],
|
|
299
|
-
[
|
|
300
|
-
'og:description',
|
|
301
|
-
(ctx, c) => {
|
|
302
|
-
ctx.description.og = c;
|
|
303
|
-
},
|
|
304
|
-
],
|
|
305
|
-
[
|
|
306
|
-
'og:image',
|
|
307
|
-
(ctx, c) => {
|
|
308
|
-
ctx.image = c;
|
|
309
|
-
},
|
|
310
|
-
],
|
|
311
|
-
[
|
|
312
|
-
'article:published_time',
|
|
313
|
-
(ctx, c) => {
|
|
314
|
-
ctx.publishedAt = c;
|
|
315
|
-
},
|
|
316
|
-
],
|
|
317
|
-
[
|
|
318
|
-
'article:modified_time',
|
|
319
|
-
(ctx, c) => {
|
|
320
|
-
ctx.modifiedAt = c;
|
|
321
|
-
},
|
|
322
|
-
],
|
|
323
|
-
]);
|
|
324
|
-
const META_NAME_HANDLERS = new Map([
|
|
325
|
-
[
|
|
326
|
-
'twitter:title',
|
|
327
|
-
(ctx, c) => {
|
|
328
|
-
ctx.title.twitter = c;
|
|
329
|
-
},
|
|
330
|
-
],
|
|
331
|
-
[
|
|
332
|
-
'twitter:description',
|
|
333
|
-
(ctx, c) => {
|
|
334
|
-
ctx.description.twitter = c;
|
|
335
|
-
},
|
|
336
|
-
],
|
|
337
|
-
[
|
|
338
|
-
'description',
|
|
339
|
-
(ctx, c) => {
|
|
340
|
-
ctx.description.standard = c;
|
|
341
|
-
},
|
|
342
|
-
],
|
|
343
|
-
[
|
|
344
|
-
'author',
|
|
345
|
-
(ctx, c) => {
|
|
346
|
-
ctx.author = c;
|
|
347
|
-
},
|
|
348
|
-
],
|
|
349
|
-
]);
|
|
350
|
-
function processMetaTag(ctx, tag) {
|
|
351
|
-
const content = tag.getAttribute('content')?.trim();
|
|
352
|
-
if (!content)
|
|
353
|
-
return;
|
|
354
|
-
const property = tag.getAttribute('property');
|
|
355
|
-
if (property)
|
|
356
|
-
META_PROPERTY_HANDLERS.get(property)?.(ctx, content);
|
|
357
|
-
const name = tag.getAttribute('name');
|
|
358
|
-
if (name)
|
|
359
|
-
META_NAME_HANDLERS.get(name)?.(ctx, content);
|
|
360
|
-
}
|
|
361
|
-
function buildMetaContext(document) {
|
|
362
|
-
const ctx = { title: {}, description: {} };
|
|
363
|
-
for (const tag of document.querySelectorAll('meta')) {
|
|
364
|
-
processMetaTag(ctx, tag);
|
|
365
|
-
}
|
|
366
|
-
const titleEl = document.querySelector('title');
|
|
367
|
-
if (!ctx.title.standard && titleEl?.textContent) {
|
|
368
|
-
ctx.title.standard = titleEl.textContent.trim();
|
|
369
|
-
}
|
|
370
|
-
return ctx;
|
|
371
|
-
}
|
|
372
|
-
function resolveMetadataFromContext(ctx) {
|
|
373
|
-
const metadata = {};
|
|
374
|
-
const resolvedTitle = ctx.title.og ?? ctx.title.twitter ?? ctx.title.standard;
|
|
375
|
-
const resolvedDesc = ctx.description.og ?? ctx.description.twitter ?? ctx.description.standard;
|
|
376
|
-
if (resolvedTitle)
|
|
377
|
-
metadata.title = resolvedTitle;
|
|
378
|
-
if (resolvedDesc)
|
|
379
|
-
metadata.description = resolvedDesc;
|
|
380
|
-
if (ctx.author)
|
|
381
|
-
metadata.author = ctx.author;
|
|
382
|
-
if (ctx.image)
|
|
383
|
-
metadata.image = ctx.image;
|
|
384
|
-
if (ctx.publishedAt)
|
|
385
|
-
metadata.publishedAt = ctx.publishedAt;
|
|
386
|
-
if (ctx.modifiedAt)
|
|
387
|
-
metadata.modifiedAt = ctx.modifiedAt;
|
|
388
|
-
return metadata;
|
|
389
|
-
}
|
|
390
|
-
function extractMetadata(document, baseUrl) {
|
|
391
|
-
const ctx = buildMetaContext(document);
|
|
392
|
-
const metadata = resolveMetadataFromContext(ctx);
|
|
393
|
-
if (baseUrl) {
|
|
394
|
-
const icon32 = document.querySelector('link[rel="icon"][sizes="32x32"]');
|
|
395
|
-
const href = icon32?.getAttribute('href');
|
|
396
|
-
if (href) {
|
|
397
|
-
const resolved = resolveFaviconUrl(href, baseUrl);
|
|
398
|
-
if (resolved)
|
|
399
|
-
metadata.favicon = resolved;
|
|
400
|
-
}
|
|
401
|
-
}
|
|
402
|
-
return metadata;
|
|
403
|
-
}
|
|
404
|
-
function resolveFaviconUrl(href, baseUrl) {
|
|
405
|
-
const trimmed = href.trim();
|
|
406
|
-
if (!trimmed)
|
|
407
|
-
return undefined;
|
|
408
|
-
if (trimmed.toLowerCase().startsWith('data:'))
|
|
409
|
-
return undefined;
|
|
410
|
-
try {
|
|
411
|
-
const resolved = new URL(trimmed, baseUrl);
|
|
412
|
-
if (resolved.protocol !== 'http:' && resolved.protocol !== 'https:') {
|
|
413
|
-
return undefined;
|
|
414
|
-
}
|
|
415
|
-
return resolved.toString();
|
|
416
|
-
}
|
|
417
|
-
catch {
|
|
418
|
-
return undefined;
|
|
419
|
-
}
|
|
420
|
-
}
|
|
421
226
|
function isReadabilityCompatible(doc) {
|
|
422
227
|
if (!isObject(doc))
|
|
423
228
|
return false;
|
|
@@ -587,427 +392,6 @@ export function extractContent(html, url, options = {
|
|
|
587
392
|
const result = extractContentContext(html, url, options);
|
|
588
393
|
return { article: result.article, metadata: result.metadata };
|
|
589
394
|
}
|
|
590
|
-
const CODE_BLOCK = {
|
|
591
|
-
fence: '```',
|
|
592
|
-
format: (code, language = '') => `\`\`\`${language}\n${code}\n\`\`\``,
|
|
593
|
-
};
|
|
594
|
-
function buildInlineCode(content) {
|
|
595
|
-
const trimmed = content.trim();
|
|
596
|
-
if (!trimmed)
|
|
597
|
-
return '``';
|
|
598
|
-
let maxBackticks = 0;
|
|
599
|
-
let currentRun = 0;
|
|
600
|
-
for (const char of trimmed) {
|
|
601
|
-
if (char === '`')
|
|
602
|
-
currentRun += 1;
|
|
603
|
-
else {
|
|
604
|
-
if (currentRun > maxBackticks)
|
|
605
|
-
maxBackticks = currentRun;
|
|
606
|
-
currentRun = 0;
|
|
607
|
-
}
|
|
608
|
-
}
|
|
609
|
-
if (currentRun > maxBackticks)
|
|
610
|
-
maxBackticks = currentRun;
|
|
611
|
-
const delimiter = '`'.repeat(maxBackticks + 1);
|
|
612
|
-
const padding = trimmed.startsWith('`') || trimmed.endsWith('`') ? ' ' : '';
|
|
613
|
-
return `${delimiter}${padding}${trimmed}${padding}${delimiter}`;
|
|
614
|
-
}
|
|
615
|
-
function deriveAltFromImageUrl(src) {
|
|
616
|
-
if (!src)
|
|
617
|
-
return '';
|
|
618
|
-
try {
|
|
619
|
-
const isAbsolute = URL.canParse(src);
|
|
620
|
-
let parsed = null;
|
|
621
|
-
if (isAbsolute) {
|
|
622
|
-
parsed = new URL(src);
|
|
623
|
-
}
|
|
624
|
-
else if (URL.canParse(src, 'http://localhost')) {
|
|
625
|
-
parsed = new URL(src, 'http://localhost');
|
|
626
|
-
}
|
|
627
|
-
if (!parsed)
|
|
628
|
-
return '';
|
|
629
|
-
if (isAbsolute &&
|
|
630
|
-
parsed.protocol !== 'http:' &&
|
|
631
|
-
parsed.protocol !== 'https:') {
|
|
632
|
-
return '';
|
|
633
|
-
}
|
|
634
|
-
const { pathname } = parsed;
|
|
635
|
-
const segments = pathname.split('/');
|
|
636
|
-
const filename = segments.pop() ?? '';
|
|
637
|
-
if (!filename)
|
|
638
|
-
return '';
|
|
639
|
-
const dotIndex = filename.lastIndexOf('.');
|
|
640
|
-
const name = dotIndex > 0 ? filename.slice(0, dotIndex) : filename;
|
|
641
|
-
return name.replace(/[_-]+/g, ' ').trim();
|
|
642
|
-
}
|
|
643
|
-
catch {
|
|
644
|
-
return '';
|
|
645
|
-
}
|
|
646
|
-
}
|
|
647
|
-
function hasGetAttribute(value) {
|
|
648
|
-
return (isObject(value) &&
|
|
649
|
-
typeof value.getAttribute === 'function');
|
|
650
|
-
}
|
|
651
|
-
function isCodeBlock(parent) {
|
|
652
|
-
const tagName = getTagName(parent);
|
|
653
|
-
return tagName === 'PRE' || tagName === 'WRAPPED-PRE';
|
|
654
|
-
}
|
|
655
|
-
function resolveAttributeLanguage(node) {
|
|
656
|
-
const getAttribute = hasGetAttribute(node)
|
|
657
|
-
? node.getAttribute.bind(node)
|
|
658
|
-
: undefined;
|
|
659
|
-
const className = getAttribute?.('class') ?? '';
|
|
660
|
-
const dataLanguage = getAttribute?.('data-language') ?? '';
|
|
661
|
-
return resolveLanguageFromAttributes(className, dataLanguage);
|
|
662
|
-
}
|
|
663
|
-
function findLanguageFromCodeChild(node) {
|
|
664
|
-
if (!isLikeNode(node))
|
|
665
|
-
return undefined;
|
|
666
|
-
const childNodes = Array.from(node.childNodes ?? []);
|
|
667
|
-
for (const child of childNodes) {
|
|
668
|
-
if (!isLikeNode(child))
|
|
669
|
-
continue;
|
|
670
|
-
const raw = child.rawTagName;
|
|
671
|
-
const tagName = typeof raw === 'string' ? raw.toUpperCase() : '';
|
|
672
|
-
if (tagName === 'CODE')
|
|
673
|
-
return resolveAttributeLanguage(child);
|
|
674
|
-
}
|
|
675
|
-
return undefined;
|
|
676
|
-
}
|
|
677
|
-
function createCodeBlockPostprocessor(language) {
|
|
678
|
-
return ({ content }) => {
|
|
679
|
-
const trimmed = content.trim();
|
|
680
|
-
if (!trimmed)
|
|
681
|
-
return '';
|
|
682
|
-
const resolvedLanguage = language ?? detectLanguageFromCode(trimmed) ?? '';
|
|
683
|
-
return CODE_BLOCK.format(trimmed, resolvedLanguage);
|
|
684
|
-
};
|
|
685
|
-
}
|
|
686
|
-
function buildInlineCodeTranslator() {
|
|
687
|
-
return {
|
|
688
|
-
spaceIfRepeatingChar: true,
|
|
689
|
-
noEscape: true,
|
|
690
|
-
postprocess: ({ content }) => buildInlineCode(content),
|
|
691
|
-
};
|
|
692
|
-
}
|
|
693
|
-
function buildCodeTranslator(ctx) {
|
|
694
|
-
const inlineCodeTranslator = buildInlineCodeTranslator();
|
|
695
|
-
if (!isObject(ctx))
|
|
696
|
-
return inlineCodeTranslator;
|
|
697
|
-
const { parent } = ctx;
|
|
698
|
-
if (!isCodeBlock(parent))
|
|
699
|
-
return inlineCodeTranslator;
|
|
700
|
-
return { noEscape: true, preserveWhitespace: true };
|
|
701
|
-
}
|
|
702
|
-
function extractFirstSrcsetUrl(srcset) {
|
|
703
|
-
const first = srcset.split(',')[0];
|
|
704
|
-
if (!first)
|
|
705
|
-
return '';
|
|
706
|
-
return first.trim().split(/\s+/)[0] ?? '';
|
|
707
|
-
}
|
|
708
|
-
const LAZY_SRC_ATTRIBUTES = [
|
|
709
|
-
'data-src',
|
|
710
|
-
'data-lazy-src',
|
|
711
|
-
'data-original',
|
|
712
|
-
'data-srcset',
|
|
713
|
-
];
|
|
714
|
-
function isDataUri(value) {
|
|
715
|
-
return value.startsWith('data:');
|
|
716
|
-
}
|
|
717
|
-
function extractNonDataSrcsetUrl(value) {
|
|
718
|
-
const url = extractFirstSrcsetUrl(value);
|
|
719
|
-
return url && !isDataUri(url) ? url : undefined;
|
|
720
|
-
}
|
|
721
|
-
function resolveLazySrc(getAttribute) {
|
|
722
|
-
for (const attr of LAZY_SRC_ATTRIBUTES) {
|
|
723
|
-
const lazy = getAttribute(attr);
|
|
724
|
-
if (!lazy || isDataUri(lazy))
|
|
725
|
-
continue;
|
|
726
|
-
if (attr === 'data-srcset') {
|
|
727
|
-
const url = extractNonDataSrcsetUrl(lazy);
|
|
728
|
-
if (url)
|
|
729
|
-
return url;
|
|
730
|
-
continue;
|
|
731
|
-
}
|
|
732
|
-
return lazy;
|
|
733
|
-
}
|
|
734
|
-
return undefined;
|
|
735
|
-
}
|
|
736
|
-
function resolveImageSrc(getAttribute) {
|
|
737
|
-
if (!getAttribute)
|
|
738
|
-
return '';
|
|
739
|
-
const srcRaw = getAttribute('src') ?? '';
|
|
740
|
-
if (srcRaw && !isDataUri(srcRaw))
|
|
741
|
-
return srcRaw;
|
|
742
|
-
// First check common lazy-loading attributes that may contain non-data URLs before falling back to the native srcset, as some sites use data URIs in lazy attributes while still providing valid URLs in srcset.
|
|
743
|
-
const lazySrc = resolveLazySrc(getAttribute);
|
|
744
|
-
if (lazySrc)
|
|
745
|
-
return lazySrc;
|
|
746
|
-
// If the src is a data URI or missing, check srcset for a valid URL. Some sites use srcset with data URIs in src and actual URLs in srcset for responsive images.
|
|
747
|
-
const srcset = getAttribute('srcset');
|
|
748
|
-
if (srcset) {
|
|
749
|
-
const url = extractNonDataSrcsetUrl(srcset);
|
|
750
|
-
if (url)
|
|
751
|
-
return url;
|
|
752
|
-
}
|
|
753
|
-
// If the only available src is a data URI, we choose to omit it rather than include the raw data in the alt text or URL, as data URIs can be very long and are not useful in Markdown output.
|
|
754
|
-
if (isDataUri(srcRaw))
|
|
755
|
-
return '[data URI removed]';
|
|
756
|
-
return '';
|
|
757
|
-
}
|
|
758
|
-
function buildImageTranslator(ctx) {
|
|
759
|
-
if (!isObject(ctx))
|
|
760
|
-
return { content: '' };
|
|
761
|
-
const { node } = ctx;
|
|
762
|
-
const getAttribute = hasGetAttribute(node)
|
|
763
|
-
? node.getAttribute.bind(node)
|
|
764
|
-
: undefined;
|
|
765
|
-
const src = resolveImageSrc(getAttribute);
|
|
766
|
-
const existingAlt = getAttribute?.('alt') ?? '';
|
|
767
|
-
const alt = existingAlt.trim() || deriveAltFromImageUrl(src);
|
|
768
|
-
const markdown = ``;
|
|
769
|
-
return { content: markdown };
|
|
770
|
-
}
|
|
771
|
-
const GFM_ALERT_MAP = new Map([
|
|
772
|
-
['note', 'NOTE'],
|
|
773
|
-
['info', 'NOTE'],
|
|
774
|
-
['tip', 'TIP'],
|
|
775
|
-
['hint', 'TIP'],
|
|
776
|
-
['warning', 'WARNING'],
|
|
777
|
-
['warn', 'WARNING'],
|
|
778
|
-
['caution', 'CAUTION'],
|
|
779
|
-
['danger', 'CAUTION'],
|
|
780
|
-
['important', 'IMPORTANT'],
|
|
781
|
-
]);
|
|
782
|
-
function resolveGfmAlertType(className) {
|
|
783
|
-
const lower = className.toLowerCase();
|
|
784
|
-
for (const [key, type] of GFM_ALERT_MAP) {
|
|
785
|
-
if (lower.includes(key))
|
|
786
|
-
return type;
|
|
787
|
-
}
|
|
788
|
-
return undefined;
|
|
789
|
-
}
|
|
790
|
-
function resolveDlNodeName(child) {
|
|
791
|
-
if (!isLikeNode(child))
|
|
792
|
-
return '';
|
|
793
|
-
const raw = child.nodeName;
|
|
794
|
-
return typeof raw === 'string' ? raw.toUpperCase() : '';
|
|
795
|
-
}
|
|
796
|
-
function resolveDlTextContent(child) {
|
|
797
|
-
if (!isLikeNode(child))
|
|
798
|
-
return '';
|
|
799
|
-
const raw = child.textContent;
|
|
800
|
-
return typeof raw === 'string' ? raw.trim() : '';
|
|
801
|
-
}
|
|
802
|
-
function buildDlChildFragment(child) {
|
|
803
|
-
const nodeName = resolveDlNodeName(child);
|
|
804
|
-
if (nodeName === 'DT')
|
|
805
|
-
return `**${resolveDlTextContent(child)}**\n`;
|
|
806
|
-
if (nodeName === 'DD')
|
|
807
|
-
return `: ${resolveDlTextContent(child)}\n`;
|
|
808
|
-
return null;
|
|
809
|
-
}
|
|
810
|
-
function hasComplexTableLayout(node) {
|
|
811
|
-
if (!isLikeNode(node))
|
|
812
|
-
return false;
|
|
813
|
-
const innerHTML = typeof node.innerHTML === 'string' ? node.innerHTML : '';
|
|
814
|
-
return /(?:colspan|rowspan)=["']?[2-9]/i.test(innerHTML);
|
|
815
|
-
}
|
|
816
|
-
function buildPreTranslator(ctx) {
|
|
817
|
-
if (!isObject(ctx))
|
|
818
|
-
return {};
|
|
819
|
-
const { node } = ctx;
|
|
820
|
-
const attributeLanguage = resolveAttributeLanguage(node) ?? findLanguageFromCodeChild(node);
|
|
821
|
-
return {
|
|
822
|
-
noEscape: true,
|
|
823
|
-
preserveWhitespace: true,
|
|
824
|
-
postprocess: createCodeBlockPostprocessor(attributeLanguage),
|
|
825
|
-
};
|
|
826
|
-
}
|
|
827
|
-
function getNodeAttr(node) {
|
|
828
|
-
if (!isLikeNode(node))
|
|
829
|
-
return undefined;
|
|
830
|
-
return typeof node.getAttribute === 'function'
|
|
831
|
-
? node.getAttribute.bind(node)
|
|
832
|
-
: undefined;
|
|
833
|
-
}
|
|
834
|
-
function buildDivTranslator(ctx) {
|
|
835
|
-
if (!isObject(ctx))
|
|
836
|
-
return {};
|
|
837
|
-
const { node } = ctx;
|
|
838
|
-
const getAttribute = getNodeAttr(node);
|
|
839
|
-
if (!getAttribute)
|
|
840
|
-
return {};
|
|
841
|
-
const className = getAttribute('class') ?? '';
|
|
842
|
-
if (className.includes('mermaid')) {
|
|
843
|
-
return {
|
|
844
|
-
noEscape: true,
|
|
845
|
-
preserveWhitespace: true,
|
|
846
|
-
postprocess: ({ content }) => `\n\n\`\`\`mermaid\n${content.trim()}\n\`\`\`\n\n`,
|
|
847
|
-
};
|
|
848
|
-
}
|
|
849
|
-
const isAdmonition = className.includes('admonition') ||
|
|
850
|
-
className.includes('callout') ||
|
|
851
|
-
className.includes('custom-block') ||
|
|
852
|
-
getAttribute('role') === 'alert' ||
|
|
853
|
-
/\b(note|tip|info|warning|danger|caution|important)\b/i.test(className);
|
|
854
|
-
if (isAdmonition) {
|
|
855
|
-
return {
|
|
856
|
-
postprocess: ({ content }) => {
|
|
857
|
-
const alertType = resolveGfmAlertType(className);
|
|
858
|
-
const lines = content.trim().split('\n');
|
|
859
|
-
const header = alertType ? `> [!${alertType}]\n` : '';
|
|
860
|
-
return `\n\n${header}> ${lines.join('\n> ')}\n\n`;
|
|
861
|
-
},
|
|
862
|
-
};
|
|
863
|
-
}
|
|
864
|
-
if (!className.includes('type'))
|
|
865
|
-
return {};
|
|
866
|
-
return {
|
|
867
|
-
postprocess: ({ content }) => {
|
|
868
|
-
const lines = content.split('\n');
|
|
869
|
-
const separated = [];
|
|
870
|
-
for (let i = 0; i < lines.length; i++) {
|
|
871
|
-
const line = lines[i] ?? '';
|
|
872
|
-
const nextLine = i < lines.length - 1 ? (lines[i + 1] ?? '') : '';
|
|
873
|
-
separated.push(line);
|
|
874
|
-
if (line.trim() &&
|
|
875
|
-
nextLine.trim() &&
|
|
876
|
-
line.includes(':') &&
|
|
877
|
-
nextLine.includes(':') &&
|
|
878
|
-
!line.startsWith(' ') &&
|
|
879
|
-
!nextLine.startsWith(' ')) {
|
|
880
|
-
separated.push('');
|
|
881
|
-
}
|
|
882
|
-
}
|
|
883
|
-
return separated.join('\n');
|
|
884
|
-
},
|
|
885
|
-
};
|
|
886
|
-
}
|
|
887
|
-
function buildSectionTranslator(ctx) {
|
|
888
|
-
if (isObject(ctx)) {
|
|
889
|
-
const { node } = ctx;
|
|
890
|
-
const getAttribute = getNodeAttr(node);
|
|
891
|
-
if (getAttribute?.('class')?.includes('tsd-member')) {
|
|
892
|
-
return {
|
|
893
|
-
postprocess: ({ content }) => `\n\n \n\n${content}\n\n`,
|
|
894
|
-
};
|
|
895
|
-
}
|
|
896
|
-
}
|
|
897
|
-
return {
|
|
898
|
-
postprocess: ({ content }) => `\n\n${content}\n\n`,
|
|
899
|
-
};
|
|
900
|
-
}
|
|
901
|
-
function buildSpanTranslator(ctx) {
|
|
902
|
-
if (!isObject(ctx))
|
|
903
|
-
return {};
|
|
904
|
-
const { node } = ctx;
|
|
905
|
-
const getAttribute = getNodeAttr(node);
|
|
906
|
-
if (!getAttribute)
|
|
907
|
-
return {};
|
|
908
|
-
const dataAs = getAttribute('data-as') ?? '';
|
|
909
|
-
if (dataAs === 'p') {
|
|
910
|
-
return {
|
|
911
|
-
postprocess: ({ content }) => `\n\n${content.trim()}\n\n`,
|
|
912
|
-
};
|
|
913
|
-
}
|
|
914
|
-
return {};
|
|
915
|
-
}
|
|
916
|
-
function buildMermaidPreTranslator(ctx) {
|
|
917
|
-
if (!isObject(ctx))
|
|
918
|
-
return buildPreTranslator(ctx);
|
|
919
|
-
const { node } = ctx;
|
|
920
|
-
const getAttribute = getNodeAttr(node);
|
|
921
|
-
if (!getAttribute)
|
|
922
|
-
return buildPreTranslator(ctx);
|
|
923
|
-
const className = getAttribute('class') ?? '';
|
|
924
|
-
if (className.includes('mermaid')) {
|
|
925
|
-
return {
|
|
926
|
-
noEscape: true,
|
|
927
|
-
preserveWhitespace: true,
|
|
928
|
-
postprocess: ({ content }) => `\n\n\`\`\`mermaid\n${content.trim()}\n\`\`\`\n\n`,
|
|
929
|
-
};
|
|
930
|
-
}
|
|
931
|
-
return buildPreTranslator(ctx);
|
|
932
|
-
}
|
|
933
|
-
function createCustomTranslators() {
|
|
934
|
-
return {
|
|
935
|
-
code: (ctx) => buildCodeTranslator(ctx),
|
|
936
|
-
img: (ctx) => buildImageTranslator(ctx),
|
|
937
|
-
table: (ctx) => {
|
|
938
|
-
if (!isObject(ctx))
|
|
939
|
-
return {};
|
|
940
|
-
const { node } = ctx;
|
|
941
|
-
if (hasComplexTableLayout(node)) {
|
|
942
|
-
return {
|
|
943
|
-
postprocess: ({ content }) => {
|
|
944
|
-
const trimmed = content.trim();
|
|
945
|
-
if (!trimmed)
|
|
946
|
-
return '';
|
|
947
|
-
return `\n\n${trimmed}\n\n`;
|
|
948
|
-
},
|
|
949
|
-
};
|
|
950
|
-
}
|
|
951
|
-
return {};
|
|
952
|
-
},
|
|
953
|
-
dl: (ctx) => {
|
|
954
|
-
if (!isObject(ctx))
|
|
955
|
-
return { content: '' };
|
|
956
|
-
const { node } = ctx;
|
|
957
|
-
if (!isLikeNode(node))
|
|
958
|
-
return { content: '' };
|
|
959
|
-
const childNodes = Array.from(node.childNodes ?? []);
|
|
960
|
-
let items = '';
|
|
961
|
-
for (const child of childNodes) {
|
|
962
|
-
const fragment = buildDlChildFragment(child);
|
|
963
|
-
if (fragment !== null)
|
|
964
|
-
items += fragment;
|
|
965
|
-
}
|
|
966
|
-
return { content: items ? `\n${items}\n` : '' };
|
|
967
|
-
},
|
|
968
|
-
div: buildDivTranslator,
|
|
969
|
-
kbd: () => ({
|
|
970
|
-
postprocess: ({ content }) => `\`${content}\``,
|
|
971
|
-
}),
|
|
972
|
-
mark: () => ({
|
|
973
|
-
postprocess: ({ content }) => `==${content}==`,
|
|
974
|
-
}),
|
|
975
|
-
sub: () => ({
|
|
976
|
-
postprocess: ({ content }) => `~${content}~`,
|
|
977
|
-
}),
|
|
978
|
-
sup: () => ({
|
|
979
|
-
postprocess: ({ content }) => `^${content}^`,
|
|
980
|
-
}),
|
|
981
|
-
section: buildSectionTranslator,
|
|
982
|
-
details: () => ({
|
|
983
|
-
postprocess: ({ content }) => {
|
|
984
|
-
const trimmed = content.trim();
|
|
985
|
-
if (!trimmed)
|
|
986
|
-
return '';
|
|
987
|
-
return `\n\n${trimmed}\n\n`;
|
|
988
|
-
},
|
|
989
|
-
}),
|
|
990
|
-
summary: () => ({
|
|
991
|
-
postprocess: ({ content }) => `${content.trim()}\n\n`,
|
|
992
|
-
}),
|
|
993
|
-
span: buildSpanTranslator,
|
|
994
|
-
pre: buildMermaidPreTranslator,
|
|
995
|
-
};
|
|
996
|
-
}
|
|
997
|
-
let markdownConverter = null;
|
|
998
|
-
function getMarkdownConverter() {
|
|
999
|
-
markdownConverter ??= new NodeHtmlMarkdown({
|
|
1000
|
-
codeFence: CODE_BLOCK.fence,
|
|
1001
|
-
codeBlockStyle: 'fenced',
|
|
1002
|
-
emDelimiter: '_',
|
|
1003
|
-
bulletMarker: '-',
|
|
1004
|
-
globalEscape: [/[\\`*_~]/gm, '\\$&'],
|
|
1005
|
-
}, createCustomTranslators());
|
|
1006
|
-
return markdownConverter;
|
|
1007
|
-
}
|
|
1008
|
-
function translateHtmlFragmentToMarkdown(html) {
|
|
1009
|
-
return getMarkdownConverter().translate(html).trim();
|
|
1010
|
-
}
|
|
1011
395
|
function isWhitespaceChar(code) {
|
|
1012
396
|
return code === 9 || code === 10 || code === 12 || code === 13 || code === 32;
|
|
1013
397
|
}
|
|
@@ -1701,4 +1085,3 @@ export async function transformHtmlToMarkdown(html, url, options) {
|
|
|
1701
1085
|
export async function transformBufferToMarkdown(htmlBuffer, url, options) {
|
|
1702
1086
|
return transformInputToMarkdown(htmlBuffer, url, options);
|
|
1703
1087
|
}
|
|
1704
|
-
//# sourceMappingURL=transform.js.map
|