@j0hanz/fetch-url-mcp 1.4.0 → 1.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (215) hide show
  1. package/dist/cli.d.ts +2 -3
  2. package/dist/cli.js +1 -2
  3. package/dist/http/auth.d.ts +5 -3
  4. package/dist/http/auth.js +64 -15
  5. package/dist/http/health.d.ts +1 -2
  6. package/dist/http/health.js +7 -18
  7. package/dist/http/helpers.d.ts +3 -4
  8. package/dist/http/helpers.js +21 -21
  9. package/dist/http/native.d.ts +0 -1
  10. package/dist/http/native.js +34 -26
  11. package/dist/http/rate-limit.d.ts +0 -1
  12. package/dist/http/rate-limit.js +3 -4
  13. package/dist/index.d.ts +0 -1
  14. package/dist/index.js +17 -18
  15. package/dist/lib/{markdown-cleanup.d.ts → content.d.ts} +4 -2
  16. package/dist/lib/content.js +1356 -0
  17. package/dist/lib/core.d.ts +253 -0
  18. package/dist/lib/core.js +1228 -0
  19. package/dist/lib/{tool-pipeline.d.ts → fetch-pipeline.d.ts} +1 -2
  20. package/dist/lib/{tool-pipeline.js → fetch-pipeline.js} +10 -19
  21. package/dist/lib/{fetch.d.ts → http.d.ts} +7 -9
  22. package/dist/lib/{fetch.js → http.js} +706 -944
  23. package/dist/lib/mcp-tools.d.ts +28 -0
  24. package/dist/lib/mcp-tools.js +107 -0
  25. package/dist/lib/{tool-progress.d.ts → progress.d.ts} +0 -1
  26. package/dist/lib/{tool-progress.js → progress.js} +8 -13
  27. package/dist/lib/task-handlers.d.ts +5 -0
  28. package/dist/lib/{mcp.js → task-handlers.js} +56 -12
  29. package/dist/lib/url.d.ts +70 -0
  30. package/dist/lib/url.js +686 -0
  31. package/dist/lib/utils.d.ts +58 -0
  32. package/dist/lib/utils.js +304 -0
  33. package/dist/prompts/index.d.ts +0 -1
  34. package/dist/prompts/index.js +0 -1
  35. package/dist/resources/index.d.ts +0 -1
  36. package/dist/resources/index.js +74 -33
  37. package/dist/resources/instructions.d.ts +0 -1
  38. package/dist/resources/instructions.js +2 -2
  39. package/dist/schemas/inputs.d.ts +0 -1
  40. package/dist/schemas/inputs.js +2 -3
  41. package/dist/schemas/outputs.d.ts +0 -1
  42. package/dist/schemas/outputs.js +1 -2
  43. package/dist/server.d.ts +0 -1
  44. package/dist/server.js +16 -26
  45. package/dist/tasks/execution.d.ts +0 -1
  46. package/dist/tasks/execution.js +27 -24
  47. package/dist/tasks/manager.d.ts +7 -3
  48. package/dist/tasks/manager.js +53 -34
  49. package/dist/tasks/owner.d.ts +1 -2
  50. package/dist/tasks/owner.js +1 -2
  51. package/dist/tasks/tool-registry.d.ts +1 -2
  52. package/dist/tasks/tool-registry.js +0 -1
  53. package/dist/tools/fetch-url.d.ts +1 -2
  54. package/dist/tools/fetch-url.js +39 -31
  55. package/dist/tools/index.d.ts +0 -1
  56. package/dist/tools/index.js +0 -1
  57. package/dist/transform/html-translators.d.ts +1 -0
  58. package/dist/transform/html-translators.js +454 -0
  59. package/dist/transform/metadata.d.ts +4 -0
  60. package/dist/transform/metadata.js +183 -0
  61. package/dist/transform/transform.d.ts +0 -1
  62. package/dist/transform/transform.js +24 -641
  63. package/dist/transform/types.d.ts +9 -11
  64. package/dist/transform/types.js +0 -1
  65. package/dist/transform/worker-pool.d.ts +0 -1
  66. package/dist/transform/worker-pool.js +7 -16
  67. package/dist/transform/workers/shared.d.ts +0 -1
  68. package/dist/transform/workers/shared.js +1 -2
  69. package/dist/transform/workers/transform-child.d.ts +0 -1
  70. package/dist/transform/workers/transform-child.js +0 -1
  71. package/dist/transform/workers/transform-worker.d.ts +0 -1
  72. package/dist/transform/workers/transform-worker.js +0 -1
  73. package/package.json +6 -3
  74. package/dist/cli.d.ts.map +0 -1
  75. package/dist/cli.js.map +0 -1
  76. package/dist/http/auth.d.ts.map +0 -1
  77. package/dist/http/auth.js.map +0 -1
  78. package/dist/http/health.d.ts.map +0 -1
  79. package/dist/http/health.js.map +0 -1
  80. package/dist/http/helpers.d.ts.map +0 -1
  81. package/dist/http/helpers.js.map +0 -1
  82. package/dist/http/native.d.ts.map +0 -1
  83. package/dist/http/native.js.map +0 -1
  84. package/dist/http/rate-limit.d.ts.map +0 -1
  85. package/dist/http/rate-limit.js.map +0 -1
  86. package/dist/index.d.ts.map +0 -1
  87. package/dist/index.js.map +0 -1
  88. package/dist/lib/cache.d.ts +0 -54
  89. package/dist/lib/cache.d.ts.map +0 -1
  90. package/dist/lib/cache.js +0 -264
  91. package/dist/lib/cache.js.map +0 -1
  92. package/dist/lib/config.d.ts +0 -143
  93. package/dist/lib/config.d.ts.map +0 -1
  94. package/dist/lib/config.js +0 -476
  95. package/dist/lib/config.js.map +0 -1
  96. package/dist/lib/crypto.d.ts +0 -4
  97. package/dist/lib/crypto.d.ts.map +0 -1
  98. package/dist/lib/crypto.js +0 -56
  99. package/dist/lib/crypto.js.map +0 -1
  100. package/dist/lib/dom-noise-removal.d.ts +0 -2
  101. package/dist/lib/dom-noise-removal.d.ts.map +0 -1
  102. package/dist/lib/dom-noise-removal.js +0 -494
  103. package/dist/lib/dom-noise-removal.js.map +0 -1
  104. package/dist/lib/download.d.ts +0 -4
  105. package/dist/lib/download.d.ts.map +0 -1
  106. package/dist/lib/download.js +0 -106
  107. package/dist/lib/download.js.map +0 -1
  108. package/dist/lib/errors.d.ts +0 -14
  109. package/dist/lib/errors.d.ts.map +0 -1
  110. package/dist/lib/errors.js +0 -72
  111. package/dist/lib/errors.js.map +0 -1
  112. package/dist/lib/fetch-content.d.ts +0 -5
  113. package/dist/lib/fetch-content.d.ts.map +0 -1
  114. package/dist/lib/fetch-content.js +0 -164
  115. package/dist/lib/fetch-content.js.map +0 -1
  116. package/dist/lib/fetch-stream.d.ts +0 -5
  117. package/dist/lib/fetch-stream.d.ts.map +0 -1
  118. package/dist/lib/fetch-stream.js +0 -29
  119. package/dist/lib/fetch-stream.js.map +0 -1
  120. package/dist/lib/fetch.d.ts.map +0 -1
  121. package/dist/lib/fetch.js.map +0 -1
  122. package/dist/lib/host-normalization.d.ts +0 -2
  123. package/dist/lib/host-normalization.d.ts.map +0 -1
  124. package/dist/lib/host-normalization.js +0 -91
  125. package/dist/lib/host-normalization.js.map +0 -1
  126. package/dist/lib/ip-blocklist.d.ts +0 -9
  127. package/dist/lib/ip-blocklist.d.ts.map +0 -1
  128. package/dist/lib/ip-blocklist.js +0 -79
  129. package/dist/lib/ip-blocklist.js.map +0 -1
  130. package/dist/lib/json.d.ts +0 -2
  131. package/dist/lib/json.d.ts.map +0 -1
  132. package/dist/lib/json.js +0 -45
  133. package/dist/lib/json.js.map +0 -1
  134. package/dist/lib/language-detection.d.ts +0 -3
  135. package/dist/lib/language-detection.d.ts.map +0 -1
  136. package/dist/lib/language-detection.js +0 -355
  137. package/dist/lib/language-detection.js.map +0 -1
  138. package/dist/lib/markdown-cleanup.d.ts.map +0 -1
  139. package/dist/lib/markdown-cleanup.js +0 -532
  140. package/dist/lib/markdown-cleanup.js.map +0 -1
  141. package/dist/lib/mcp-lifecycle.d.ts +0 -5
  142. package/dist/lib/mcp-lifecycle.d.ts.map +0 -1
  143. package/dist/lib/mcp-lifecycle.js +0 -51
  144. package/dist/lib/mcp-lifecycle.js.map +0 -1
  145. package/dist/lib/mcp-validator.d.ts +0 -17
  146. package/dist/lib/mcp-validator.d.ts.map +0 -1
  147. package/dist/lib/mcp-validator.js +0 -45
  148. package/dist/lib/mcp-validator.js.map +0 -1
  149. package/dist/lib/mcp.d.ts +0 -4
  150. package/dist/lib/mcp.d.ts.map +0 -1
  151. package/dist/lib/mcp.js.map +0 -1
  152. package/dist/lib/observability.d.ts +0 -23
  153. package/dist/lib/observability.d.ts.map +0 -1
  154. package/dist/lib/observability.js +0 -238
  155. package/dist/lib/observability.js.map +0 -1
  156. package/dist/lib/server-tuning.d.ts +0 -15
  157. package/dist/lib/server-tuning.d.ts.map +0 -1
  158. package/dist/lib/server-tuning.js +0 -49
  159. package/dist/lib/server-tuning.js.map +0 -1
  160. package/dist/lib/session.d.ts +0 -45
  161. package/dist/lib/session.d.ts.map +0 -1
  162. package/dist/lib/session.js +0 -263
  163. package/dist/lib/session.js.map +0 -1
  164. package/dist/lib/timer-utils.d.ts +0 -13
  165. package/dist/lib/timer-utils.d.ts.map +0 -1
  166. package/dist/lib/timer-utils.js +0 -44
  167. package/dist/lib/timer-utils.js.map +0 -1
  168. package/dist/lib/tool-errors.d.ts +0 -12
  169. package/dist/lib/tool-errors.d.ts.map +0 -1
  170. package/dist/lib/tool-errors.js +0 -55
  171. package/dist/lib/tool-errors.js.map +0 -1
  172. package/dist/lib/tool-pipeline.d.ts.map +0 -1
  173. package/dist/lib/tool-pipeline.js.map +0 -1
  174. package/dist/lib/tool-progress.d.ts.map +0 -1
  175. package/dist/lib/tool-progress.js.map +0 -1
  176. package/dist/lib/type-guards.d.ts +0 -16
  177. package/dist/lib/type-guards.d.ts.map +0 -1
  178. package/dist/lib/type-guards.js +0 -13
  179. package/dist/lib/type-guards.js.map +0 -1
  180. package/dist/prompts/index.d.ts.map +0 -1
  181. package/dist/prompts/index.js.map +0 -1
  182. package/dist/resources/index.d.ts.map +0 -1
  183. package/dist/resources/index.js.map +0 -1
  184. package/dist/resources/instructions.d.ts.map +0 -1
  185. package/dist/resources/instructions.js.map +0 -1
  186. package/dist/schemas/inputs.d.ts.map +0 -1
  187. package/dist/schemas/inputs.js.map +0 -1
  188. package/dist/schemas/outputs.d.ts.map +0 -1
  189. package/dist/schemas/outputs.js.map +0 -1
  190. package/dist/server.d.ts.map +0 -1
  191. package/dist/server.js.map +0 -1
  192. package/dist/tasks/execution.d.ts.map +0 -1
  193. package/dist/tasks/execution.js.map +0 -1
  194. package/dist/tasks/manager.d.ts.map +0 -1
  195. package/dist/tasks/manager.js.map +0 -1
  196. package/dist/tasks/owner.d.ts.map +0 -1
  197. package/dist/tasks/owner.js.map +0 -1
  198. package/dist/tasks/tool-registry.d.ts.map +0 -1
  199. package/dist/tasks/tool-registry.js.map +0 -1
  200. package/dist/tools/fetch-url.d.ts.map +0 -1
  201. package/dist/tools/fetch-url.js.map +0 -1
  202. package/dist/tools/index.d.ts.map +0 -1
  203. package/dist/tools/index.js.map +0 -1
  204. package/dist/transform/transform.d.ts.map +0 -1
  205. package/dist/transform/transform.js.map +0 -1
  206. package/dist/transform/types.d.ts.map +0 -1
  207. package/dist/transform/types.js.map +0 -1
  208. package/dist/transform/worker-pool.d.ts.map +0 -1
  209. package/dist/transform/worker-pool.js.map +0 -1
  210. package/dist/transform/workers/shared.d.ts.map +0 -1
  211. package/dist/transform/workers/shared.js.map +0 -1
  212. package/dist/transform/workers/transform-child.d.ts.map +0 -1
  213. package/dist/transform/workers/transform-child.js.map +0 -1
  214. package/dist/transform/workers/transform-worker.d.ts.map +0 -1
  215. package/dist/transform/workers/transform-worker.js.map +0 -1
@@ -3,15 +3,16 @@ import diagnosticsChannel from 'node:diagnostics_channel';
3
3
  import { performance } from 'node:perf_hooks';
4
4
  import { isProbablyReaderable, Readability } from '@mozilla/readability';
5
5
  import { parseHTML } from 'linkedom';
6
- import { NodeHtmlMarkdown, } from 'node-html-markdown';
7
- import { config } from '../lib/config.js';
8
- import { removeNoiseFromHtml } from '../lib/dom-noise-removal.js';
9
- import { FetchError, getErrorMessage } from '../lib/errors.js';
10
- import { isRawTextContentUrl } from '../lib/fetch.js';
11
- import { detectLanguageFromCode, resolveLanguageFromAttributes, } from '../lib/language-detection.js';
12
- import { addSourceToMarkdown, buildMetadataFooter, cleanupMarkdownArtifacts, extractTitleFromRawMarkdown, isRawTextContent, } from '../lib/markdown-cleanup.js';
13
- import { getOperationId, getRequestId, logDebug, logError, logInfo, logWarn, redactUrl, } from '../lib/observability.js';
14
- import { isLikeNode, isObject } from '../lib/type-guards.js';
6
+ import { removeNoiseFromHtml } from '../lib/content.js';
7
+ import { addSourceToMarkdown, buildMetadataFooter, cleanupMarkdownArtifacts, extractTitleFromRawMarkdown, isRawTextContent, } from '../lib/content.js';
8
+ import { config } from '../lib/core.js';
9
+ import { getOperationId, getRequestId, logDebug, logError, logInfo, logWarn, redactUrl, } from '../lib/core.js';
10
+ import { isRawTextContentUrl } from '../lib/http.js';
11
+ import { createAbortError, throwIfAborted } from '../lib/utils.js';
12
+ import { FetchError, getErrorMessage } from '../lib/utils.js';
13
+ import { isObject } from '../lib/utils.js';
14
+ import { translateHtmlFragmentToMarkdown } from './html-translators.js';
15
+ import { extractMetadata, extractMetadataFromHead, mergeMetadata, } from './metadata.js';
15
16
  import { getOrCreateWorkerPool, getWorkerPoolStats, shutdownWorkerPool, } from './worker-pool.js';
16
17
  const utf8Decoder = new TextDecoder('utf-8');
17
18
  function decodeInput(input, encoding) {
@@ -30,43 +31,9 @@ function decodeInput(input, encoding) {
30
31
  return utf8Decoder.decode(input);
31
32
  }
32
33
  }
33
- function getTagName(node) {
34
- if (!isLikeNode(node))
35
- return '';
36
- const raw = node.tagName;
37
- return typeof raw === 'string' ? raw.toUpperCase() : '';
38
- }
39
34
  function asError(value) {
40
35
  return value instanceof Error ? value : undefined;
41
36
  }
42
- function getAbortReason(signal) {
43
- const record = isObject(signal) ? signal : null;
44
- return record && 'reason' in record ? record['reason'] : undefined;
45
- }
46
- function isTimeoutAbortReason(reason) {
47
- return reason instanceof Error && reason.name === 'TimeoutError';
48
- }
49
- function throwIfAborted(signal, url, stage) {
50
- if (!signal?.aborted)
51
- return;
52
- const reason = getAbortReason(signal);
53
- if (isTimeoutAbortReason(reason)) {
54
- throw new FetchError('Request timeout', url, 504, {
55
- reason: 'timeout',
56
- stage,
57
- });
58
- }
59
- throw new FetchError('Request was canceled', url, 499, {
60
- reason: 'aborted',
61
- stage,
62
- });
63
- }
64
- function createAbortError(url, stage) {
65
- return new FetchError('Request was canceled', url, 499, {
66
- reason: 'aborted',
67
- stage,
68
- });
69
- }
70
37
  const abortPolicy = { throwIfAborted, createAbortError };
71
38
  function buildTransformSignal(signal) {
72
39
  const { timeoutMs } = config.transform;
@@ -213,9 +180,20 @@ function trimUtf8Buffer(buffer, maxBytes) {
213
180
  function trimDanglingTagFragment(content) {
214
181
  const lastOpen = content.lastIndexOf('<');
215
182
  const lastClose = content.lastIndexOf('>');
216
- if (lastOpen > lastClose &&
217
- /^<([a-zA-Z/!?]|$)/.test(content.substring(lastOpen))) {
218
- return content.substring(0, lastOpen);
183
+ if (lastOpen > lastClose) {
184
+ if (lastOpen === content.length - 1) {
185
+ return content.substring(0, lastOpen);
186
+ }
187
+ const code = content.codePointAt(lastOpen + 1);
188
+ if (code !== undefined &&
189
+ (code === 47 || // '/'
190
+ code === 33 || // '!'
191
+ code === 63 || // '?'
192
+ (code >= 65 && code <= 90) || // A-Z
193
+ (code >= 97 && code <= 122)) // a-z
194
+ ) {
195
+ return content.substring(0, lastOpen);
196
+ }
219
197
  }
220
198
  return content;
221
199
  }
@@ -245,179 +223,6 @@ function willTruncate(html) {
245
223
  const maxSize = config.constants.maxHtmlSize;
246
224
  return (maxSize > 0 && (html.length > maxSize || getUtf8ByteLength(html) > maxSize));
247
225
  }
248
- const HEAD_END_PATTERN = /<\/head\s*>|<body\b/i;
249
- const MAX_HEAD_SCAN_LENGTH = 50_000;
250
- function extractHeadSection(html) {
251
- if (html.length <= MAX_HEAD_SCAN_LENGTH) {
252
- const match = HEAD_END_PATTERN.exec(html);
253
- return match ? html.substring(0, match.index) : null;
254
- }
255
- const searchText = html.substring(0, MAX_HEAD_SCAN_LENGTH);
256
- const match = HEAD_END_PATTERN.exec(searchText);
257
- if (!match)
258
- return null;
259
- return html.substring(0, match.index);
260
- }
261
- function extractMetadataFromHead(html, baseUrl) {
262
- const headSection = extractHeadSection(html);
263
- if (!headSection)
264
- return null;
265
- try {
266
- const { document } = parseHTML(`<!DOCTYPE html><html>${headSection}</head><body></body></html>`);
267
- return extractMetadata(document, baseUrl);
268
- }
269
- catch {
270
- return null;
271
- }
272
- }
273
- function mergeMetadata(early, late) {
274
- if (!early)
275
- return late;
276
- const merged = {};
277
- const keys = [
278
- 'title',
279
- 'description',
280
- 'author',
281
- 'image',
282
- 'publishedAt',
283
- 'modifiedAt',
284
- ];
285
- for (const key of keys) {
286
- const value = late[key] ?? early[key];
287
- if (value !== undefined)
288
- merged[key] = value;
289
- }
290
- return merged;
291
- }
292
- const META_PROPERTY_HANDLERS = new Map([
293
- [
294
- 'og:title',
295
- (ctx, c) => {
296
- ctx.title.og = c;
297
- },
298
- ],
299
- [
300
- 'og:description',
301
- (ctx, c) => {
302
- ctx.description.og = c;
303
- },
304
- ],
305
- [
306
- 'og:image',
307
- (ctx, c) => {
308
- ctx.image = c;
309
- },
310
- ],
311
- [
312
- 'article:published_time',
313
- (ctx, c) => {
314
- ctx.publishedAt = c;
315
- },
316
- ],
317
- [
318
- 'article:modified_time',
319
- (ctx, c) => {
320
- ctx.modifiedAt = c;
321
- },
322
- ],
323
- ]);
324
- const META_NAME_HANDLERS = new Map([
325
- [
326
- 'twitter:title',
327
- (ctx, c) => {
328
- ctx.title.twitter = c;
329
- },
330
- ],
331
- [
332
- 'twitter:description',
333
- (ctx, c) => {
334
- ctx.description.twitter = c;
335
- },
336
- ],
337
- [
338
- 'description',
339
- (ctx, c) => {
340
- ctx.description.standard = c;
341
- },
342
- ],
343
- [
344
- 'author',
345
- (ctx, c) => {
346
- ctx.author = c;
347
- },
348
- ],
349
- ]);
350
- function processMetaTag(ctx, tag) {
351
- const content = tag.getAttribute('content')?.trim();
352
- if (!content)
353
- return;
354
- const property = tag.getAttribute('property');
355
- if (property)
356
- META_PROPERTY_HANDLERS.get(property)?.(ctx, content);
357
- const name = tag.getAttribute('name');
358
- if (name)
359
- META_NAME_HANDLERS.get(name)?.(ctx, content);
360
- }
361
- function buildMetaContext(document) {
362
- const ctx = { title: {}, description: {} };
363
- for (const tag of document.querySelectorAll('meta')) {
364
- processMetaTag(ctx, tag);
365
- }
366
- const titleEl = document.querySelector('title');
367
- if (!ctx.title.standard && titleEl?.textContent) {
368
- ctx.title.standard = titleEl.textContent.trim();
369
- }
370
- return ctx;
371
- }
372
- function resolveMetadataFromContext(ctx) {
373
- const metadata = {};
374
- const resolvedTitle = ctx.title.og ?? ctx.title.twitter ?? ctx.title.standard;
375
- const resolvedDesc = ctx.description.og ?? ctx.description.twitter ?? ctx.description.standard;
376
- if (resolvedTitle)
377
- metadata.title = resolvedTitle;
378
- if (resolvedDesc)
379
- metadata.description = resolvedDesc;
380
- if (ctx.author)
381
- metadata.author = ctx.author;
382
- if (ctx.image)
383
- metadata.image = ctx.image;
384
- if (ctx.publishedAt)
385
- metadata.publishedAt = ctx.publishedAt;
386
- if (ctx.modifiedAt)
387
- metadata.modifiedAt = ctx.modifiedAt;
388
- return metadata;
389
- }
390
- function extractMetadata(document, baseUrl) {
391
- const ctx = buildMetaContext(document);
392
- const metadata = resolveMetadataFromContext(ctx);
393
- if (baseUrl) {
394
- const icon32 = document.querySelector('link[rel="icon"][sizes="32x32"]');
395
- const href = icon32?.getAttribute('href');
396
- if (href) {
397
- const resolved = resolveFaviconUrl(href, baseUrl);
398
- if (resolved)
399
- metadata.favicon = resolved;
400
- }
401
- }
402
- return metadata;
403
- }
404
- function resolveFaviconUrl(href, baseUrl) {
405
- const trimmed = href.trim();
406
- if (!trimmed)
407
- return undefined;
408
- if (trimmed.toLowerCase().startsWith('data:'))
409
- return undefined;
410
- try {
411
- const resolved = new URL(trimmed, baseUrl);
412
- if (resolved.protocol !== 'http:' && resolved.protocol !== 'https:') {
413
- return undefined;
414
- }
415
- return resolved.toString();
416
- }
417
- catch {
418
- return undefined;
419
- }
420
- }
421
226
  function isReadabilityCompatible(doc) {
422
227
  if (!isObject(doc))
423
228
  return false;
@@ -587,427 +392,6 @@ export function extractContent(html, url, options = {
587
392
  const result = extractContentContext(html, url, options);
588
393
  return { article: result.article, metadata: result.metadata };
589
394
  }
590
- const CODE_BLOCK = {
591
- fence: '```',
592
- format: (code, language = '') => `\`\`\`${language}\n${code}\n\`\`\``,
593
- };
594
- function buildInlineCode(content) {
595
- const trimmed = content.trim();
596
- if (!trimmed)
597
- return '``';
598
- let maxBackticks = 0;
599
- let currentRun = 0;
600
- for (const char of trimmed) {
601
- if (char === '`')
602
- currentRun += 1;
603
- else {
604
- if (currentRun > maxBackticks)
605
- maxBackticks = currentRun;
606
- currentRun = 0;
607
- }
608
- }
609
- if (currentRun > maxBackticks)
610
- maxBackticks = currentRun;
611
- const delimiter = '`'.repeat(maxBackticks + 1);
612
- const padding = trimmed.startsWith('`') || trimmed.endsWith('`') ? ' ' : '';
613
- return `${delimiter}${padding}${trimmed}${padding}${delimiter}`;
614
- }
615
- function deriveAltFromImageUrl(src) {
616
- if (!src)
617
- return '';
618
- try {
619
- const isAbsolute = URL.canParse(src);
620
- let parsed = null;
621
- if (isAbsolute) {
622
- parsed = new URL(src);
623
- }
624
- else if (URL.canParse(src, 'http://localhost')) {
625
- parsed = new URL(src, 'http://localhost');
626
- }
627
- if (!parsed)
628
- return '';
629
- if (isAbsolute &&
630
- parsed.protocol !== 'http:' &&
631
- parsed.protocol !== 'https:') {
632
- return '';
633
- }
634
- const { pathname } = parsed;
635
- const segments = pathname.split('/');
636
- const filename = segments.pop() ?? '';
637
- if (!filename)
638
- return '';
639
- const dotIndex = filename.lastIndexOf('.');
640
- const name = dotIndex > 0 ? filename.slice(0, dotIndex) : filename;
641
- return name.replace(/[_-]+/g, ' ').trim();
642
- }
643
- catch {
644
- return '';
645
- }
646
- }
647
- function hasGetAttribute(value) {
648
- return (isObject(value) &&
649
- typeof value.getAttribute === 'function');
650
- }
651
- function isCodeBlock(parent) {
652
- const tagName = getTagName(parent);
653
- return tagName === 'PRE' || tagName === 'WRAPPED-PRE';
654
- }
655
- function resolveAttributeLanguage(node) {
656
- const getAttribute = hasGetAttribute(node)
657
- ? node.getAttribute.bind(node)
658
- : undefined;
659
- const className = getAttribute?.('class') ?? '';
660
- const dataLanguage = getAttribute?.('data-language') ?? '';
661
- return resolveLanguageFromAttributes(className, dataLanguage);
662
- }
663
- function findLanguageFromCodeChild(node) {
664
- if (!isLikeNode(node))
665
- return undefined;
666
- const childNodes = Array.from(node.childNodes ?? []);
667
- for (const child of childNodes) {
668
- if (!isLikeNode(child))
669
- continue;
670
- const raw = child.rawTagName;
671
- const tagName = typeof raw === 'string' ? raw.toUpperCase() : '';
672
- if (tagName === 'CODE')
673
- return resolveAttributeLanguage(child);
674
- }
675
- return undefined;
676
- }
677
- function createCodeBlockPostprocessor(language) {
678
- return ({ content }) => {
679
- const trimmed = content.trim();
680
- if (!trimmed)
681
- return '';
682
- const resolvedLanguage = language ?? detectLanguageFromCode(trimmed) ?? '';
683
- return CODE_BLOCK.format(trimmed, resolvedLanguage);
684
- };
685
- }
686
- function buildInlineCodeTranslator() {
687
- return {
688
- spaceIfRepeatingChar: true,
689
- noEscape: true,
690
- postprocess: ({ content }) => buildInlineCode(content),
691
- };
692
- }
693
- function buildCodeTranslator(ctx) {
694
- const inlineCodeTranslator = buildInlineCodeTranslator();
695
- if (!isObject(ctx))
696
- return inlineCodeTranslator;
697
- const { parent } = ctx;
698
- if (!isCodeBlock(parent))
699
- return inlineCodeTranslator;
700
- return { noEscape: true, preserveWhitespace: true };
701
- }
702
- function extractFirstSrcsetUrl(srcset) {
703
- const first = srcset.split(',')[0];
704
- if (!first)
705
- return '';
706
- return first.trim().split(/\s+/)[0] ?? '';
707
- }
708
- const LAZY_SRC_ATTRIBUTES = [
709
- 'data-src',
710
- 'data-lazy-src',
711
- 'data-original',
712
- 'data-srcset',
713
- ];
714
- function isDataUri(value) {
715
- return value.startsWith('data:');
716
- }
717
- function extractNonDataSrcsetUrl(value) {
718
- const url = extractFirstSrcsetUrl(value);
719
- return url && !isDataUri(url) ? url : undefined;
720
- }
721
- function resolveLazySrc(getAttribute) {
722
- for (const attr of LAZY_SRC_ATTRIBUTES) {
723
- const lazy = getAttribute(attr);
724
- if (!lazy || isDataUri(lazy))
725
- continue;
726
- if (attr === 'data-srcset') {
727
- const url = extractNonDataSrcsetUrl(lazy);
728
- if (url)
729
- return url;
730
- continue;
731
- }
732
- return lazy;
733
- }
734
- return undefined;
735
- }
736
- function resolveImageSrc(getAttribute) {
737
- if (!getAttribute)
738
- return '';
739
- const srcRaw = getAttribute('src') ?? '';
740
- if (srcRaw && !isDataUri(srcRaw))
741
- return srcRaw;
742
- // First check common lazy-loading attributes that may contain non-data URLs before falling back to the native srcset, as some sites use data URIs in lazy attributes while still providing valid URLs in srcset.
743
- const lazySrc = resolveLazySrc(getAttribute);
744
- if (lazySrc)
745
- return lazySrc;
746
- // If the src is a data URI or missing, check srcset for a valid URL. Some sites use srcset with data URIs in src and actual URLs in srcset for responsive images.
747
- const srcset = getAttribute('srcset');
748
- if (srcset) {
749
- const url = extractNonDataSrcsetUrl(srcset);
750
- if (url)
751
- return url;
752
- }
753
- // If the only available src is a data URI, we choose to omit it rather than include the raw data in the alt text or URL, as data URIs can be very long and are not useful in Markdown output.
754
- if (isDataUri(srcRaw))
755
- return '[data URI removed]';
756
- return '';
757
- }
758
- function buildImageTranslator(ctx) {
759
- if (!isObject(ctx))
760
- return { content: '' };
761
- const { node } = ctx;
762
- const getAttribute = hasGetAttribute(node)
763
- ? node.getAttribute.bind(node)
764
- : undefined;
765
- const src = resolveImageSrc(getAttribute);
766
- const existingAlt = getAttribute?.('alt') ?? '';
767
- const alt = existingAlt.trim() || deriveAltFromImageUrl(src);
768
- const markdown = `![${alt}](${src})`;
769
- return { content: markdown };
770
- }
771
- const GFM_ALERT_MAP = new Map([
772
- ['note', 'NOTE'],
773
- ['info', 'NOTE'],
774
- ['tip', 'TIP'],
775
- ['hint', 'TIP'],
776
- ['warning', 'WARNING'],
777
- ['warn', 'WARNING'],
778
- ['caution', 'CAUTION'],
779
- ['danger', 'CAUTION'],
780
- ['important', 'IMPORTANT'],
781
- ]);
782
- function resolveGfmAlertType(className) {
783
- const lower = className.toLowerCase();
784
- for (const [key, type] of GFM_ALERT_MAP) {
785
- if (lower.includes(key))
786
- return type;
787
- }
788
- return undefined;
789
- }
790
- function resolveDlNodeName(child) {
791
- if (!isLikeNode(child))
792
- return '';
793
- const raw = child.nodeName;
794
- return typeof raw === 'string' ? raw.toUpperCase() : '';
795
- }
796
- function resolveDlTextContent(child) {
797
- if (!isLikeNode(child))
798
- return '';
799
- const raw = child.textContent;
800
- return typeof raw === 'string' ? raw.trim() : '';
801
- }
802
- function buildDlChildFragment(child) {
803
- const nodeName = resolveDlNodeName(child);
804
- if (nodeName === 'DT')
805
- return `**${resolveDlTextContent(child)}**\n`;
806
- if (nodeName === 'DD')
807
- return `: ${resolveDlTextContent(child)}\n`;
808
- return null;
809
- }
810
- function hasComplexTableLayout(node) {
811
- if (!isLikeNode(node))
812
- return false;
813
- const innerHTML = typeof node.innerHTML === 'string' ? node.innerHTML : '';
814
- return /(?:colspan|rowspan)=["']?[2-9]/i.test(innerHTML);
815
- }
816
- function buildPreTranslator(ctx) {
817
- if (!isObject(ctx))
818
- return {};
819
- const { node } = ctx;
820
- const attributeLanguage = resolveAttributeLanguage(node) ?? findLanguageFromCodeChild(node);
821
- return {
822
- noEscape: true,
823
- preserveWhitespace: true,
824
- postprocess: createCodeBlockPostprocessor(attributeLanguage),
825
- };
826
- }
827
- function getNodeAttr(node) {
828
- if (!isLikeNode(node))
829
- return undefined;
830
- return typeof node.getAttribute === 'function'
831
- ? node.getAttribute.bind(node)
832
- : undefined;
833
- }
834
- function buildDivTranslator(ctx) {
835
- if (!isObject(ctx))
836
- return {};
837
- const { node } = ctx;
838
- const getAttribute = getNodeAttr(node);
839
- if (!getAttribute)
840
- return {};
841
- const className = getAttribute('class') ?? '';
842
- if (className.includes('mermaid')) {
843
- return {
844
- noEscape: true,
845
- preserveWhitespace: true,
846
- postprocess: ({ content }) => `\n\n\`\`\`mermaid\n${content.trim()}\n\`\`\`\n\n`,
847
- };
848
- }
849
- const isAdmonition = className.includes('admonition') ||
850
- className.includes('callout') ||
851
- className.includes('custom-block') ||
852
- getAttribute('role') === 'alert' ||
853
- /\b(note|tip|info|warning|danger|caution|important)\b/i.test(className);
854
- if (isAdmonition) {
855
- return {
856
- postprocess: ({ content }) => {
857
- const alertType = resolveGfmAlertType(className);
858
- const lines = content.trim().split('\n');
859
- const header = alertType ? `> [!${alertType}]\n` : '';
860
- return `\n\n${header}> ${lines.join('\n> ')}\n\n`;
861
- },
862
- };
863
- }
864
- if (!className.includes('type'))
865
- return {};
866
- return {
867
- postprocess: ({ content }) => {
868
- const lines = content.split('\n');
869
- const separated = [];
870
- for (let i = 0; i < lines.length; i++) {
871
- const line = lines[i] ?? '';
872
- const nextLine = i < lines.length - 1 ? (lines[i + 1] ?? '') : '';
873
- separated.push(line);
874
- if (line.trim() &&
875
- nextLine.trim() &&
876
- line.includes(':') &&
877
- nextLine.includes(':') &&
878
- !line.startsWith(' ') &&
879
- !nextLine.startsWith(' ')) {
880
- separated.push('');
881
- }
882
- }
883
- return separated.join('\n');
884
- },
885
- };
886
- }
887
- function buildSectionTranslator(ctx) {
888
- if (isObject(ctx)) {
889
- const { node } = ctx;
890
- const getAttribute = getNodeAttr(node);
891
- if (getAttribute?.('class')?.includes('tsd-member')) {
892
- return {
893
- postprocess: ({ content }) => `\n\n&nbsp;\n\n${content}\n\n`,
894
- };
895
- }
896
- }
897
- return {
898
- postprocess: ({ content }) => `\n\n${content}\n\n`,
899
- };
900
- }
901
- function buildSpanTranslator(ctx) {
902
- if (!isObject(ctx))
903
- return {};
904
- const { node } = ctx;
905
- const getAttribute = getNodeAttr(node);
906
- if (!getAttribute)
907
- return {};
908
- const dataAs = getAttribute('data-as') ?? '';
909
- if (dataAs === 'p') {
910
- return {
911
- postprocess: ({ content }) => `\n\n${content.trim()}\n\n`,
912
- };
913
- }
914
- return {};
915
- }
916
- function buildMermaidPreTranslator(ctx) {
917
- if (!isObject(ctx))
918
- return buildPreTranslator(ctx);
919
- const { node } = ctx;
920
- const getAttribute = getNodeAttr(node);
921
- if (!getAttribute)
922
- return buildPreTranslator(ctx);
923
- const className = getAttribute('class') ?? '';
924
- if (className.includes('mermaid')) {
925
- return {
926
- noEscape: true,
927
- preserveWhitespace: true,
928
- postprocess: ({ content }) => `\n\n\`\`\`mermaid\n${content.trim()}\n\`\`\`\n\n`,
929
- };
930
- }
931
- return buildPreTranslator(ctx);
932
- }
933
- function createCustomTranslators() {
934
- return {
935
- code: (ctx) => buildCodeTranslator(ctx),
936
- img: (ctx) => buildImageTranslator(ctx),
937
- table: (ctx) => {
938
- if (!isObject(ctx))
939
- return {};
940
- const { node } = ctx;
941
- if (hasComplexTableLayout(node)) {
942
- return {
943
- postprocess: ({ content }) => {
944
- const trimmed = content.trim();
945
- if (!trimmed)
946
- return '';
947
- return `\n\n${trimmed}\n\n`;
948
- },
949
- };
950
- }
951
- return {};
952
- },
953
- dl: (ctx) => {
954
- if (!isObject(ctx))
955
- return { content: '' };
956
- const { node } = ctx;
957
- if (!isLikeNode(node))
958
- return { content: '' };
959
- const childNodes = Array.from(node.childNodes ?? []);
960
- let items = '';
961
- for (const child of childNodes) {
962
- const fragment = buildDlChildFragment(child);
963
- if (fragment !== null)
964
- items += fragment;
965
- }
966
- return { content: items ? `\n${items}\n` : '' };
967
- },
968
- div: buildDivTranslator,
969
- kbd: () => ({
970
- postprocess: ({ content }) => `\`${content}\``,
971
- }),
972
- mark: () => ({
973
- postprocess: ({ content }) => `==${content}==`,
974
- }),
975
- sub: () => ({
976
- postprocess: ({ content }) => `~${content}~`,
977
- }),
978
- sup: () => ({
979
- postprocess: ({ content }) => `^${content}^`,
980
- }),
981
- section: buildSectionTranslator,
982
- details: () => ({
983
- postprocess: ({ content }) => {
984
- const trimmed = content.trim();
985
- if (!trimmed)
986
- return '';
987
- return `\n\n${trimmed}\n\n`;
988
- },
989
- }),
990
- summary: () => ({
991
- postprocess: ({ content }) => `${content.trim()}\n\n`,
992
- }),
993
- span: buildSpanTranslator,
994
- pre: buildMermaidPreTranslator,
995
- };
996
- }
997
- let markdownConverter = null;
998
- function getMarkdownConverter() {
999
- markdownConverter ??= new NodeHtmlMarkdown({
1000
- codeFence: CODE_BLOCK.fence,
1001
- codeBlockStyle: 'fenced',
1002
- emDelimiter: '_',
1003
- bulletMarker: '-',
1004
- globalEscape: [/[\\`*_~]/gm, '\\$&'],
1005
- }, createCustomTranslators());
1006
- return markdownConverter;
1007
- }
1008
- function translateHtmlFragmentToMarkdown(html) {
1009
- return getMarkdownConverter().translate(html).trim();
1010
- }
1011
395
  function isWhitespaceChar(code) {
1012
396
  return code === 9 || code === 10 || code === 12 || code === 13 || code === 32;
1013
397
  }
@@ -1701,4 +1085,3 @@ export async function transformHtmlToMarkdown(html, url, options) {
1701
1085
  export async function transformBufferToMarkdown(htmlBuffer, url, options) {
1702
1086
  return transformInputToMarkdown(htmlBuffer, url, options);
1703
1087
  }
1704
- //# sourceMappingURL=transform.js.map