@page2ai/core 0.1.11 → 0.1.13

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -8,15 +8,40 @@
8
8
  // v0.2 will replace this with the full lib/core/html-to-md.ts pipeline via
9
9
  // PageAdapter, once the browser renderer is refactored to accept an adapter.
10
10
  import { CORE_VERSION } from '../shared/constants.js';
11
- import { absUrl, chooseCodeFence, cleanInline, escapeMd, escapeMdTableCell } from '../shared/utils.js';
12
- const SKIP_TAGS = new Set(['SCRIPT', 'STYLE', 'NOSCRIPT', 'TEMPLATE', 'SVG', 'CANVAS']);
11
+ import { absUrl, chooseCodeFence, cleanInline, escapeMd, longestRun } from '../shared/utils.js';
12
+ // HEAD/TITLE: the root can be documentElement (see usableBody). The form widgets and embeds hold
13
+ // option lists and "your browser does not support" fallbacks, never page text; they used to stay
14
+ // out only because the walk dropped all bare text (P2A-001), and must not arrive with the fix.
15
+ const SKIP_TAGS = new Set([
16
+ 'SCRIPT', 'STYLE', 'NOSCRIPT', 'TEMPLATE', 'SVG', 'CANVAS', 'HEAD', 'TITLE',
17
+ 'SELECT', 'DATALIST', 'TEXTAREA', 'IFRAME', 'OBJECT', 'EMBED', 'AUDIO', 'VIDEO',
18
+ ]);
13
19
  const INLINE_TAGS = new Set(['A', 'B', 'STRONG', 'I', 'EM', 'CODE', 'SPAN', 'SMALL', 'MARK', 'U', 'S', 'DEL', 'INS', 'SUB', 'SUP', 'ABBR', 'CITE', 'Q', 'DFN', 'KBD', 'SAMP', 'VAR', 'TIME']);
20
+ // Phrasing-level tags. Sitting directly in a container (`<div><span>Plan</span> starter</div>`)
21
+ // they join the text around them into one paragraph instead of being walked as containers.
22
+ const PHRASING_TAGS = new Set([
23
+ ...INLINE_TAGS, 'IMG', 'BR', 'WBR', 'BUTTON', 'LABEL', 'OUTPUT', 'DATA', 'BDI', 'BDO', 'FONT',
24
+ 'BIG', 'TT', 'STRIKE', 'ACRONYM', 'NOBR', 'RUBY', 'RT', 'RP', 'METER', 'PROGRESS', 'INPUT',
25
+ ]);
26
+ // A run made of nothing but these renders one element per line, as it did before runs existed.
27
+ const RUN_ATOMS = new Set(['A', 'IMG', 'BR']);
28
+ // Block-level tags met in INLINE position (a card <div> inside a link, a styling <div> folded into
29
+ // an <li>): the browser starts a new line for them, so their text is spaced from its neighbours.
30
+ const INLINE_BLOCK_GAP = new Set([
31
+ 'ADDRESS', 'ARTICLE', 'ASIDE', 'BLOCKQUOTE', 'DD', 'DETAILS', 'DIALOG', 'DIV', 'DL', 'DT',
32
+ 'FIELDSET', 'FIGCAPTION', 'FIGURE', 'FOOTER', 'FORM', 'H1', 'H2', 'H3', 'H4', 'H5', 'H6',
33
+ 'HEADER', 'HR', 'LI', 'MAIN', 'NAV', 'OL', 'P', 'PRE', 'SECTION', 'SUMMARY', 'TABLE', 'TBODY',
34
+ 'TD', 'TFOOT', 'TH', 'THEAD', 'TR', 'UL',
35
+ ]);
14
36
  const KNOWN_LANGS = new Set(['bash', 'sh', 'shell', 'zsh', 'console', 'shellsession', 'powershell', 'ps1', 'bat', 'cmd', 'python', 'py', 'javascript', 'js', 'typescript', 'ts', 'tsx', 'jsx', 'json', 'jsonc', 'yaml', 'yml', 'toml', 'ini', 'html', 'xml', 'css', 'scss', 'less', 'sass', 'sql', 'graphql', 'go', 'rust', 'java', 'kotlin', 'swift', 'c', 'cpp', 'csharp', 'php', 'ruby', 'r', 'lua', 'perl', 'dart', 'scala', 'elixir', 'haskell', 'markdown', 'md', 'mdx', 'mermaid', 'dockerfile', 'docker', 'nginx', 'diff', 'http', 'proto', 'tex', 'latex', 'vue', 'svelte', 'astro', 'plaintext', 'text', 'txt']);
15
37
  const yq = (s) => '"' + String(s).replace(/"/g, '\\"') + '"';
38
+ // The page's own <title>. querySelector('title') also finds an inline icon's <svg><title>, which became
39
+ // a fake `# IconName` heading on any page or fragment whose <head> has no title (grok, T109).
40
+ export const documentTitle = (doc) => [...doc.querySelectorAll('title')].find((t) => !t.closest('svg'))?.textContent || '';
16
41
  export const renderBasicMarkdown = (doc, adapter, opts) => {
17
42
  const parts = [];
18
- const root = pickContentRoot(doc);
19
- const rawTitle = cleanInline(doc.querySelector('title')?.textContent || '');
43
+ const root = pickContentRoot(doc, opts);
44
+ const rawTitle = cleanInline(documentTitle(doc));
20
45
  // The site suffix is dropped, not relocated: an earlier draft emitted it as a `site_name:`
21
46
  // frontmatter field, which put the exact chrome string the cleanup had just removed back
22
47
  // into the output one line lower. Provenance already lives in `source:`.
@@ -27,14 +52,19 @@ export const renderBasicMarkdown = (doc, adapter, opts) => {
27
52
  if (title)
28
53
  parts.push(`# ${title}`, '');
29
54
  if (root && root.children) {
30
- const tabGroups = discoverTabGroups(root);
55
+ const tabGroups = discoverTabGroups(root, opts);
31
56
  const capturedPanelEls = new WeakSet();
32
57
  tabGroups.forEach((g) => g.panels.forEach((p) => capturedPanelEls.add(p.el)));
33
58
  const lines = [];
34
59
  void adapter;
35
60
  const dedupe = { pendingTitle: title ? squeezeText(title) : '' };
36
- walkWithTabs(root, lines, 0, opts, tabGroups, capturedPanelEls, dedupe);
37
- parts.push(...lines);
61
+ walkWithTabs(root, lines, opts, tabGroups, capturedPanelEls, dedupe);
62
+ // An empty walk is said out loud: a bare "" or a lone title reads as "the page is empty"
63
+ // when the cause is client-side rendering or markup the walk did not understand (P2A-003).
64
+ if (lines.some((l) => l.trim()))
65
+ parts.push(...lines);
66
+ else
67
+ parts.push('', '<!-- AI: page has no extractable body content -->', '');
38
68
  }
39
69
  else {
40
70
  parts.push('', '<!-- AI: page has no extractable body content -->', '');
@@ -118,7 +148,7 @@ const splitSiteSuffix = (doc, root, rawTitle) => {
118
148
  : first;
119
149
  };
120
150
  const buildFrontmatter = (doc, baseUrl, titleOverride) => {
121
- const title = titleOverride ?? cleanInline(doc.querySelector('title')?.textContent || '');
151
+ const title = titleOverride ?? cleanInline(documentTitle(doc));
122
152
  const description = cleanInline(doc.querySelector('meta[name="description"]')?.getAttribute('content') || '');
123
153
  const lang = doc.documentElement?.getAttribute('lang') || 'unknown';
124
154
  const canonical = absUrl(doc.querySelector('link[rel="canonical"]')?.getAttribute('href') || '', baseUrl);
@@ -225,11 +255,24 @@ const usableBody = (doc) => {
225
255
  return body;
226
256
  return doc.documentElement || body || null;
227
257
  };
228
- const pickContentRoot = (doc) => {
258
+ const pickContentRoot = (doc, opts) => {
229
259
  // The semantic container first. <main> is unique per document by spec, so unlike <article>
230
- // it cannot be a component.
231
- const semantic = doc.querySelector('main') ||
232
- doc.querySelector('[role="main"]');
260
+ // it cannot be a component. Unique among the SHOWN ones: a page may hold several <main> elements as long as all but
261
+ // one are hidden (HTML, "the main element"), and a hidden one became the root and printed its whole text (C7c, codex).
262
+ // So a <main> under no `hidden` at all comes first, one inside a React streaming segment only when there is none, and
263
+ // one under a hidden tab panel or `until-found` section never: those are shown as one part of the page, so as the
264
+ // root they printed only themselves and dropped the visible <main> (T124 panel: codex, grok, mimo, spark, agy).
265
+ const clearUp = (e, exempt) => {
266
+ for (let a = e; a; a = a.parentElement)
267
+ if (underHiddenAttr(a, opts) && !exempt(a))
268
+ return false;
269
+ return true;
270
+ };
271
+ const pick = (selector) => {
272
+ const all = [...doc.querySelectorAll(selector)];
273
+ return all.find((e) => clearUp(e, () => false)) || all.find((e) => clearUp(e, isReactSegment));
274
+ };
275
+ const semantic = pick('main') || pick('[role="main"]') || null;
233
276
  const container = semantic || usableBody(doc);
234
277
  if (!container)
235
278
  return null;
@@ -239,7 +282,9 @@ const pickContentRoot = (doc) => {
239
282
  // comments. On docs.astro.build the first <article> is `class="card sl-flex"` holding 155
240
283
  // characters while <main> holds 1,362, so the tool emitted a heading, one question and one
241
284
  // sentence for the whole page. Our own documentation site runs that framework.
285
+ // An <article> under a hidden ancestor is not a candidate: as the root it would print what the browser hides.
242
286
  const articles = Array.from(container.querySelectorAll('article'))
287
+ .filter((a) => !hiddenUpTo(a, container, opts))
243
288
  .map((a) => ({ el: a, n: contentTextLength(a) }))
244
289
  .sort((x, y) => y.n - x.n);
245
290
  if (articles.length) {
@@ -270,7 +315,7 @@ const pickContentRoot = (doc) => {
270
315
  // Static tab discovery: aria-controls chains, role="tablist" siblings, and
271
316
  // data-state attribute pairs. Emits '## Tab: {label}' sections. Does NOT
272
317
  // click — for Node we treat each panel as ALREADY present in the DOM.
273
- const discoverTabGroups = (root) => {
318
+ const discoverTabGroups = (root, opts) => {
274
319
  const groups = [];
275
320
  const seenContainers = new WeakSet();
276
321
  // Path 1: [role="tablist"] with aria-controls chain
@@ -280,7 +325,8 @@ const discoverTabGroups = (root) => {
280
325
  const tabs = [...tl.querySelectorAll('[role="tab"], button')];
281
326
  const panels = [];
282
327
  for (const tab of tabs) {
283
- const label = cleanInline(tab.textContent || '');
328
+ // A tab's label is its name, without the text of hidden elements (T124 panel: `One<span hidden>x</span>` read Onex).
329
+ const label = cleanInline(nameText(tab, opts));
284
330
  if (!label)
285
331
  continue;
286
332
  const controls = tab.getAttribute('aria-controls');
@@ -291,7 +337,9 @@ const discoverTabGroups = (root) => {
291
337
  if (!doc)
292
338
  continue;
293
339
  const panel = doc.getElementById(controls);
294
- if (!panel)
340
+ // A panel outside the content root is not part of it: a hidden block anywhere on the page was one aria-controls
341
+ // away from being printed (T124 panel: ocmimo).
342
+ if (!panel || !root.contains(panel))
295
343
  continue;
296
344
  panels.push({ label, el: panel });
297
345
  }
@@ -318,7 +366,7 @@ const discoverTabGroups = (root) => {
318
366
  const labelledText = labelledBy && doc
319
367
  ? labelledBy
320
368
  .split(/\s+/)
321
- .map((id) => doc.getElementById(id)?.textContent || '')
369
+ .map((id) => { const e = doc.getElementById(id); return e ? nameText(e, opts) : ''; })
322
370
  .join(' ')
323
371
  .replace(/\s+/g, ' ')
324
372
  .trim()
@@ -337,12 +385,68 @@ const discoverTabGroups = (root) => {
337
385
  });
338
386
  return groups;
339
387
  };
340
- const walkWithTabs = (el, out, depth, opts, tabGroups, capturedPanelEls, dedupe) => {
341
- for (const child of el.children) {
342
- renderNodeWithTabs(child, out, depth, opts, tabGroups, capturedPanelEls, dedupe);
388
+ const walkWithTabs = (el, out, opts, tabGroups, capturedPanelEls, dedupe) => walkNodes([...el.childNodes], out, opts, tabGroups, capturedPanelEls, dedupe);
389
+ const walkNodes = (nodes, out, opts, tabGroups, capturedPanelEls, dedupe) => {
390
+ // Child NODES, not element children. Text and phrasing elements sitting directly in a container
391
+ // collect into a run that renders as one paragraph, flushed before each block child. The walk
392
+ // used to iterate `el.children`, so text outside a recognised block tag vanished without a
393
+ // trace: `<div>Hello</div>`, a metric tile's label and value, text beside a `<br>`, a section's
394
+ // own sentence (P2A-001: on a server-rendered admin page 25 of 26 key/value labels and 10 of 12
395
+ // tiles were lost while every heading and table survived, so the output looked complete).
396
+ let run = [];
397
+ const flush = () => {
398
+ if (!run.length)
399
+ return;
400
+ const nodes = run;
401
+ run = [];
402
+ const hasText = nodes.some((n) => n.nodeType === 3 && /\S/.test(n.textContent || ''));
403
+ // Links and images with no text between them (a nav row, an icon strip) keep the old
404
+ // one-per-line rendering, so a page without stray text renders exactly as before.
405
+ if (!hasText && nodes.every((n) => n.nodeType !== 1 || RUN_ATOMS.has(n.tagName))) {
406
+ for (const n of nodes) {
407
+ if (n.nodeType === 1)
408
+ renderNodeWithTabs(n, out, opts, tabGroups, capturedPanelEls, dedupe);
409
+ }
410
+ return;
411
+ }
412
+ // A tab panel nested inside a run element is rendered under its own `### Tab:` heading.
413
+ const text = escapeBlockStarts(renderInlineNodes(nodes, opts, capturedPanelEls).trim());
414
+ if (text)
415
+ out.push('', text, '');
416
+ };
417
+ // A custom element (`<relative-time>`, `<my-badge>`) has no user-agent style, so it is inline unless
418
+ // its own CSS says otherwise. Among the bare words of its parent it is part of the sentence
419
+ // ("opened Oct 3 · 5 comments", not three paragraphs); among element siblings only (a Material
420
+ // card's title and subtitle) it stays a block, which is how such components are usually styled.
421
+ const parentHasText = nodes.some((n) => n.nodeType === 3 && /\S/.test(n.textContent || ''));
422
+ for (const node of nodes) {
423
+ if (node.nodeType === 3 /* TEXT_NODE */) {
424
+ run.push(node);
425
+ continue;
426
+ }
427
+ if (node.nodeType !== 1 /* ELEMENT_NODE */)
428
+ continue;
429
+ const c = node;
430
+ // Skipped where they stand, without ending the run: renderNode would drop them anyway. A hidden block that holds code
431
+ // goes on to renderNode, which prints only its code.
432
+ if (SKIP_TAGS.has(c.tagName) || hiddenKind(c, opts) === 'hidden')
433
+ continue;
434
+ const phrasing = PHRASING_TAGS.has(c.tagName) || (parentHasText && c.tagName.includes('-'));
435
+ if (phrasing && !capturedPanelEls.has(c) && !tabGroups.some((g) => g.container === c)) {
436
+ if (isAriaHidden(c))
437
+ continue;
438
+ // A link wrapping a whole card (<a><div><h3>..</h3><p>..</p></div></a>) stays a block.
439
+ if (!hasBlockContent(c)) {
440
+ run.push(c);
441
+ continue;
442
+ }
443
+ }
444
+ flush();
445
+ renderNodeWithTabs(c, out, opts, tabGroups, capturedPanelEls, dedupe);
343
446
  }
447
+ flush();
344
448
  };
345
- const renderNodeWithTabs = (el, out, depth, opts, tabGroups, capturedPanelEls, dedupe) => {
449
+ const renderNodeWithTabs = (el, out, opts, tabGroups, capturedPanelEls, dedupe) => {
346
450
  // If THIS element is a captured tab panel, skip — it will be rendered as
347
451
  // part of its tab group section, not inline.
348
452
  if (capturedPanelEls.has(el))
@@ -352,22 +456,32 @@ const renderNodeWithTabs = (el, out, depth, opts, tabGroups, capturedPanelEls, d
352
456
  if (group) {
353
457
  for (const panel of group.panels) {
354
458
  out.push('', `### Tab: ${escapeMd(panel.label)}`, '');
355
- walkWithTabs(panel.el, out, depth, opts, tabGroups, capturedPanelEls, dedupe);
459
+ walkWithTabs(panel.el, out, opts, tabGroups, capturedPanelEls, dedupe);
356
460
  }
357
461
  return;
358
462
  }
359
- renderNode(el, out, depth, opts, tabGroups, capturedPanelEls, dedupe);
463
+ renderNode(el, out, opts, tabGroups, capturedPanelEls, dedupe);
360
464
  };
361
- const renderNode = (el, out, depth, opts, tabGroups, capturedPanelEls, dedupe) => {
465
+ // Every block renders flush left; a list item indents what it holds (renderListItem).
466
+ const renderNode = (el, out, opts, tabGroups, capturedPanelEls, dedupe) => {
362
467
  const tag = el.tagName;
363
468
  if (SKIP_TAGS.has(tag))
364
469
  return;
365
- if (el.getAttribute('aria-hidden') === 'true')
470
+ if (isAriaHidden(el))
471
+ return;
472
+ const hidden = hiddenKind(el, opts);
473
+ if (hidden === 'hidden')
366
474
  return;
367
- const pfx = ' '.repeat(depth);
475
+ if (hidden === 'code') {
476
+ for (const pre of outerPres(el, opts))
477
+ renderCodeBlock(pre, out, opts);
478
+ return;
479
+ }
368
480
  if (/^H[1-6]$/.test(tag)) {
369
481
  const level = Number(tag.slice(1));
370
- const text = renderInline(el, opts).trim();
482
+ // A heading is one line, so a <br> in it, in its text or its code, is a space (T124 panel: mimo,
483
+ // `<h2>Run <code>a<br>b</code> now</h2>` put the rest of the heading into a paragraph under it).
484
+ const text = renderInline(el, opts).replace(/\s*\n\s*/g, ' ').trim();
371
485
  if (text) {
372
486
  if (dedupe.pendingTitle) {
373
487
  // Match on textContent, not the rendered markdown: escaping and link syntax
@@ -382,27 +496,27 @@ const renderNode = (el, out, depth, opts, tabGroups, capturedPanelEls, dedupe) =
382
496
  return;
383
497
  }
384
498
  if (tag === 'PRE') {
385
- renderCodeBlock(el, out);
499
+ renderCodeBlock(el, out, opts);
386
500
  return;
387
501
  }
388
502
  if (tag === 'P') {
389
- const text = renderInline(el, opts).trim();
503
+ const text = escapeBlockStarts(renderInline(el, opts, capturedPanelEls).trim());
390
504
  if (text)
391
505
  out.push('', text, '');
392
506
  return;
393
507
  }
394
508
  if (tag === 'A') {
395
509
  const href = absUrl(el.getAttribute('href') || '', opts.baseUrl);
396
- const text = renderInline(el, opts).trim() || cleanInline(el.textContent || '');
510
+ const text = linkText(el, opts, href, capturedPanelEls);
397
511
  if (text && href)
398
- out.push(`${pfx}[${escapeMd(text)}](${href})`);
512
+ out.push(`[${text}](${mdUrl(href)})`);
399
513
  else if (text)
400
- out.push(`${pfx}${escapeMd(text)}`);
514
+ out.push(escapeBlockStart(text));
401
515
  return;
402
516
  }
403
517
  if (tag === 'BLOCKQUOTE') {
404
518
  const inner = [];
405
- walkWithTabs(el, inner, depth, opts, tabGroups, capturedPanelEls, dedupe);
519
+ walkWithTabs(el, inner, opts, tabGroups, capturedPanelEls, dedupe);
406
520
  const text = inner.join('\n').trim();
407
521
  if (text) {
408
522
  out.push('');
@@ -412,26 +526,30 @@ const renderNode = (el, out, depth, opts, tabGroups, capturedPanelEls, dedupe) =
412
526
  return;
413
527
  }
414
528
  if (tag === 'UL' || tag === 'OL') {
415
- const items = [...el.children].filter((c) => c.tagName === 'LI');
529
+ // A hidden item takes no number, as in a browser.
530
+ const items = [...el.children].filter((c) => c.tagName === 'LI' && !isHiddenEl(c, opts));
416
531
  let i = 1;
417
532
  for (const li of items) {
418
533
  const marker = tag === 'OL' ? `${i}.` : '-';
419
- renderListItem(li, marker, out, depth, pfx, opts, tabGroups, capturedPanelEls, dedupe);
534
+ renderListItem(li, marker, out, opts, tabGroups, capturedPanelEls, dedupe);
420
535
  i++;
421
536
  }
422
537
  return;
423
538
  }
424
539
  if (tag === 'TABLE') {
425
- renderTable(el, out);
540
+ renderTable(el, out, opts);
426
541
  return;
427
542
  }
543
+ // A <dl> without <dt>/<dd> is a layout box and falls through to the generic walk.
544
+ if (tag === 'DL' && renderDefinitionList(el, out, opts, tabGroups, capturedPanelEls, dedupe))
545
+ return;
428
546
  if (tag === 'IMG') {
429
547
  if (!opts.includeImages)
430
548
  return;
431
549
  const alt = cleanInline(el.getAttribute('alt') || '');
432
550
  const src = absUrl(el.getAttribute('src') || el.getAttribute('data-src') || '', opts.baseUrl);
433
551
  if (src)
434
- out.push(`${pfx}![${escapeMd(alt || 'image')}](${src})`);
552
+ out.push(`![${escapeMd(alt || 'image')}](${mdUrl(src)})`);
435
553
  return;
436
554
  }
437
555
  if (tag === 'HR') {
@@ -442,7 +560,7 @@ const renderNode = (el, out, depth, opts, tabGroups, capturedPanelEls, dedupe) =
442
560
  out.push('');
443
561
  return;
444
562
  }
445
- walkWithTabs(el, out, depth, opts, tabGroups, capturedPanelEls, dedupe);
563
+ walkWithTabs(el, out, opts, tabGroups, capturedPanelEls, dedupe);
446
564
  };
447
565
  // Block tags that, as direct children of an <li>, render below the bullet instead of being
448
566
  // flattened into its text. UL/OL were always here; the rest joined after the Sphinx measurement
@@ -457,46 +575,88 @@ const hasBlockContent = (el) => !!el.querySelector('p, div, pre, table, ul, ol,
457
575
  // authored. The first approach rendered all inline text first and all blocks after it, which
458
576
  // reordered `<li>Text1 <pre>..</pre> Text2</li>` into Text1 Text2 pre — a reviewer-found
459
577
  // defect, fixed before it ever shipped.
460
- const renderListItem = (li, marker, out, depth, pfx, opts, tabGroups, capturedPanelEls, dedupe) => {
578
+ //
579
+ // Each block child renders flush left into its own buffer and is then indented under the marker,
580
+ // so a CommonMark parser keeps it inside the item (BL-37). Blocks used to start at column 0 after
581
+ // a blank line, which ends the list: the code of every "1. Install:" step left its list and the
582
+ // numbering restarted, an item STARTING with a block printed a bare `-` and a blank line (an
583
+ // empty item), and a nested list indented by depth alone did not nest under `10.`. A bare marker
584
+ // is never printed now: the first block line goes on the marker line (`- ```js`), because a bare
585
+ // `-` below a line of text is a setext underline and made that text a `##` heading.
586
+ const renderListItem = (li, marker, out, opts, tabGroups, capturedPanelEls, dedupe) => {
587
+ // The item's content starts one space after the marker. Later lines sit three spaces in, as
588
+ // they always have, or at the content column when the marker is wider (`10. ` needs four). The
589
+ // rest of a block opened on the marker line sits exactly at the content column: a fence opened
590
+ // there strips no indentation from its code, so any extra space would enter the code.
591
+ const exact = ' '.repeat(marker.length + 1);
592
+ const cont = ' '.repeat(Math.max(3, marker.length + 1));
461
593
  let emittedMarker = false;
594
+ // The last block ended on a content line (a nested list does): text after it needs a blank line,
595
+ // or it continues that list's last item.
596
+ let openBlock = false;
597
+ // A <p> is its own paragraph: without a blank line between them, `<li><p>A</p><p>B</p></li>` printed
598
+ // `- A` / ` B`, one paragraph "A B" to a parser (T110 panel: agy, grok; there before BL-37).
599
+ let prev = '';
462
600
  let run = [];
463
- const emitLine = (text) => {
464
- if (!text)
465
- return;
601
+ const emit = (lines, kind) => {
602
+ let i = 0;
603
+ let indent = cont;
466
604
  if (!emittedMarker) {
467
- out.push(`${pfx}${marker} ${text}`);
605
+ while (i < lines.length && !lines[i].trim())
606
+ i++;
607
+ if (i === lines.length)
608
+ return;
609
+ // `- ---` is a thematic break, not an item holding one (CommonMark 4.1: the break wins), so the item and
610
+ // the text after it left the list. A rule opening a bulleted item is written with stars (T110 panel: grok).
611
+ const first = lines[i++];
612
+ out.push(`${marker} ${marker === '-' && /^-{3,}$/.test(first) ? '***' : first}`);
468
613
  emittedMarker = true;
614
+ if (kind === 'block')
615
+ indent = exact;
469
616
  }
470
- else {
471
- out.push(`${pfx} ${text}`);
617
+ else if (out[out.length - 1] !== '' && (lines[0] ?? '').trim() !== '' &&
618
+ (openBlock || kind === 'para' || (kind === 'text' && prev === 'para'))) {
619
+ // After a nested list a block needs the blank line too: `- - a` / ` x` made `x` part of item `a`
620
+ // (T110 panel: mimo).
621
+ out.push('');
472
622
  }
623
+ for (; i < lines.length; i++)
624
+ out.push(lines[i] ? indent + lines[i] : '');
625
+ openBlock = kind === 'block' && lines[lines.length - 1].trim() !== '';
626
+ prev = kind;
627
+ };
628
+ const emitText = (raw, kind) => {
629
+ // `- 1. x` would be a list inside the item and `- # x` a heading inside it.
630
+ if (raw)
631
+ emit(escapeBlockStarts(raw).split('\n'), kind);
473
632
  };
474
633
  const flushInline = () => {
475
- const text = renderInlineNodes(run, opts).trim();
634
+ const text = renderInlineNodes(run, opts, capturedPanelEls).trim();
476
635
  run = [];
477
- emitLine(text);
636
+ emitText(text, 'text');
478
637
  };
479
638
  for (const node of li.childNodes) {
480
639
  if (node.nodeType === 1 /* ELEMENT_NODE */) {
481
640
  const c = node;
641
+ if (hiddenKind(c, opts) === 'hidden')
642
+ continue;
482
643
  if (c.tagName === 'P') {
483
644
  flushInline();
484
- emitLine(renderInline(c, opts).trim());
645
+ emitText(renderInline(c, opts, capturedPanelEls).trim(), 'para');
485
646
  continue;
486
647
  }
487
648
  if (LI_BLOCK_TAGS.has(c.tagName)) {
488
- // A DIV with no block structure of its own is a styling wrapper: fold its children
489
- // into the inline run so the bullet survives.
649
+ // A DIV with no block structure of its own is a styling wrapper: fold it into the
650
+ // inline run so the bullet survives. Pushed whole, not as its children, so two such
651
+ // DIVs stay apart (INLINE_BLOCK_GAP) now that inline parts are no longer space-joined.
490
652
  if (c.tagName === 'DIV' && !hasBlockContent(c)) {
491
- run.push(...c.childNodes);
653
+ run.push(c);
492
654
  continue;
493
655
  }
494
656
  flushInline();
495
- if (!emittedMarker) {
496
- out.push(`${pfx}${marker}`);
497
- emittedMarker = true;
498
- }
499
- renderNodeWithTabs(c, out, depth + 1, opts, tabGroups, capturedPanelEls, dedupe);
657
+ const inner = [];
658
+ renderNodeWithTabs(c, inner, opts, tabGroups, capturedPanelEls, dedupe);
659
+ emit(inner.join('\n').split('\n'), 'block');
500
660
  continue;
501
661
  }
502
662
  }
@@ -504,14 +664,252 @@ const renderListItem = (li, marker, out, depth, pfx, opts, tabGroups, capturedPa
504
664
  }
505
665
  flushInline();
506
666
  };
507
- const renderInline = (el, opts, excludeEls) => renderInlineNodes([...el.childNodes], opts, excludeEls);
667
+ const renderInline = (el, opts, excludeEls) => renderInlineNodes(hiddenKind(el, opts) === 'code' ? outerPres(el, opts) : [...el.childNodes], opts, excludeEls);
668
+ const isAriaHidden = (el) => (el.getAttribute('aria-hidden') || '').toLowerCase() === 'true';
669
+ const hiddenKind = (el, opts) => {
670
+ if (opts.includeHidden || !el.hasAttribute('hidden'))
671
+ return 'shown';
672
+ if ((el.getAttribute('hidden') || '').trim().toLowerCase() === 'until-found')
673
+ return 'shown';
674
+ // A tab panel is a block: all 27 on the corpus are <div>s, and `<a><span hidden role="tabpanel">` printed its text
675
+ // as the link's (T124 panel: mimo).
676
+ if ((el.getAttribute('role') === 'tabpanel' && !PHRASING_TAGS.has(el.tagName)) || el.tagName === 'PRE' || isReactSegment(el))
677
+ return 'shown';
678
+ return el.querySelector('pre') ? 'code' : 'hidden';
679
+ };
680
+ const isHiddenEl = (el, opts) => hiddenKind(el, opts) !== 'shown';
681
+ // Under the attribute at all, the kept kinds included. Code, a link's name, a tab's label and the page's <main> must not
682
+ // come from a hidden element of any kind (T124 panel: `<a><span hidden role="tabpanel">x</span></a>` was named x, and a
683
+ // <main> inside a hidden tab panel became the root and dropped the visible <main>).
684
+ const underHiddenAttr = (el, opts) => !opts.includeHidden && el.hasAttribute('hidden');
685
+ // React streaming segments as react-dom's server writes them (ReactFizzConfigDOM.js, startSegment*): `<div hidden id=…>`,
686
+ // in a table context `<table hidden id=…>` or a `<table hidden>` whose <tbody>, <tr> or <colgroup> carries the id. The id
687
+ // is the render's identifierPrefix, `S:` and the segment number in hex. A child's `S:` id counted on any hidden element
688
+ // (C14, grok), and so did any element and anything after `S:`: `<img hidden id="S:0">` printed its alt and `id="S:evil"`
689
+ // passed (T124 panel: codex, mimo). An HTML parser puts a <tbody> around `<table><tr>`, so a saved or re-serialised page
690
+ // has `<table hidden><tbody><tr id=…>` (grok).
691
+ const SEGMENT_ID = /S:[0-9a-f]+$/;
692
+ const isReactSegment = (el) => {
693
+ if (el.tagName === 'DIV')
694
+ return SEGMENT_ID.test(el.id);
695
+ if (el.tagName !== 'TABLE')
696
+ return false;
697
+ if (SEGMENT_ID.test(el.id))
698
+ return true;
699
+ return [...el.children].some((c) => (/^(?:TBODY|TR|COLGROUP)$/.test(c.tagName) && SEGMENT_ID.test(c.id)) ||
700
+ (c.tagName === 'TBODY' && !c.id && [...c.children].some((r) => r.tagName === 'TR' && SEGMENT_ID.test(r.id))));
701
+ };
702
+ // The <pre> elements of a block, outermost only, without those in a hidden list item, row, cell or other entry of a
703
+ // structure: such an entry is left out whole wherever it stands (T124 panel: codex, `<div hidden><ul><li hidden><pre>`
704
+ // printed the code that the same <li hidden> outside a hidden block drops).
705
+ const HIDDEN_ENTRY = /^(?:LI|TR|TD|TH|CAPTION|DT|DD|THEAD|TBODY|TFOOT)$/;
706
+ const outerPres = (el, opts) => [...el.querySelectorAll('pre')].filter((p) => {
707
+ if (p.parentElement?.closest('pre'))
708
+ return false;
709
+ for (let e = p.parentElement; e && e !== el; e = e.parentElement)
710
+ if (HIDDEN_ENTRY.test(e.tagName) && isHiddenEl(e, opts))
711
+ return false;
712
+ return true;
713
+ });
714
+ // A line of page TEXT that starts with Markdown block syntax becomes that block: `# x` a heading, `> x`
715
+ // a quote, `1. x` a list, `---` a rule, `<div` raw HTML, and an unclosed ``` fence swallows every line
716
+ // after it (T109 panel: `<div>```</div><p>AFTER</p>` put AFTER inside a code block). The renderer's own
717
+ // inline markup never starts a line with any of these, so escaping each emitted line start is safe.
718
+ const escapeBlockStart = (line) => {
719
+ const lead = /^[ \t]*/.exec(line)[0];
720
+ const rest = line.slice(lead.length);
721
+ if (/^(?:#{1,6}(?:\s|$)|>|[-+*](?:\s|$)|~{3}|<[A-Za-z/!?])/.test(rest) ||
722
+ /^`{3,}[^`]*$/.test(rest) ||
723
+ /^(?:=+|-+)\s*$/.test(rest) ||
724
+ /^([*_-])(?:\s*\1){2,}\s*$/.test(rest))
725
+ return `${lead}\\${rest}`;
726
+ const ol = /^\d{1,9}(?=[.)](?:\s|$))/.exec(rest);
727
+ return ol ? `${lead}${ol[0]}\\${rest.slice(ol[0].length)}` : line;
728
+ };
729
+ const escapeBlockStarts = (text) => text.split('\n').map(escapeBlockStart).join('\n');
730
+ const escapeLinkText = (t) => t.replace(/[\\[\]]/g, '\\$&');
731
+ // A link destination may hold parentheses only in balanced pairs, and no spaces, angle brackets or
732
+ // ASCII control characters (CommonMark 6.3): `https://x.test/a)b` ended the link after `a`, and a
733
+ // relative href that could not be resolved kept its spaces (BL-37). Those characters are
734
+ // percent-encoded, so the URL still works when copied out of the Markdown. Only the parentheses that
735
+ // have no partner are encoded: `/wiki/Foo_(bar)?x=)` keeps `Foo_(bar)` as written (T110 panel: grok).
736
+ const mdUrl = (u) => {
737
+ const open = [];
738
+ const unpaired = new Set();
739
+ for (let i = 0; i < u.length; i++) {
740
+ if (u[i] === '(')
741
+ open.push(i);
742
+ else if (u[i] === ')' && open.pop() === undefined)
743
+ unpaired.add(i);
744
+ }
745
+ for (const i of open)
746
+ unpaired.add(i);
747
+ const parens = unpaired.size
748
+ ? u.split('').map((ch, i) => (unpaired.has(i) ? (ch === '(' ? '%28' : '%29') : ch)).join('')
749
+ : u;
750
+ return parens.replace(/[\s<>\x00-\x1f\x7f]/g, (c) => encodeURIComponent(c));
751
+ };
752
+ // A link's text as Markdown, on one line: rendered inline with only the brackets and backslashes of its
753
+ // TEXT escaped. escapeMd over the already rendered text turned `<a><code>x</code></a>` into [\`x\`](u)
754
+ // and bold link text into literal \*\* stars (T109 panel: spark, grok, mimo; there since 0.1.0).
755
+ const linkText = (a, opts, href, excludeEls, trim = true) => {
756
+ const inner = renderInline(a, href ? { ...opts, inLink: true } : opts, excludeEls).replace(/[ \t]*\n[ \t]*/g, ' ');
757
+ if (inner.trim())
758
+ return trim ? inner.trim() : inner;
759
+ const fallback = cleanInline(linkName(a, opts));
760
+ return href ? escapeLinkText(fallback) : fallback;
761
+ };
762
+ // The name of a link whose content renders to nothing: its text, which keeps an icon's SVG <title>, as textContent did.
763
+ // textContent also returned the text of hidden elements (C7a, codex: `<a><span hidden>x</span></a>` printed x) and of a
764
+ // <style> or <script>, which name nothing. An image's alt is not used here: in a table cell the image already renders as
765
+ // its alt (C11a), and with includeImages: false an image link stays out, as the option says (the alt fallback wrote
766
+ // `[User avatar](profile)` before every author on the phpBB corpus page, T124 A/B).
767
+ const linkName = (a, opts) => nameText(a, opts);
768
+ // The text an element is named by, a link that renders empty or a tab: its text without what is not drawn, which is a
769
+ // hidden element of any kind, the fallback of a skipped element (<noscript>, <canvas>, <object>, <video>), and inside an
770
+ // SVG the parts SVG does not render (<defs>, <symbol>, <desc>, <metadata>). T124 panel: codex, grok, agy. An SVG's
771
+ // <title> and <text> stay: the <title> names an icon.
772
+ const SVG_UNDRAWN = /^(?:DEFS|SYMBOL|DESC|METADATA|STYLE|SCRIPT|CLIPPATH|MASK|PATTERN|MARKER|LINEARGRADIENT|RADIALGRADIENT|FILTER)$/;
773
+ const nameText = (el, opts) => {
774
+ let s = '';
775
+ const walk = (n, svg) => {
776
+ if (n.nodeType === 3 /* TEXT_NODE */) {
777
+ s += n.textContent || '';
778
+ return;
779
+ }
780
+ if (n.nodeType !== 1 /* ELEMENT_NODE */)
781
+ return;
782
+ const e = n;
783
+ const tag = e.tagName.toUpperCase();
784
+ if (underHiddenAttr(e, opts) || (svg ? SVG_UNDRAWN.test(tag) : SKIP_TAGS.has(tag) && tag !== 'SVG'))
785
+ return;
786
+ for (const c of e.childNodes)
787
+ walk(c, svg || tag === 'SVG');
788
+ };
789
+ for (const c of el.childNodes)
790
+ walk(c, false);
791
+ return s;
792
+ };
793
+ // The text of code as a browser lays it out: a <br> is a line break, and hidden elements are left out. textContent
794
+ // skips the first and keeps the second: Docusaurus ends every line of a code block with a <br> and writes no newline
795
+ // character, so `cd my-website` and `npx docusaurus start` came out as one line (benchmark corpus, T124), `x<br>y` in
796
+ // inline code read `xy` (C5, codex), and `<code>a<span hidden>s</span>b</code>` printed the hidden s (C7b, codex).
797
+ // Outside a <pre> a newline in the source is only whitespace, so inline code marks a <br> with BR_MARK instead.
798
+ const BR_MARK = '\u0000';
799
+ const codeText = (el, opts, br = '\n') => {
800
+ let s = '';
801
+ const walk = (n) => {
802
+ if (n.nodeType === 3 /* TEXT_NODE */) {
803
+ s += n.textContent || '';
804
+ return;
805
+ }
806
+ if (n.nodeType !== 1 /* ELEMENT_NODE */)
807
+ return;
808
+ const e = n;
809
+ // Hidden elements of every kind and skipped ones before the <br>: `<br hidden>` breaks no line, a <style> in code is
810
+ // not code, and a tab panel or an `S:` span inside code is no code tab (T124 panel: codex, grok, agy).
811
+ if (underHiddenAttr(e, opts) || SKIP_TAGS.has(e.tagName))
812
+ return;
813
+ if (e.tagName === 'BR') {
814
+ s += br;
815
+ return;
816
+ }
817
+ for (const c of e.childNodes)
818
+ walk(c);
819
+ };
820
+ for (const c of el.childNodes)
821
+ walk(c);
822
+ return s;
823
+ };
824
+ // Lines of code as inline Markdown: one code span per line, the lines joined by a line break, which a table cell writes
825
+ // as <br> (C28: a <pre> in a cell folded into one line; C5). Blank lines at either end are dropped.
826
+ const codeLines = (raw) => {
827
+ const lines = raw.map(collapseWs);
828
+ while (lines.length > 1 && !lines[0].trim())
829
+ lines.shift();
830
+ while (lines.length > 1 && !lines[lines.length - 1].trim())
831
+ lines.pop();
832
+ return lines.map((l) => wrapInline(l, codeSpan)).join('\n');
833
+ };
834
+ // A code span needs a fence longer than any backtick run inside it, padded with a space when the text
835
+ // starts or ends with a backtick (CommonMark 6.1): `a`b` closed after "a", and ``x`` lost its backticks.
836
+ const codeSpan = (t) => {
837
+ const fence = '`'.repeat(longestRun(t, '`') + 1);
838
+ const pad = t.startsWith('`') || t.endsWith('`') ? ' ' : '';
839
+ return `${fence}${pad}${t}${pad}${fence}`;
840
+ };
841
+ const WORD_START = /^[\p{L}\p{N}]/u;
842
+ const WORD_END = /[\p{L}\p{N}]$/u;
843
+ const PUNCT_START = /^[\p{P}\p{S}]/u;
844
+ const PUNCT_END = /[\p{P}\p{S}]$/u;
845
+ // Scripts written without spaces between words.
846
+ const UNSPACED_START = /^[\p{sc=Han}\p{sc=Hiragana}\p{sc=Katakana}\p{sc=Thai}\p{sc=Lao}\p{sc=Khmer}\p{sc=Myanmar}]/u;
847
+ const UNSPACED_END = /[\p{sc=Han}\p{sc=Hiragana}\p{sc=Katakana}\p{sc=Thai}\p{sc=Lao}\p{sc=Khmer}\p{sc=Myanmar}]$/u;
848
+ // Two ELEMENTS with nothing between them in the source get a space (a flex row of
849
+ // `<span>Plan</span><span>starter</span>`, a pilcrow link after a heading link: the gap is CSS), except
850
+ // where the junction sits inside one token: a script written without spaces (`東`+`京`), a currency
851
+ // and its amount (`$`+`19.99`), punctuation that attaches to what precedes it (`19`+`.99`, `60`+`%`),
852
+ // an opening bracket and what follows it, an `@` (`user`+`@x.test`). The blanket space split all of
853
+ // those (T109 panel). The panel's own proposal, a space only between two alphanumerics, glued
854
+ // `Inc.·[Trademark`, `more.[View docs` and `👍No` on the benchmark corpus, so it was not taken.
855
+ const wordsTouch = (left, right) => !!left && !!right &&
856
+ !(UNSPACED_END.test(left) || UNSPACED_START.test(right) ||
857
+ (/\p{Sc}$/u.test(left) && /^\p{N}/u.test(right)) ||
858
+ /^[.,;:!?%)\]}»”’@]/u.test(right) || /[([{«“‘@]$/u.test(left));
859
+ const collapseWs = (t) => t.replace(/ /g, ' ').replace(/[ \t\r\n\f]+/g, ' ');
860
+ // Markup keeps its content's edge whitespace OUTSIDE the delimiters: `<b> bold </b>` must become
861
+ // ` **bold** `, because `** bold **` is not emphasis.
862
+ const wrapInline = (inner, wrap) => {
863
+ const core = inner.trim();
864
+ if (!core)
865
+ return /\s/.test(inner) ? ' ' : '';
866
+ return (/^\s/.test(inner) ? ' ' : '') + wrap(core) + (/\s$/.test(inner) ? ' ' : '');
867
+ };
868
+ // Inline parts are concatenated with the source's own whitespace, never a separator of ours.
869
+ // linkedom splits text at EVERY character reference - `account&#39;s` arrives as the text nodes
870
+ // "account", "'", "s", and `caf&eacute;` as "caf", "é" - so the old `parts.join(' ')` broke each
871
+ // word holding one (P2A-002), and put a space between a link and the full stop after it. Two
872
+ // ELEMENTS with nothing between them get a space only between words (wordsTouch).
873
+ //
874
+ // Where markup is glued to its neighbour, the junction must still parse as intended (T109, checked
875
+ // with micromark): `**Note:**text` and `word**(x)**` are literal stars because the delimiter sits
876
+ // between punctuation and a letter, so emphasis is spaced from the letter there; `Hey!` + `[link](u)`
877
+ // is an image and `C:\` + `[d](u)` escapes the bracket, so that `!` or `\` is escaped.
878
+ //
879
+ // No textContent fallback for elements that render empty, except a link's name: it could only
880
+ // resurface text the skip list removes on purpose (script, style, an icon's SVG <title>).
508
881
  const renderInlineNodes = (nodes, opts, excludeEls) => {
509
- const parts = [];
882
+ let s = '';
883
+ let afterElement = false;
884
+ let lastVisible = '';
885
+ let punctEdge = false;
886
+ const add = (part, fromElement, visible = part, leadPunct = false) => {
887
+ if (!part)
888
+ return;
889
+ if (s && !/\s$/.test(s) && !/^\s/.test(part)) {
890
+ if (fromElement && afterElement && ((!opts.tight && wordsTouch(lastVisible, visible)) || (s.endsWith('*') && part.startsWith('*')))) {
891
+ s += ' ';
892
+ }
893
+ else if ((punctEdge && WORD_START.test(part)) || (leadPunct && WORD_END.test(s))) {
894
+ s += ' ';
895
+ }
896
+ else if (fromElement && /(?:^|[^\\])(?:\\\\)*\\$/.test(s) && /^[[!*`_]/.test(part)) {
897
+ s += '\\';
898
+ }
899
+ else if (fromElement && part.startsWith('[') && /(?:^|[^\\])(?:\\\\)*!$/.test(s)) {
900
+ s = `${s.slice(0, -1)}\\!`;
901
+ }
902
+ }
903
+ s += part;
904
+ afterElement = fromElement;
905
+ lastVisible = visible;
906
+ punctEdge = false;
907
+ };
908
+ const visibleText = (el, rendered) => collapseWs(el.textContent || '').trim() || rendered.trim();
510
909
  for (const node of nodes) {
511
910
  if (node.nodeType === 3 /* TEXT_NODE */) {
512
- const t = cleanInline(node.textContent || '');
513
- if (t)
514
- parts.push(t);
911
+ const t = collapseWs(node.textContent || '');
912
+ add(opts.inLink ? escapeLinkText(t) : t, false, t);
515
913
  continue;
516
914
  }
517
915
  if (node.nodeType !== 1 /* ELEMENT_NODE */)
@@ -520,64 +918,144 @@ const renderInlineNodes = (nodes, opts, excludeEls) => {
520
918
  if (excludeEls && excludeEls.has(c))
521
919
  continue;
522
920
  const tag = c.tagName;
523
- if (SKIP_TAGS.has(tag))
921
+ // aria-hidden is NOT skipped here, although the block walk skips it: docs.sentry.io puts the whole text
922
+ // of 15 <h2>s inside `<a aria-hidden="true">`, and skipping it deleted every one (T109 A/B). The
923
+ // attribute hides from screen readers, not from the screen. `hidden` hides from both (isHiddenEl).
924
+ const hidden = hiddenKind(c, opts);
925
+ if (SKIP_TAGS.has(tag) || hidden === 'hidden')
524
926
  continue;
927
+ if (hidden === 'code') {
928
+ // Each <pre> is set on its own lines by the PRE branch below.
929
+ const code = renderInlineNodes(outerPres(c, opts), opts, excludeEls);
930
+ add(code, true, code.trim());
931
+ continue;
932
+ }
525
933
  if (tag === 'BR') {
526
- parts.push('\n');
934
+ add('\n', false);
527
935
  continue;
528
936
  }
529
- if (tag === 'A') {
530
- const href = absUrl(c.getAttribute('href') || '', opts.baseUrl);
531
- const text = renderInline(c, opts) || cleanInline(c.textContent || '');
532
- if (text && href)
533
- parts.push(`[${escapeMd(text)}](${href})`);
534
- else if (text)
535
- parts.push(escapeMd(text));
937
+ if (tag === 'PRE') {
938
+ // A <pre> in inline position, which is in a table cell or a link: its lines stay lines (codeLines).
939
+ const code = codeLines(codeText(c, opts).replace(/\r/g, '').split('\n'));
940
+ add(`\n${code}\n`, true, collapseWs(c.textContent || '').trim());
941
+ continue;
942
+ }
943
+ if (tag === 'TABLE') {
944
+ // In inline position a table is nested in a table cell (the block walk renders every other one): its rows
945
+ // become lines of the cell instead of rows of the outer table (P2A-013).
946
+ add(` ${flattenTable(c, opts, excludeEls)} `, true, collapseWs(c.textContent || '').trim());
536
947
  continue;
537
948
  }
538
- if (tag === 'STRONG' || tag === 'B') {
539
- const t = renderInline(c, opts) || cleanInline(c.textContent || '');
540
- if (t)
541
- parts.push(`**${t}**`);
949
+ if (tag === 'A') {
950
+ const href = absUrl(c.getAttribute('href') || '', opts.baseUrl);
951
+ const text = linkText(c, opts, href, excludeEls, false);
952
+ add(wrapInline(text, (t) => (href ? `[${t}](${mdUrl(href)})` : t)), true, visibleText(c, text));
542
953
  continue;
543
954
  }
544
- if (tag === 'EM' || tag === 'I') {
545
- const t = renderInline(c, opts) || cleanInline(c.textContent || '');
546
- if (t)
547
- parts.push(`*${t}*`);
955
+ if (tag === 'STRONG' || tag === 'B' || tag === 'EM' || tag === 'I') {
956
+ const mark = tag === 'STRONG' || tag === 'B' ? '**' : '*';
957
+ const inner = renderInline(c, opts, excludeEls);
958
+ const core = inner.trim();
959
+ add(wrapInline(inner, (t) => `${mark}${t}${mark}`), true, visibleText(c, inner), PUNCT_START.test(core));
960
+ punctEdge = PUNCT_END.test(core) && !/\s$/.test(inner);
548
961
  continue;
549
962
  }
550
963
  if (tag === 'CODE') {
551
- const t = cleanInline(c.textContent || '');
552
- if (t)
553
- parts.push(`\`${t}\``);
964
+ const lines = codeText(c, opts, BR_MARK).split(BR_MARK);
965
+ add(codeLines(lines), true, collapseWs(lines.join(' ')).trim());
554
966
  continue;
555
967
  }
556
968
  if (tag === 'IMG') {
557
- // Same includeImages contract as the block-level IMG branch: an image
558
- // wrapped in a link, list item or table cell arrives here instead.
559
- if (!opts.includeImages)
969
+ // Same includeImages contract and src fallback as the block-level IMG branch: an image
970
+ // wrapped in a link, list item, table cell or text run arrives here instead.
971
+ if (!opts.includeImages) {
972
+ // A table cell shows no image but keeps its alt text, the text that stands for an image which is not shown (HTML,
973
+ // "the img element"): a severity icon's CRITICAL came out as an empty cell (C11b, T121 panel). An empty alt marks
974
+ // a decorative image and adds nothing, and so does aria-hidden="true" (T124 panel: agy, a bullet image wrote
975
+ // "bullet point" before every value).
976
+ const alt = opts.imageAlt && !isAriaHidden(c) ? cleanInline(c.getAttribute('alt') || '') : '';
977
+ if (alt)
978
+ add(` ${opts.inLink ? escapeLinkText(alt) : alt} `, true, alt);
560
979
  continue;
980
+ }
561
981
  const alt = cleanInline(c.getAttribute('alt') || '');
562
- const src = absUrl(c.getAttribute('src') || '', opts.baseUrl);
982
+ const src = absUrl(c.getAttribute('src') || c.getAttribute('data-src') || '', opts.baseUrl);
563
983
  if (src)
564
- parts.push(`![${escapeMd(alt || 'image')}](${src})`);
984
+ add(`![${escapeMd(alt || 'image')}](${mdUrl(src)})`, true, alt || 'image');
985
+ continue;
986
+ }
987
+ // Any other element, inline or a container in inline position (a P or styling DIV inside an
988
+ // <li>, a custom element): recurse so links and emphasis inside it survive as markdown.
989
+ const inner = renderInline(c, opts, excludeEls);
990
+ add(INLINE_BLOCK_GAP.has(tag) ? ` ${inner} ` : inner, true, visibleText(c, inner));
991
+ }
992
+ return s.replace(/[ \t]+/g, ' ').replace(/ ?\n ?/g, '\n');
993
+ };
994
+ // <dl> as the extension renders it (lib/core/html-to-md.ts in page2ai-extension): each term bold
995
+ // on its own line, each definition on a `: ` line below it, which is the definition-list syntax
996
+ // of Pandoc and PHP Markdown Extra and still plain to read where neither applies. HTML allows a
997
+ // <div> around each term/definition group, so groups are looked into one level deep. Returns
998
+ // false when the list holds no <dt>/<dd> at all, so the caller walks it as a plain container.
999
+ //
1000
+ // Unlike the extension, a <dd> holding block content is walked, not flattened: Sphinx puts a
1001
+ // whole API entry (paragraphs, code examples, tables, nested <dl>s) inside one <dd>, and the first
1002
+ // draft of this branch turned docs.python.org's code fences and tables into one line of text.
1003
+ const renderDefinitionList = (dl, out, opts, tabGroups, capturedPanelEls, dedupe) => {
1004
+ const isItem = (n) => n.nodeType === 1 && (n.tagName === 'DT' || n.tagName === 'DD');
1005
+ const isGroup = (n) => n.nodeType === 1 && n.tagName === 'DIV' && [...n.childNodes].some(isItem);
1006
+ if (![...dl.childNodes].some((n) => isItem(n) || isGroup(n)))
1007
+ return false;
1008
+ // Everything else in the list (a note, a heading, bare text: invalid HTML that browsers still show)
1009
+ // is walked where it stands. The first version collected only <dt>/<dd> and silently dropped the
1010
+ // rest, text 0.1.11 had printed (T109 panel, agy + grok: `<dl><p>Note</p><dt>..` lost "Note").
1011
+ const seq = [];
1012
+ for (const n of dl.childNodes) {
1013
+ if (n.nodeType === 1 && isHiddenEl(n, opts))
1014
+ continue;
1015
+ if (isGroup(n))
1016
+ seq.push(...[...n.childNodes].filter((c) => c.nodeType !== 1 || !isHiddenEl(c, opts)));
1017
+ else
1018
+ seq.push(n);
1019
+ }
1020
+ let lines = [];
1021
+ let stray = [];
1022
+ const flushLines = () => {
1023
+ if (lines.length)
1024
+ out.push('', ...lines, '');
1025
+ lines = [];
1026
+ };
1027
+ const flushStray = () => {
1028
+ if (stray.length)
1029
+ walkNodes(stray, out, opts, tabGroups, capturedPanelEls, dedupe);
1030
+ stray = [];
1031
+ };
1032
+ for (const n of seq) {
1033
+ if (!isItem(n)) {
1034
+ if (n.nodeType === 1 ? !SKIP_TAGS.has(n.tagName) : n.nodeType === 3 && /\S/.test(n.textContent || '')) {
1035
+ flushLines();
1036
+ stray.push(n);
1037
+ }
565
1038
  continue;
566
1039
  }
567
- if (INLINE_TAGS.has(tag)) {
568
- const t = renderInline(c, opts) || cleanInline(c.textContent || '');
569
- if (t)
570
- parts.push(t);
1040
+ flushStray();
1041
+ const it = n;
1042
+ if (it.tagName === 'DD' && hasBlockContent(it)) {
1043
+ flushLines();
1044
+ walkWithTabs(it, out, opts, tabGroups, capturedPanelEls, dedupe);
571
1045
  continue;
572
1046
  }
573
- // Unknown container in inline position (a P or styling DIV inside an <li>, a custom
574
- // element): recurse so links and emphasis inside it survive as markdown. The old fallback
575
- // took raw textContent, which kept the words but stripped every link.
576
- const t = renderInlineNodes([...c.childNodes], opts, excludeEls) || cleanInline(c.textContent || '');
577
- if (t)
578
- parts.push(t);
1047
+ const text = renderInline(it, opts, capturedPanelEls).trim();
1048
+ if (!text)
1049
+ continue;
1050
+ if (it.tagName === 'DT') {
1051
+ lines.push(`**${text}**`);
1052
+ continue;
1053
+ }
1054
+ text.split('\n').filter((l) => l.trim()).forEach((l, i) => lines.push(`${i === 0 ? ': ' : ' '}${i === 0 ? l : escapeBlockStart(l)}`));
579
1055
  }
580
- return parts.join(' ').replace(/\s+\n\s+/g, '\n');
1056
+ flushStray();
1057
+ flushLines();
1058
+ return true;
581
1059
  };
582
1060
  // Aliases that authors write but fence renderers do not highlight.
583
1061
  const LANG_ALIASES = { python3: 'python', py3: 'python' };
@@ -596,9 +1074,16 @@ const normalizeLang = (raw, knownOnly) => {
596
1074
  return l;
597
1075
  return '';
598
1076
  };
599
- const renderCodeBlock = (pre, out) => {
600
- const codeEl = pre.querySelector('code') || pre;
601
- const text = (codeEl.textContent || '').replace(/\r/g, '').replace(/^\n+|\n+$/g, '');
1077
+ const renderCodeBlock = (pre, out, opts) => {
1078
+ // The first <code> not under a hidden element: a hidden one replaced the visible text (T124 panel: codex,
1079
+ // `<pre>VISIBLE<code hidden>SECRET</code>TAIL</pre>` printed SECRET where Chromium shows VISIBLETAIL).
1080
+ const codeEl = [...pre.querySelectorAll('code')].find((c) => {
1081
+ for (let e = c; e && e !== pre; e = e.parentElement)
1082
+ if (underHiddenAttr(e, opts))
1083
+ return false;
1084
+ return true;
1085
+ }) || pre;
1086
+ const text = codeText(codeEl, opts).replace(/\r/g, '').replace(/^\n+|\n+$/g, '');
602
1087
  if (!text.trim())
603
1088
  return;
604
1089
  // The language label lives wherever the site generator put it, and that is rarely one place:
@@ -661,11 +1146,116 @@ const renderCodeBlock = (pre, out) => {
661
1146
  text.split('\n').forEach((l) => out.push(l));
662
1147
  out.push(fence, '');
663
1148
  };
664
- const renderTable = (tbl, out) => {
665
- const rows = [...tbl.querySelectorAll('tr')];
1149
+ // The table's own rows and cells. A descendant query also took those of a table nested in a cell: the inner rows came
1150
+ // out again as rows of the outer table and the inner cells widened the row holding it (P2A-013: 2 columns became 5).
1151
+ // Rows and cells under `hidden`, or inside a hidden row group, are left out (P2A-012).
1152
+ const hiddenUpTo = (el, stop, opts) => {
1153
+ for (let e = el; e && e !== stop; e = e.parentElement)
1154
+ if (isHiddenEl(e, opts))
1155
+ return true;
1156
+ return false;
1157
+ };
1158
+ const ownRows = (tbl, opts) => [...tbl.querySelectorAll('tr')].filter((tr) => tr.closest('table') === tbl && !hiddenUpTo(tr, tbl, opts));
1159
+ const ownCells = (tr, opts) => [...tr.querySelectorAll('th, td')].filter((c) => c.closest('tr') === tr && !hiddenUpTo(c, tr, opts));
1160
+ const ownCaption = (tbl, opts) => [...tbl.children].find((c) => c.tagName === 'CAPTION' && !isHiddenEl(c, opts));
1161
+ // A cell as inline Markdown on one line, rendered like any other inline content: links keep their target and code its
1162
+ // backticks (P2A-010: every row of a users list lost its link), whitespace collapses as a browser collapses it (a
1163
+ // pretty-printed template's indentation used to become `<br> value<br>`), a block child is spaced from its neighbours
1164
+ // and a <br> stays a line break (`<br>`). An image is written as its alt text: the image itself stays out of cells, as it
1165
+ // always has.
1166
+ const CELL = { includeImages: false, tight: true, imageAlt: true };
1167
+ const cellMd = (el, opts) => renderInline(el, { ...opts, ...CELL })
1168
+ .replace(/ *\n */g, '\n')
1169
+ .replace(/^\n+|\n+$/g, '')
1170
+ .replace(/\|/g, '\\|')
1171
+ .replace(/\n/g, '<br>')
1172
+ .replace(/[ \t]+/g, ' ')
1173
+ .trim();
1174
+ // The rows as a grid of slots, laid out as the HTML table model lays them: a cell spanning columns or rows leaves `null`
1175
+ // in the slots it covers, so the cells after it stay under their own headers (C6, codex: the cell after a colspan="2"
1176
+ // landed one column left). Spans are clamped as browsers clamp them (colspan 1..1000, rowspan 0..65534), and a span ends
1177
+ // with its row group, a <thead>, a <tbody>, a <tfoot> or a run of rows straight under <table>; rowspan="0" spans the
1178
+ // rest of the group. Chromium puts the <tbody> row after a <thead> rowspan="2" under the headers, where the grid pushed
1179
+ // it one column right (HTML: "A cell cannot cover slots that are from two or more row groups"; T124 panel: codex, grok,
1180
+ // mimo, agy; audits/T125-2026-10-16/scripts/browser_layout_check.py). A row under `hidden` takes no part: Chromium lays a
1181
+ // rowspan over the next shown row. Only the columns in which some cell starts are kept: `colspan="100"` on a details
1182
+ // row means "the full width", not a hundred columns. The extension pads a colspan with empty cells the same way (max 20,
1183
+ // no rowspan).
1184
+ // The page decides the spans, so the slots they cover are budgeted: past SPAN_SLOTS the table gets the plain layout,
1185
+ // cells in source order, which is what earlier versions printed for every table.
1186
+ const SPAN_SLOTS = 100000;
1187
+ const tableGrid = (rows, opts) => {
1188
+ const grid = rows.map(() => []);
1189
+ const starts = new Set();
1190
+ let covered = 0;
1191
+ const groupEnd = [];
1192
+ for (let r = rows.length - 1; r >= 0; r--) {
1193
+ groupEnd[r] = r + 1 < rows.length && rows[r + 1].parentElement === rows[r].parentElement ? groupEnd[r + 1] : r + 1;
1194
+ }
1195
+ for (let r = 0; r < rows.length; r++) {
1196
+ let col = 0;
1197
+ const left = groupEnd[r] - r;
1198
+ for (const cell of ownCells(rows[r], opts)) {
1199
+ while (grid[r][col] !== undefined)
1200
+ col++;
1201
+ const cs = Math.min(Math.max(parseInt(cell.getAttribute('colspan') || '', 10) || 1, 1), 1000);
1202
+ const rsRaw = parseInt(cell.getAttribute('rowspan') || '', 10);
1203
+ const rs = rsRaw === 0 ? left : Math.min(Math.max(rsRaw || 1, 1), 65534, left);
1204
+ covered += cs * rs - 1;
1205
+ if (covered > SPAN_SLOTS)
1206
+ return rows.map((tr) => ownCells(tr, opts));
1207
+ for (let dr = 0; dr < rs; dr++)
1208
+ for (let dc = 0; dc < cs; dc++)
1209
+ grid[r + dr][col + dc] = dr || dc ? null : cell;
1210
+ starts.add(col);
1211
+ col += cs;
1212
+ }
1213
+ }
1214
+ const keep = [...starts].sort((a, b) => a - b);
1215
+ return grid.map((row) => keep.map((c) => row[c] || null));
1216
+ };
1217
+ // A table nested in a cell, as lines of that cell: its caption, then one line per row, a header cell joined to the
1218
+ // data cell after it by ": " (the key/value tables of a details row read `k: v`) and other cells by " · ". Under a header
1219
+ // row (two or more header cells and nothing else) an empty cell keeps its place, `a · · c`: dropping it moved every later
1220
+ // value under the header before it (C18, agy).
1221
+ const flattenTable = (tbl, opts, excludeEls) => {
1222
+ const one = (el) => renderInline(el, opts, excludeEls).replace(/\s+/g, ' ').trim();
1223
+ const cap = ownCaption(tbl, opts);
1224
+ const lines = cap && one(cap) ? [one(cap)] : [];
1225
+ const grid = tableGrid(ownRows(tbl, opts), opts);
1226
+ const head = (grid[0] || []).filter((c) => !!c);
1227
+ const positional = head.length >= 2 && head.every((c) => c.tagName === 'TH');
1228
+ for (const row of grid) {
1229
+ const cells = row.map((c) => ({ t: c ? one(c) : '', th: c?.tagName === 'TH' }));
1230
+ while (cells.length && !cells[cells.length - 1].t)
1231
+ cells.pop();
1232
+ let line = '';
1233
+ let started = false;
1234
+ let afterTh = false;
1235
+ for (const { t, th } of cells) {
1236
+ if (!t && !positional)
1237
+ continue;
1238
+ if (started)
1239
+ line += afterTh && !th ? (line.endsWith(':') ? ' ' : ': ') : ' · ';
1240
+ line += t;
1241
+ started = true;
1242
+ afterTh = th;
1243
+ }
1244
+ if (line)
1245
+ lines.push(line);
1246
+ }
1247
+ return lines.join('\n');
1248
+ };
1249
+ const renderTable = (tbl, out, opts) => {
1250
+ const rows = ownRows(tbl, opts);
666
1251
  if (!rows.length)
667
1252
  return;
668
- const data = rows.map((tr) => [...tr.querySelectorAll('th, td')].map((c) => escapeMdTableCell(c.textContent || '')));
1253
+ // <caption> text was never printed (grok, T109): it names the table.
1254
+ const capEl = ownCaption(tbl, opts);
1255
+ const caption = capEl ? escapeBlockStart(renderInline(capEl, { ...opts, ...CELL }).replace(/\s+/g, ' ').trim()) : '';
1256
+ if (caption)
1257
+ out.push('', caption);
1258
+ const data = tableGrid(rows, opts).map((row) => row.map((c) => (c ? cellMd(c, opts) : '')));
669
1259
  const width = Math.max(...data.map((r) => r.length));
670
1260
  const padded = data.map((r) => { while (r.length < width)
671
1261
  r.push(''); return r; });