@ai-matrx/content-ir 0.19.17 → 0.19.19

package/CHANGELOG.md

@@ -1,5 +1,15 @@
 # Changelog
 
+## 0.19.19
+### Unreleased
+
+- `/source` **THE ONE DELIBERATE GFM DEVIATION** (verify-RC-B4 round 16 ruling: content never silently disappears): a line opening an HTML block whose element every sanitizer strips — `STRIPPED_HTML_BLOCK_NAMES` (`link`, `nav`, `header`, `footer`, `article`, `aside`, `main`, `form`, `iframe`, `title`, … the type-6 names neither the screen's raw-HTML allow-list nor print's GitHub schema keeps) — ends that block at its own line, so a table or list under it is read as markdown (`insideHtmlBlock`; new `opensStrippedHtmlBlock(trimmed)`). GFM keeps such lines as raw HTML the sanitizer then empties; a stored answer quoting `<link rel="canonical" …/>` above its content lost every list that way. Python twin moved; `tableStarts` vectors +4 (the frontend's GFM judge applies the deviation). Documented in FEATURE.md § `source/`.
+
+## 0.19.18
+### Unreleased
+
+- `/source` **An unclosed HTML block tag owns nothing** (verify-RC-B4 round 9): a line-leading `<div>`, `<section>`, `<details>`… that never closes is no `xml_container` island running to the end of the text — GFM ends that HTML block at the blank line, so the text after it stays prose (the Visual editor showed a whole note swallowed as a "Section · div, not closed" island). The tag stays an inline `xml_tag` island; a closed container is still one island; an unclosed custom tag still owns the rest. Python twin moved; island vectors +5.
+
 ## 0.19.17
 ### Unreleased
 

package/dist/index.cjs

@@ -6059,6 +6059,210 @@ var XmlContainerTracker = class {
   }
 };
 
+// source/gfm-table.ts
+function splitRowSegments(line) {
+  const segments = [];
+  let start = 0;
+  for (let i = 0; i < line.length; i += 1) {
+    const ch = line[i];
+    if (ch === "\\") {
+      i += 1;
+      continue;
+    }
+    if (ch === "|") {
+      segments.push(line.slice(start, i));
+      start = i + 1;
+    }
+  }
+  segments.push(line.slice(start));
+  return segments;
+}
+function rowCells(line) {
+  const cells = splitRowSegments(line).map((cell) => cell.trim());
+  if (cells.length > 1 && cells[0] === "") cells.shift();
+  if (cells.length > 1 && cells[cells.length - 1] === "") cells.pop();
+  else if (cells.length === 1 && cells[0] === "" && !line.includes("|")) cells.pop();
+  return cells;
+}
+function unescapeCellPipes(cell) {
+  return cell.replace(/\\\|/g, "|");
+}
+var BLOCK_START = /^(?:[-+*](?:\s|$)|\d{1,9}[.)](?:\s|$)|>|#{1,6}(?:\s|$)|(?:-\s*){3,}$|(?:\*\s*){3,}$|(?:_\s*){3,}$|\[\^[^\]\s]+\]:)/;
+var startsBlock = (trimmed) => BLOCK_START.test(trimmed) || opensStrictFence(trimmed) || startsHtmlBlock(trimmed);
+var HTML_RAW = /^<(?:script|pre|style|textarea)(?:[\s>]|$)/i;
+var HTML_COMMENT_ETC = /^(?:<!--|<\?|<![A-Za-z]|<!\[CDATA\[)/;
+var HTML_BLOCK_NAMES = "address|article|aside|base|basefont|blockquote|body|caption|center|col|colgroup|dd|details|dialog|dir|div|dl|dt|fieldset|figcaption|figure|footer|form|frame|frameset|h[1-6]|head|header|hr|html|iframe|legend|li|link|main|menu|menuitem|nav|noframes|ol|optgroup|option|p|param|search|section|summary|table|tbody|td|tfoot|th|thead|title|tr|track|ul";
+var HTML_BLOCK_TAG = new RegExp(`^</?(?:${HTML_BLOCK_NAMES})(?:[\\s>]|/>|$)`, "i");
+var ATTRIBUTE = `\\s+[A-Za-z_:][A-Za-z0-9_.:-]*(?:\\s*=\\s*(?:[^\\s"'=<>\`]+|'[^']*'|"[^"]*"))?`;
+var HTML_COMPLETE_TAG = new RegExp(`^(?:<[A-Za-z][A-Za-z0-9-]*(?:${ATTRIBUTE})*\\s*/?>|</[A-Za-z][A-Za-z0-9-]*\\s*>)\\s*$`);
+var STRIPPED_HTML_BLOCK_NAMES = "address|article|aside|base|basefont|body|center|dialog|dir|fieldset|footer|form|frame|frameset|head|header|html|iframe|legend|link|main|menu|menuitem|nav|noframes|optgroup|option|param|search|title|track";
+var STRIPPED_HTML_BLOCK_TAG = new RegExp(`^</?(?:${STRIPPED_HTML_BLOCK_NAMES})(?:[\\s>]|/>|$)`, "i");
+function opensStrippedHtmlBlock(trimmed) {
+  return STRIPPED_HTML_BLOCK_TAG.test(trimmed);
+}
+var HTML_BLOCK_NAME_SET = /* @__PURE__ */ new Set([
+  // Condition 1 (raw text) and condition 6 (block) tag names — CommonMark 0.31.
+  "script",
+  "pre",
+  "style",
+  "textarea",
+  ...HTML_BLOCK_NAMES.replace("h[1-6]", "h1|h2|h3|h4|h5|h6").split("|")
+]);
+function isHtmlBlockTagName(name) {
+  return HTML_BLOCK_NAME_SET.has(name.toLowerCase());
+}
+function startsHtmlBlock(trimmed) {
+  if (!trimmed.startsWith("<")) return false;
+  return HTML_RAW.test(trimmed) || HTML_COMMENT_ETC.test(trimmed) || HTML_BLOCK_TAG.test(trimmed) || HTML_COMPLETE_TAG.test(trimmed);
+}
+function lineIndent(line) {
+  let column = 0;
+  for (const ch of line) {
+    if (ch === " ") column += 1;
+    else if (ch === "	") column += 4 - column % 4;
+    else break;
+  }
+  return column;
+}
+var LIST_MARKER = /^([ \t]*)([-+*]|\d{1,9}[.)])([ \t]+|$)/;
+function markerContentColumn(marker) {
+  const markerEnd = lineIndent(marker[1] ?? "") + (marker[2] ?? "").length;
+  const gap = marker[3] ? lineIndent(" ".repeat(markerEnd) + marker[3]) - markerEnd : 1;
+  return markerEnd + (gap >= 1 && gap <= 4 ? gap : 1);
+}
+function tableContainerIndent(lines, start) {
+  return tableContainerIndentOf(lines, start, false);
+}
+function tableContainerIndentOf(lines, start, ownItem) {
+  const own = ownItem ? LIST_MARKER.exec(lines[start] ?? "") : null;
+  if (own) return markerContentColumn(own);
+  const header = lineIndent(lines[start] ?? "");
+  if (header === 0) return 0;
+  for (let k = start - 1; k >= 0; k -= 1) {
+    const line = lines[k] ?? "";
+    if (!line.trim()) continue;
+    const indent = lineIndent(line);
+    const marker = LIST_MARKER.exec(line);
+    if (marker && indent < header) {
+      const content = markerContentColumn(marker);
+      if (content <= header) return content;
+      continue;
+    }
+    if (indent === 0) return 0;
+  }
+  return 0;
+}
+var DELIMITER_CELL = /^:?-+:?$/;
+function isGfmDelimiterRow(line) {
+  const trimmed = line.trim();
+  if (!trimmed.includes("|")) return false;
+  const cells = rowCells(trimmed);
+  return cells.length > 0 && cells.every((cell) => DELIMITER_CELL.test(cell));
+}
+function isPipeLedRow(line) {
+  const trimmed = line.trim();
+  return trimmed.startsWith("|") && trimmed.includes("|", 1);
+}
+function isTableDelimiter(line) {
+  if (line === void 0 || !isGfmDelimiterRow(line)) return false;
+  return !BLOCK_START.test(line.trim());
+}
+function startsPipelessTable(header, delimiter) {
+  const head = header.trim();
+  if (!head || isPipeLedRow(head) || !head.startsWith("|") && startsBlock(head)) return false;
+  if (!isTableDelimiter(delimiter)) return false;
+  return rowCells(head).length === rowCells(delimiter.trim()).length;
+}
+function continuesTable(line, containerIndent = 0) {
+  const trimmed = line.trim();
+  if (!trimmed) return false;
+  if (lineIndent(line) - containerIndent >= 4) return false;
+  return trimmed.startsWith("|") || !startsBlock(trimmed);
+}
+function opensTable(lines, index) {
+  const line = lines[index] ?? "";
+  if (!isPipeLedRow(line) && !startsPipelessTable(line, lines[index + 1])) return false;
+  return !isLazyContinuation(lines, index) && !insideHtmlBlock(lines, index);
+}
+var QUOTE_LINE = /^ {0,3}>/;
+var THEMATIC_BREAK = /^(?:(?:-[ \t]*){3,}|(?:\*[ \t]*){3,}|(?:_[ \t]*){3,})$/;
+var breaksParagraph = (trimmed) => /^#{1,6}(?:\s|$)/.test(trimmed) || THEMATIC_BREAK.test(trimmed) || opensStrictFence(trimmed) || startsHtmlBlock(trimmed);
+function htmlBlockEnd(trimmed) {
+  if (HTML_RAW.test(trimmed)) return /<\/(?:script|pre|style|textarea)>/i;
+  if (trimmed.startsWith("<!--")) return /-->/;
+  if (trimmed.startsWith("<?")) return /\?>/;
+  if (trimmed.startsWith("<![CDATA[")) return /\]\]>/;
+  if (/^<![A-Za-z]/.test(trimmed)) return />/;
+  if (HTML_BLOCK_TAG.test(trimmed)) return null;
+  if (HTML_COMPLETE_TAG.test(trimmed)) return null;
+  return void 0;
+}
+function insideHtmlBlock(lines, index) {
+  let start = index;
+  while (start > 0 && (lines[start - 1] ?? "").trim()) start -= 1;
+  let open;
+  let paragraph = false;
+  for (let k = start; k < index; k += 1) {
+    const line = lines[k] ?? "";
+    if (open !== void 0) {
+      if (open && open.test(line)) open = void 0;
+      continue;
+    }
+    const trimmed = line.trim();
+    const end = lineIndent(line) < 4 ? htmlBlockEnd(trimmed) : void 0;
+    const type7 = end === null && !HTML_BLOCK_TAG.test(trimmed);
+    const interrupts = end !== void 0 && !(paragraph && type7);
+    if (!interrupts) {
+      paragraph = true;
+      continue;
+    }
+    paragraph = false;
+    if (opensStrippedHtmlBlock(trimmed)) continue;
+    open = end && end.test(trimmed.slice(2)) ? void 0 : end;
+  }
+  return open !== void 0;
+}
+function isLazyContinuation(lines, index) {
+  const header = lines[index] ?? "";
+  let first = -1;
+  for (let k = index - 1; k >= 0; k -= 1) {
+    const line = lines[k] ?? "";
+    const trimmed = line.trim();
+    if (!trimmed) break;
+    if (breaksParagraph(trimmed)) break;
+    const marker = LIST_MARKER.exec(line);
+    if (marker) {
+      const rest = line.slice(marker[0].length).trim();
+      if (!rest || breaksParagraph(rest)) break;
+      first = k;
+      break;
+    }
+    first = k;
+    if (QUOTE_LINE.test(line)) break;
+  }
+  if (first < 0) return false;
+  const start = lines[first] ?? "";
+  if (QUOTE_LINE.test(start)) return !QUOTE_LINE.test(header);
+  return lineIndent(header) < tableContainerIndentOf(lines, first, true);
+}
+function tableStartsAt(lines, index) {
+  const header = lines[index] ?? "";
+  const delimiter = lines[index + 1];
+  if (!opensTable(lines, index) || !isTableDelimiter(delimiter)) return false;
+  if (rowCells(header).length !== rowCells(delimiter).length) return false;
+  return lineIndent(header) - tableContainerIndent(lines, index) < 4;
+}
+function findTableStart(lines, from = 0) {
+  for (let i = Math.max(0, from); i < lines.length; i += 1) if (opensTable(lines, i)) return i;
+  return -1;
+}
+function findTableEnd(lines, start) {
+  const container = tableContainerIndent(lines, start);
+  let end = Math.min(lines.length, start + 2);
+  while (end < lines.length && continuesTable(lines[end] ?? "", container)) end += 1;
+  return end;
+}
+
 // source/tokenize.ts
 var NOT_WHITESPACE = /\S/;
 var DIRECTIVE_OPEN = /^(:{2,})([A-Za-z][\w-]*)/;
@@ -6900,9 +7104,12 @@ var Tokenizer = class {
           });
         }
       }
+      if (isHtmlBlockTagName(root)) return null;
       return this.island("xml_container", p, this.n, false, { tag: root });
     }
     if (isUnclosedGenericXmlOpening(seg)) {
+      const name = /^\s*<([A-Za-z_][\w.:-]*)/.exec(seg)?.[1] ?? "";
+      if (isHtmlBlockTagName(name)) return null;
       return this.island("xml_container", p, this.n, false, {});
     }
     return null;
@@ -7726,204 +7933,6 @@ function resolveReferenceLinks(fragment, defs) {
   return resolved.replace(new RegExp(`${PLACEHOLDER}(\\d+)${PLACEHOLDER}`, "g"), (_m, i) => spans[Number(i)] ?? "");
 }
 
-// source/gfm-table.ts
-function splitRowSegments(line) {
-  const segments = [];
-  let start = 0;
-  for (let i = 0; i < line.length; i += 1) {
-    const ch = line[i];
-    if (ch === "\\") {
-      i += 1;
-      continue;
-    }
-    if (ch === "|") {
-      segments.push(line.slice(start, i));
-      start = i + 1;
-    }
-  }
-  segments.push(line.slice(start));
-  return segments;
-}
-function rowCells(line) {
-  const cells = splitRowSegments(line).map((cell) => cell.trim());
-  if (cells.length > 1 && cells[0] === "") cells.shift();
-  if (cells.length > 1 && cells[cells.length - 1] === "") cells.pop();
-  else if (cells.length === 1 && cells[0] === "" && !line.includes("|")) cells.pop();
-  return cells;
-}
-function unescapeCellPipes(cell) {
-  return cell.replace(/\\\|/g, "|");
-}
-var BLOCK_START = /^(?:[-+*](?:\s|$)|\d{1,9}[.)](?:\s|$)|>|#{1,6}(?:\s|$)|(?:-\s*){3,}$|(?:\*\s*){3,}$|(?:_\s*){3,}$|\[\^[^\]\s]+\]:)/;
-var startsBlock = (trimmed) => BLOCK_START.test(trimmed) || opensStrictFence(trimmed) || startsHtmlBlock(trimmed);
-var HTML_RAW = /^<(?:script|pre|style|textarea)(?:[\s>]|$)/i;
-var HTML_COMMENT_ETC = /^(?:<!--|<\?|<![A-Za-z]|<!\[CDATA\[)/;
-var HTML_BLOCK_NAMES = "address|article|aside|base|basefont|blockquote|body|caption|center|col|colgroup|dd|details|dialog|dir|div|dl|dt|fieldset|figcaption|figure|footer|form|frame|frameset|h[1-6]|head|header|hr|html|iframe|legend|li|link|main|menu|menuitem|nav|noframes|ol|optgroup|option|p|param|search|section|summary|table|tbody|td|tfoot|th|thead|title|tr|track|ul";
-var HTML_BLOCK_TAG = new RegExp(`^</?(?:${HTML_BLOCK_NAMES})(?:[\\s>]|/>|$)`, "i");
-var ATTRIBUTE = `\\s+[A-Za-z_:][A-Za-z0-9_.:-]*(?:\\s*=\\s*(?:[^\\s"'=<>\`]+|'[^']*'|"[^"]*"))?`;
-var HTML_COMPLETE_TAG = new RegExp(`^(?:<[A-Za-z][A-Za-z0-9-]*(?:${ATTRIBUTE})*\\s*/?>|</[A-Za-z][A-Za-z0-9-]*\\s*>)\\s*$`);
-var HTML_BLOCK_NAME_SET = /* @__PURE__ */ new Set([
-  // Condition 1 (raw text) and condition 6 (block) tag names — CommonMark 0.31.
-  "script",
-  "pre",
-  "style",
-  "textarea",
-  ...HTML_BLOCK_NAMES.replace("h[1-6]", "h1|h2|h3|h4|h5|h6").split("|")
-]);
-function isHtmlBlockTagName(name) {
-  return HTML_BLOCK_NAME_SET.has(name.toLowerCase());
-}
-function startsHtmlBlock(trimmed) {
-  if (!trimmed.startsWith("<")) return false;
-  return HTML_RAW.test(trimmed) || HTML_COMMENT_ETC.test(trimmed) || HTML_BLOCK_TAG.test(trimmed) || HTML_COMPLETE_TAG.test(trimmed);
-}
-function lineIndent(line) {
-  let column = 0;
-  for (const ch of line) {
-    if (ch === " ") column += 1;
-    else if (ch === "	") column += 4 - column % 4;
-    else break;
-  }
-  return column;
-}
-var LIST_MARKER = /^([ \t]*)([-+*]|\d{1,9}[.)])([ \t]+|$)/;
-function markerContentColumn(marker) {
-  const markerEnd = lineIndent(marker[1] ?? "") + (marker[2] ?? "").length;
-  const gap = marker[3] ? lineIndent(" ".repeat(markerEnd) + marker[3]) - markerEnd : 1;
-  return markerEnd + (gap >= 1 && gap <= 4 ? gap : 1);
-}
-function tableContainerIndent(lines, start) {
-  return tableContainerIndentOf(lines, start, false);
-}
-function tableContainerIndentOf(lines, start, ownItem) {
-  const own = ownItem ? LIST_MARKER.exec(lines[start] ?? "") : null;
-  if (own) return markerContentColumn(own);
-  const header = lineIndent(lines[start] ?? "");
-  if (header === 0) return 0;
-  for (let k = start - 1; k >= 0; k -= 1) {
-    const line = lines[k] ?? "";
-    if (!line.trim()) continue;
-    const indent = lineIndent(line);
-    const marker = LIST_MARKER.exec(line);
-    if (marker && indent < header) {
-      const content = markerContentColumn(marker);
-      if (content <= header) return content;
-      continue;
-    }
-    if (indent === 0) return 0;
-  }
-  return 0;
-}
-var DELIMITER_CELL = /^:?-+:?$/;
-function isGfmDelimiterRow(line) {
-  const trimmed = line.trim();
-  if (!trimmed.includes("|")) return false;
-  const cells = rowCells(trimmed);
-  return cells.length > 0 && cells.every((cell) => DELIMITER_CELL.test(cell));
-}
-function isPipeLedRow(line) {
-  const trimmed = line.trim();
-  return trimmed.startsWith("|") && trimmed.includes("|", 1);
-}
-function isTableDelimiter(line) {
-  if (line === void 0 || !isGfmDelimiterRow(line)) return false;
-  return !BLOCK_START.test(line.trim());
-}
-function startsPipelessTable(header, delimiter) {
-  const head = header.trim();
-  if (!head || isPipeLedRow(head) || !head.startsWith("|") && startsBlock(head)) return false;
-  if (!isTableDelimiter(delimiter)) return false;
-  return rowCells(head).length === rowCells(delimiter.trim()).length;
-}
-function continuesTable(line, containerIndent = 0) {
-  const trimmed = line.trim();
-  if (!trimmed) return false;
-  if (lineIndent(line) - containerIndent >= 4) return false;
-  return trimmed.startsWith("|") || !startsBlock(trimmed);
-}
-function opensTable(lines, index) {
-  const line = lines[index] ?? "";
-  if (!isPipeLedRow(line) && !startsPipelessTable(line, lines[index + 1])) return false;
-  return !isLazyContinuation(lines, index) && !insideHtmlBlock(lines, index);
-}
-var QUOTE_LINE = /^ {0,3}>/;
-var THEMATIC_BREAK = /^(?:(?:-[ \t]*){3,}|(?:\*[ \t]*){3,}|(?:_[ \t]*){3,})$/;
-var breaksParagraph = (trimmed) => /^#{1,6}(?:\s|$)/.test(trimmed) || THEMATIC_BREAK.test(trimmed) || opensStrictFence(trimmed) || startsHtmlBlock(trimmed);
-function htmlBlockEnd(trimmed) {
-  if (HTML_RAW.test(trimmed)) return /<\/(?:script|pre|style|textarea)>/i;
-  if (trimmed.startsWith("<!--")) return /-->/;
-  if (trimmed.startsWith("<?")) return /\?>/;
-  if (trimmed.startsWith("<![CDATA[")) return /\]\]>/;
-  if (/^<![A-Za-z]/.test(trimmed)) return />/;
-  if (HTML_BLOCK_TAG.test(trimmed)) return null;
-  if (HTML_COMPLETE_TAG.test(trimmed)) return null;
-  return void 0;
-}
-function insideHtmlBlock(lines, index) {
-  let start = index;
-  while (start > 0 && (lines[start - 1] ?? "").trim()) start -= 1;
-  let open;
-  let paragraph = false;
-  for (let k = start; k < index; k += 1) {
-    const line = lines[k] ?? "";
-    if (open !== void 0) {
-      if (open && open.test(line)) open = void 0;
-      continue;
-    }
-    const trimmed = line.trim();
-    const end = lineIndent(line) < 4 ? htmlBlockEnd(trimmed) : void 0;
-    const type7 = end === null && !HTML_BLOCK_TAG.test(trimmed);
-    const interrupts = end !== void 0 && !(paragraph && type7);
-    if (!interrupts) {
-      paragraph = true;
-      continue;
-    }
-    paragraph = false;
-    open = end && end.test(trimmed.slice(2)) ? void 0 : end;
-  }
-  return open !== void 0;
-}
-function isLazyContinuation(lines, index) {
-  const header = lines[index] ?? "";
-  let first = -1;
-  for (let k = index - 1; k >= 0; k -= 1) {
-    const line = lines[k] ?? "";
-    const trimmed = line.trim();
-    if (!trimmed) break;
-    if (breaksParagraph(trimmed)) break;
-    const marker = LIST_MARKER.exec(line);
-    if (marker) {
-      const rest = line.slice(marker[0].length).trim();
-      if (!rest || breaksParagraph(rest)) break;
-      first = k;
-      break;
-    }
-    first = k;
-    if (QUOTE_LINE.test(line)) break;
-  }
-  if (first < 0) return false;
-  const start = lines[first] ?? "";
-  if (QUOTE_LINE.test(start)) return !QUOTE_LINE.test(header);
-  return lineIndent(header) < tableContainerIndentOf(lines, first, true);
-}
-function tableStartsAt(lines, index) {
-  const header = lines[index] ?? "";
-  const delimiter = lines[index + 1];
-  if (!opensTable(lines, index) || !isTableDelimiter(delimiter)) return false;
-  if (rowCells(header).length !== rowCells(delimiter).length) return false;
-  return lineIndent(header) - tableContainerIndent(lines, index) < 4;
-}
-function findTableStart(lines, from = 0) {
-  for (let i = Math.max(0, from); i < lines.length; i += 1) if (opensTable(lines, i)) return i;
-  return -1;
-}
-function findTableEnd(lines, start) {
-  const container = tableContainerIndent(lines, start);
-  let end = Math.min(lines.length, start + 2);
-  while (end < lines.length && continuesTable(lines[end] ?? "", container)) end += 1;
-  return end;
-}
-
 exports.ACTION_BY_CLASS = ACTION_BY_CLASS;
 exports.AFTER_WINDOW = AFTER_WINDOW;
 exports.BacktickRuns = BacktickRuns;
@@ -8057,6 +8066,7 @@ exports.nounFamily = nounFamily;
 exports.nounLabel = nounLabel;
 exports.nounTitleColumn = nounTitleColumn;
 exports.openParseSession = openParseSession;
+exports.opensStrippedHtmlBlock = opensStrippedHtmlBlock;
 exports.opensTable = opensTable;
 exports.pairDisplayMath = pairDisplayMath;
 exports.parseDirectiveSlug = parseDirectiveSlug;