@effected/markdown 0.2.0

package/internal/blocks/table.js

@@ -0,0 +1,378 @@
+import { Table, TableCell, TableRow } from "../../MarkdownNode.js";
+import { tableCellChildren, tableRowChildren } from "../blockTypes.js";
+import { ESCAPABLE } from "../unescape.js";
+import { sliceWithSegments } from "../segments.js";
+
+//#region src/internal/blocks/table.ts
+/**
+* Upstream's `MAX_AUTOCOMPLETED_CELLS`: the ceiling on cells the parser
+* invents to pad short rows, which is what stops a wide header followed by a
+* million one-character lines from allocating unboundedly.
+*/
+const MAX_AUTOCOMPLETED_CELLS = 524288;
+const reEscapable = new RegExp(`^${ESCAPABLE}`);
+/** re2c's `spacechar` class: the delimiter-row grammar's idea of whitespace. */
+const isSpaceChar = (char) => char === " " || char === "	" || char === "\v" || char === "\f";
+/** `_scan_table_cell_end`: `[|] spacechar*`, the pipe between two cells. */
+const scanCellEnd = (text, from) => {
+	if (text.charAt(from) !== "|") return 0;
+	let index = from + 1;
+	while (isSpaceChar(text.charAt(index))) index += 1;
+	return index - from;
+};
+/**
+* `_scan_table_cell`: `(escaped_char|[^|\r\n])+`.
+*
+* A backslash before ASCII punctuation takes the punctuation with it, which is
+* the whole mechanism behind escaped pipes: `\|` is one cell character, a bare
+* `|` ends the cell.
+*/
+const scanCell = (text, from) => {
+	let index = from;
+	while (index < text.length) {
+		const char = text.charAt(index);
+		if (char === "\\" && reEscapable.test(text.charAt(index + 1))) {
+			index += 2;
+			continue;
+		}
+		if (char === "|" || char === "\r" || char === "\n") break;
+		index += 1;
+	}
+	return index - from;
+};
+/** `_scan_table_row_end`: `spacechar* newline`. */
+const scanRowEnd = (text, from) => {
+	let index = from;
+	while (isSpaceChar(text.charAt(index))) index += 1;
+	if (text.charAt(index) === "\r") index += 1;
+	if (text.charAt(index) !== "\n") return 0;
+	return index + 1 - from;
+};
+/**
+* `_scan_table_start`: `[|]? marker ([|] marker)* [|]? spacechar* newline`,
+* where a marker is `spacechar* [:]? [-]+ [:]? spacechar*`.
+*
+* Hand-rolled rather than a regular expression: the grammar nests two
+* unbounded whitespace runs inside a repetition, which is exactly the shape a
+* backtracking engine goes exponential on, and the pathological corpus feeds
+* this scanner adversarial delimiter rows.
+*/
+const scanDelimiterRowLine = (line, from) => {
+	let index = from;
+	if (line.charAt(index) === "|") index += 1;
+	for (;;) {
+		while (isSpaceChar(line.charAt(index))) index += 1;
+		if (line.charAt(index) === ":") index += 1;
+		let dashes = 0;
+		while (line.charAt(index) === "-") {
+			index += 1;
+			dashes += 1;
+		}
+		if (dashes === 0) return false;
+		if (line.charAt(index) === ":") index += 1;
+		while (isSpaceChar(line.charAt(index))) index += 1;
+		if (line.charAt(index) !== "|") return index >= line.length;
+		index += 1;
+		let lookahead = index;
+		while (isSpaceChar(line.charAt(lookahead))) lookahead += 1;
+		if (lookahead >= line.length) return true;
+	}
+};
+/**
+* Scan `source` as one table row, upstream's `row_from_string`.
+*
+* Returns `undefined` unless the WHOLE text is consumed and at least one cell
+* came out of it — which is what makes a line either a row or not a row, with
+* no partial verdict in between.
+*/
+const rowFromSource = (source) => {
+	const { text } = source;
+	const length = text.length;
+	let cells = [];
+	let paragraphOffset = 0;
+	let expectMoreCells = true;
+	let offset = scanCellEnd(text, 0);
+	while (offset < length && expectMoreCells) {
+		const cellMatched = scanCell(text, offset);
+		const pipeMatched = scanCellEnd(text, offset + cellMatched);
+		if (cellMatched > 0 || pipeMatched > 0) {
+			let start = offset;
+			while (start > paragraphOffset && text.charAt(start - 1) !== "|") start -= 1;
+			cells.push({
+				start,
+				contentStart: offset,
+				end: offset + cellMatched
+			});
+		}
+		offset += cellMatched + pipeMatched;
+		if (pipeMatched > 0) {
+			expectMoreCells = true;
+			continue;
+		}
+		const rowEnd = scanRowEnd(text, offset);
+		offset += rowEnd;
+		if (rowEnd > 0 && offset !== length) {
+			paragraphOffset = offset;
+			cells = [];
+			offset += scanCellEnd(text, offset);
+			expectMoreCells = true;
+		} else expectMoreCells = false;
+	}
+	if (offset !== length || cells.length === 0) return;
+	return {
+		cells,
+		paragraphOffset
+	};
+};
+/**
+* Cut `[from, to)` out of `source`, dropping the backslash of every `\|` in
+* it — upstream's `unescape_pipes`.
+*
+* Upstream scans left to right and steps over the pipe it just unescaped, so
+* in `\\|` the SECOND backslash is the one that escapes: this is a byte loop,
+* not an understanding of which backslashes are themselves escaped. The
+* lookahead stops at `to`, because past it lies the delimiter pipe that ended
+* the cell and unescaping into it would eat a column boundary.
+*/
+const cellContent = (source, from, to) => {
+	const { text } = source;
+	const drops = [];
+	for (let index = from; index < to; index += 1) if (text.charAt(index) === "\\" && index + 1 < to && text.charAt(index + 1) === "|") {
+		drops.push(index);
+		index += 1;
+	}
+	if (drops.length === 0) return {
+		text: text.slice(from, to),
+		segments: rebase(sliceWithSegments(source.segments, from, to), 0)
+	};
+	let content = "";
+	const segments = [];
+	let cursor = from;
+	for (const drop of [...drops, to]) {
+		if (drop > cursor) {
+			segments.push(...rebase(sliceWithSegments(source.segments, cursor, drop), content.length));
+			content += text.slice(cursor, drop);
+		}
+		cursor = drop + 1;
+	}
+	return {
+		text: content,
+		segments
+	};
+};
+/** Shift a segment run's text offsets so it sits at `at` in a longer string. */
+const rebase = (segments, at) => at === 0 ? segments : segments.map((segment) => ({
+	textOffset: segment.textOffset + at,
+	sourceOffset: segment.sourceOffset,
+	length: segment.length
+}));
+/** The absolute source offset of `index` in a row source. */
+const offsetAt = (source, index, fallback) => {
+	for (const segment of source.segments) {
+		if (index < segment.textOffset) return segment.sourceOffset;
+		if (index <= segment.textOffset + segment.length) return segment.sourceOffset + (index - segment.textOffset);
+	}
+	const last = source.segments[source.segments.length - 1];
+	return last === void 0 ? fallback : last.sourceOffset + last.length;
+};
+/**
+* The alignment each delimiter cell declares: a leading colon means left, a
+* trailing colon right, both center, neither nothing.
+*
+* Upstream reads the first and last byte of the TRIMMED cell buffer, so the
+* whitespace the scanner allows around a marker never reaches this decision.
+*/
+const alignmentsOf = (source, row) => row.cells.map((cell) => {
+	const text = source.text.slice(cell.contentStart, cell.end).trim();
+	const left = text.startsWith(":");
+	const right = text.endsWith(":");
+	if (left && right) return "center";
+	if (left) return "left";
+	if (right) return "right";
+	return null;
+});
+/** The row source for the current line, from `nextNonspace` to its end. */
+const lineSource = (scanner) => {
+	const text = scanner.currentLine.slice(scanner.nextNonspace);
+	return {
+		text: `${text}\n`,
+		segments: [{
+			textOffset: 0,
+			sourceOffset: scanner.lineStart + scanner.nextNonspace,
+			length: text.length
+		}]
+	};
+};
+/** The row source for a paragraph's accumulated content. */
+const paragraphSource = (block) => ({
+	text: block.stringContent,
+	segments: block.segments
+});
+/**
+* Close `block` at `endOffset`.
+*
+* `finalizeBlock` is what moves the tip back up to the parent, but it dates
+* the block to the end of the last line the loop finished — which for a row
+* built inside a block start is the line before. The end is corrected here.
+*/
+const closeAt = (scanner, block, endOffset) => {
+	scanner.finalizeBlock(block, scanner.lineNumber);
+	block.endLine = scanner.lineNumber;
+	block.endOffset = endOffset;
+};
+/**
+* Build one row of `table` from `row`, as a `tableRow` block of `tableCell`
+* children, truncated or padded to the table's column count.
+*/
+const addRow = (scanner, data, source, row, rowStartOffset, rowEndOffset) => {
+	const rowBlock = scanner.addChild("tableRow", rowStartOffset - scanner.lineStart);
+	rowBlock.startOffset = rowStartOffset;
+	const used = Math.min(row.cells.length, data.columns);
+	for (let index = 0; index < used; index += 1) {
+		const cell = row.cells[index];
+		if (cell === void 0) continue;
+		const start = offsetAt(source, cell.start, rowStartOffset);
+		const cellBlock = scanner.addChild("tableCell", start - scanner.lineStart);
+		cellBlock.startOffset = start;
+		const content = cellContent(source, cell.contentStart, cell.end);
+		cellBlock.stringContent = content.text;
+		cellBlock.segments.push(...content.segments);
+		closeAt(scanner, cellBlock, offsetAt(source, cell.end, start));
+	}
+	for (let index = used; index < data.columns; index += 1) {
+		const cellBlock = scanner.addChild("tableCell", rowEndOffset - scanner.lineStart);
+		cellBlock.startOffset = rowEndOffset;
+		closeAt(scanner, cellBlock, rowEndOffset);
+	}
+	data.rows += 1;
+	data.nonemptyCells += used;
+	closeAt(scanner, rowBlock, rowEndOffset);
+};
+/**
+* The header start: a delimiter row under a paragraph promotes that paragraph
+* into a table.
+*
+* Upstream's `try_opening_table_header`.
+*/
+const tableHeaderStart = {
+	name: "tableHeader",
+	trigger: (scanner, container) => {
+		if (scanner.indented || container.type !== "paragraph" || container.data.tableVisited === true) return 0;
+		if (!scanDelimiterRowLine(scanner.currentLine, scanner.nextNonspace)) return 0;
+		const delimiterSource = lineSource(scanner);
+		const delimiterRow = rowFromSource(delimiterSource);
+		if (delimiterRow === void 0) return 0;
+		const headerSource = paragraphSource(container);
+		const headerRow = rowFromSource(headerSource);
+		if (headerRow === void 0 || headerRow.cells.length !== delimiterRow.cells.length) {
+			container.data.tableVisited = true;
+			return 0;
+		}
+		scanner.closeUnmatchedBlocks();
+		const headerStart = offsetAt(headerSource, headerRow.paragraphOffset, container.startOffset);
+		if (headerRow.paragraphOffset > 0) {
+			const reclaimed = cellContent(headerSource, 0, headerRow.paragraphOffset);
+			const paragraph = scanner.insertBefore(container, "paragraph");
+			paragraph.stringContent = reclaimed.text;
+			paragraph.segments.push(...reclaimed.segments);
+			paragraph.endOffset = headerStart;
+			paragraph.endLine = Math.max(scanner.lineNumber - 2, paragraph.startLine);
+		}
+		const table = scanner.replaceBlock(container, "table");
+		table.stringContent = "";
+		table.segments.length = 0;
+		table.startOffset = headerStart;
+		table.startLine = Math.max(scanner.lineNumber - 1, 1);
+		const data = {
+			columns: headerRow.cells.length,
+			align: alignmentsOf(delimiterSource, delimiterRow),
+			rows: 0,
+			nonemptyCells: 0
+		};
+		table.data.tableData = data;
+		const headerEnd = offsetAt(headerSource, headerSource.text.length - 1, headerStart);
+		addRow(scanner, data, headerSource, headerRow, headerStart, headerEnd);
+		scanner.advanceOffset(scanner.currentLine.length - scanner.offset, false);
+		return 2;
+	}
+};
+/**
+* The row start: any non-blank line under an open table is a row.
+*
+* Upstream's `try_opening_table_row`. It sits after every core block start, so
+* a line that opens a blockquote, a list or a fence closes the table instead
+* of becoming a row.
+*/
+const tableRowStart = {
+	name: "tableRow",
+	trigger: (scanner, container) => {
+		if (scanner.indented || scanner.blank || container.type !== "table") return 0;
+		const data = container.data.tableData;
+		if (data === void 0) return 0;
+		if (data.columns * data.rows - data.nonemptyCells > MAX_AUTOCOMPLETED_CELLS) return 0;
+		const source = lineSource(scanner);
+		const row = rowFromSource(source);
+		if (row === void 0) return 0;
+		scanner.closeUnmatchedBlocks();
+		const rowStart = scanner.lineStart + scanner.nextNonspace;
+		addRow(scanner, data, source, row, rowStart, scanner.lineStart + scanner.currentLine.length);
+		scanner.advanceOffset(scanner.currentLine.length - scanner.offset, false);
+		return 2;
+	}
+};
+/** Table: contains rows, and continues for as long as lines scan as rows. */
+const tableConstruct = {
+	type: "table",
+	acceptsLines: false,
+	canContain: (child) => child === "tableRow",
+	continue: (scanner) => {
+		if (scanner.indented) return 1;
+		return rowFromSource(lineSource(scanner)) === void 0 ? 1 : 0;
+	},
+	materialize: (block, children, context) => {
+		const rows = tableRowChildren(children);
+		if (rows.length === 0) return;
+		return Table.make({
+			align: block.data.tableData?.align ?? [],
+			children: rows,
+			position: context.position(block.startOffset, block.endOffset)
+		});
+	}
+};
+/** Table row: contains cells, and is built and closed on the line it opens. */
+const tableRowConstruct = {
+	type: "tableRow",
+	acceptsLines: false,
+	canContain: (child) => child === "tableCell",
+	continue: () => 1,
+	materialize: (block, children, context) => TableRow.make({
+		children: tableCellChildren(children),
+		position: context.position(block.startOffset, block.endOffset)
+	})
+};
+/**
+* Table cell: the block pass's one non-leaf inline host.
+*
+* Its content is the split, unescaped cell text with the source provenance of
+* every surviving character, so it goes through the same `inlineSlice` seam a
+* paragraph does — and inherits the trim, which is what makes the node's
+* position span the cell's content rather than its padding.
+*/
+const tableCellConstruct = {
+	type: "tableCell",
+	acceptsLines: false,
+	canContain: () => false,
+	continue: () => 1,
+	materialize: (block, _children, context) => {
+		const inline = context.inlineSlice(block);
+		const node = TableCell.make({
+			children: inline.children,
+			position: context.position(inline.startOffset, inline.endOffset)
+		});
+		context.registerInline(node, inline);
+		return node;
+	}
+};
+
+//#endregion
+export { tableCellConstruct, tableConstruct, tableHeaderStart, tableRowConstruct, tableRowStart };

package/internal/blocks/taskListItem.js

@@ -0,0 +1,36 @@
+//#region src/internal/blocks/taskListItem.ts
+/**
+* `_scan_tasklist`, as the generated scanner accepts it.
+*
+* Two deliberate differences from the `ext_scanners.re` SOURCE rule, which is
+* stale next to the C it generated: the bracket admits `X` as well as `x`, and
+* `spacechar` is `[ \t\v\f]` — a line terminator does not end the marker, so
+* `- [x]` alone is not a task item.
+*/
+const reTaskListMarker = /^[ \t\v\f]*(?:[-+*]|\d+[^\n])[ \t\v\f]+\[[ xX]\][ \t\v\f]/;
+/** The checkbox is three characters wide; upstream advances exactly that. */
+const MARKER_LENGTH = 3;
+/** Whether `line` carries a checked marker anywhere — upstream's `strstr` pair. */
+const isChecked = (line) => line.includes("[x]") || line.includes("[X]");
+/**
+* The GFM task-list block start: a `[ ]`, `[x]` or `[X]` checkbox opening a
+* list item's content.
+*
+* Registered in the `gfm` dialect only, immediately after the CommonMark
+* starts, which is where cmark-gfm runs its extensions.
+*/
+const taskListItemStart = {
+	name: "taskListItem",
+	trigger: (scanner, container) => {
+		if (container.type !== "listItem") return 0;
+		if (!reTaskListMarker.test(scanner.currentLine)) return 0;
+		container.data.checked = isChecked(scanner.currentLine);
+		scanner.advanceOffset(MARKER_LENGTH, false);
+		scanner.findNextNonspace();
+		scanner.advanceNextNonspace();
+		return 2;
+	}
+};
+
+//#endregion
+export { taskListItemStart };

package/internal/blocks/thematicBreak.js

@@ -0,0 +1,35 @@
+import { ThematicBreak } from "../../MarkdownNode.js";
+
+//#region src/internal/blocks/thematicBreak.ts
+const reThematicBreak = /^(?:\*[ \t]*){3,}$|^(?:_[ \t]*){3,}$|^(?:-[ \t]*){3,}$/;
+const markerCharOf = (char) => char === "-" || char === "_" || char === "*" ? char : void 0;
+/** Thematic break: one line, no children. */
+const thematicBreakConstruct = {
+	type: "thematicBreak",
+	acceptsLines: false,
+	canContain: () => false,
+	continue: () => 1,
+	materialize: (block, _children, context) => {
+		const marker = block.data.markerChar;
+		return ThematicBreak.make({
+			position: context.position(block.startOffset, block.endOffset),
+			...marker === void 0 ? {} : { markerChar: marker }
+		});
+	}
+};
+/** The thematic-break block start: three or more `-`, `_` or `*`. */
+const thematicBreakStart = {
+	name: "thematicBreak",
+	trigger: (scanner) => {
+		if (scanner.indented || !reThematicBreak.test(scanner.currentLine.slice(scanner.nextNonspace))) return 0;
+		scanner.closeUnmatchedBlocks();
+		const container = scanner.addChild("thematicBreak", scanner.nextNonspace);
+		const marker = markerCharOf(scanner.currentLine.charAt(scanner.nextNonspace));
+		if (marker !== void 0) container.data.markerChar = marker;
+		scanner.advanceOffset(scanner.currentLine.length - scanner.offset, false);
+		return 2;
+	}
+};
+
+//#endregion
+export { thematicBreakConstruct, thematicBreakStart };

package/internal/carriers.js

@@ -0,0 +1,42 @@
+//#region src/internal/carriers.ts
+/**
+* P1's error-code vocabulary. Widens as later phases add parse-error kinds;
+* P1 registers exactly one, the hardening-guard trip.
+*/
+const MARKDOWN_PARSE_ERROR_CODES = ["NestingDepthExceeded"];
+/** The engine's carrier for a recoverable-turned-fatal parse condition. */
+var RawMarkdownError = class extends Error {
+	diagnostic;
+	_tag = "RawMarkdownError";
+	constructor(diagnostic) {
+		super(diagnostic.message);
+		this.diagnostic = diagnostic;
+	}
+};
+/** Narrows `unknown` to {@link RawMarkdownError}; never a bare `instanceof` at a call site. */
+const isRawMarkdownError = (u) => u instanceof RawMarkdownError;
+/**
+* Raw guard-trip signal. The engine throws it when a hardening cap
+* (`MAX_NESTING_DEPTH`) is exceeded; it must never escape a public entry
+* point as a defect — the facade catches it and materializes a typed
+* `MarkdownParseError` carrying a `NestingDepthExceeded` diagnostic.
+*/
+var GuardExceeded = class extends Error {
+	reason;
+	limit;
+	actual;
+	offset;
+	_tag = "GuardExceeded";
+	constructor(reason, limit, actual, offset) {
+		super(`${reason}: limit ${limit}, actual ${actual}`);
+		this.reason = reason;
+		this.limit = limit;
+		this.actual = actual;
+		this.offset = offset;
+	}
+};
+/** Narrows `unknown` to {@link GuardExceeded}; never a bare `instanceof` at a call site. */
+const isGuardExceeded = (u) => u instanceof GuardExceeded;
+
+//#endregion
+export { GuardExceeded, MARKDOWN_PARSE_ERROR_CODES, RawMarkdownError, isGuardExceeded, isRawMarkdownError };

package/internal/entities.js

@@ -0,0 +1,26 @@
+import { ENTITY_MAP } from "./entityMap.js";
+
+//#region src/internal/entities.ts
+/** The largest code point Unicode defines. */
+const MAX_CODE_POINT = 1114111;
+/**
+* Decode one character reference, brackets included (`#`, `&`).
+*
+* Returns `undefined` when the reference is not one this engine can decode,
+* which the caller must render as the literal source text — never as an
+* empty string.
+*/
+const decodeEntity = (entity) => {
+	if (!entity.startsWith("&") || !entity.endsWith(";")) return;
+	const body = entity.slice(1, -1);
+	if (!body.startsWith("#")) return ENTITY_MAP.get(body);
+	const hex = body.charAt(1) === "x" || body.charAt(1) === "X";
+	const digits = hex ? body.slice(2) : body.slice(1);
+	if (digits.length === 0 || !(hex ? /^[0-9a-f]+$/i : /^[0-9]+$/).test(digits)) return;
+	const code = Number.parseInt(digits, hex ? 16 : 10);
+	if (code === 0 || code > MAX_CODE_POINT || code >= 55296 && code <= 57343) return "�";
+	return String.fromCodePoint(code);
+};
+
+//#endregion
+export { decodeEntity };