@fuzdev/mdz 0.1.0 → 0.3.0

package/src/lib/mdz_opcodes_to_nodes.ts

@@ -26,9 +26,12 @@ import type {
 	MdzNodeHeading,
 	MdzNodeCode,
 	MdzNodeListItem,
-} from './mdz.js';
-import type {MdzOpcode, MdzNodeId} from './mdz_opcodes.js';
-import {mdz_extract_single_tag, mdz_heading_id, mdz_merge_adjacent_text} from './mdz_helpers.js';
+	MdzNodeTableRow,
+	MdzNodeTableCell,
+	MdzTableAlign,
+} from './mdz.ts';
+import type {MdzOpcode, MdzNodeId} from './mdz_opcodes.ts';
+import {mdz_extract_single_tag, mdz_heading_id, mdz_merge_adjacent_text} from './mdz_helpers.ts';
 
 interface StackFrame {
 	id: MdzNodeId;
@@ -43,6 +46,8 @@ interface StackFrame {
 	ordered?: boolean;
 	start_number?: number;
 	number?: number;
+	align?: Array<MdzTableAlign>;
+	header?: boolean;
 	// metadata from close opcode (deferred)
 	end?: number;
 	reference?: string;
@@ -76,6 +81,8 @@ export const mdz_opcodes_to_nodes = (opcodes: Array<MdzOpcode>): Array<MdzNode>
 					ordered: op.ordered,
 					start_number: op.start_number,
 					number: op.number,
+					align: op.align,
+					header: op.header,
 				});
 				break;
 			}
@@ -417,6 +424,33 @@ const build_node = (frame: StackFrame, frame_children: Array<MdzNode>): MdzNode
 			return {type: 'Blockquote', children: frame_children, start: frame.start, end};
 		}
 
+		case 'Table': {
+			// rows are the only children — `end` derives from the last, matching
+			// the sync token parser
+			const children = frame_children as Array<MdzNodeTableRow>;
+			const end = children.length > 0 ? children[children.length - 1]!.end : frame.end!;
+			return {type: 'Table', align: frame.align ?? [], children, start: frame.start, end};
+		}
+
+		case 'TableRow': {
+			const children = frame_children as Array<MdzNodeTableCell>;
+			return {
+				type: 'TableRow',
+				header: frame.header ?? false,
+				children,
+				start: frame.start,
+				end: frame.end!,
+			};
+		}
+
+		case 'TableCell':
+			return {
+				type: 'TableCell',
+				children: mdz_merge_adjacent_text(frame_children),
+				start: frame.start,
+				end: frame.end!,
+			};
+
 		case 'Code': {
 			// optimistic inline code container — concatenate children text into leaf MdzNodeCode
 			let content = '';

package/src/lib/mdz_stream_parser.ts

@@ -35,7 +35,7 @@
  * @module
  */
 
-import type {MdzOpcode} from './mdz_opcodes.js';
+import type {MdzOpcode} from './mdz_opcodes.ts';
 import {
 	BACKTICK,
 	ASTERISK,
@@ -44,6 +44,7 @@ import {
 	NEWLINE,
 	HYPHEN,
 	HASH,
+	PIPE,
 	LEFT_ANGLE,
 	RIGHT_ANGLE,
 	SLASH,
@@ -61,7 +62,7 @@ import {
 	mdz_blockquote_line_has_content,
 	mdz_blockquote_strip_width,
 	mdz_remap_segments,
-} from './mdz_helpers.js';
+} from './mdz_helpers.ts';
 import {
 	type MdzStreamParserState,
 	type TryResult,
@@ -80,7 +81,7 @@ import {
 	pop_stack_entry,
 	push_stack_entry,
 	revert_failed_close,
-} from './mdz_stream_parser_state.js';
+} from './mdz_stream_parser_state.ts';
 import {
 	process_codeblock,
 	process_codeblock_forced,
@@ -88,8 +89,9 @@ import {
 	try_codeblock_open,
 	try_heading,
 	try_hr,
-} from './mdz_stream_parser_block.js';
-import {process_list_line, try_list_start} from './mdz_stream_parser_list.js';
+} from './mdz_stream_parser_block.ts';
+import {process_list_line, try_list_start} from './mdz_stream_parser_list.ts';
+import {close_table, process_table_line, try_table_start} from './mdz_stream_parser_table.ts';
 import {
 	check_close_word_boundary,
 	close_delimiter,
@@ -97,13 +99,13 @@ import {
 	try_code,
 	try_double_delimiter,
 	try_italic,
-} from './mdz_stream_parser_inline.js';
+} from './mdz_stream_parser_inline.ts';
 import {
 	try_close_tag,
 	try_complete_link,
 	try_link_open,
 	try_tag_open,
-} from './mdz_stream_parser_link.js';
+} from './mdz_stream_parser_link.ts';
 import {
 	complete_pending_url,
 	process_url_content,
@@ -111,8 +113,8 @@ import {
 	try_auto_path_absolute,
 	try_auto_path_relative,
 	try_auto_url_forced,
-} from './mdz_stream_parser_url.js';
-import {consume_delimiter_as_text, consume_text_run} from './mdz_stream_parser_text.js';
+} from './mdz_stream_parser_url.ts';
+import {consume_delimiter_as_text, consume_text_run} from './mdz_stream_parser_text.ts';
 
 /**
  * Streaming opcode parser for mdz content.
@@ -224,6 +226,11 @@ const finish_state = (state: MdzStreamParserState): void => {
 	if (state.buffer.length > 0) {
 		process_loop(state, true);
 	}
+	// close an open top-level table — forced processing ends it at the last
+	// row, but an already-drained buffer (everything consumed) needs this net
+	if (state.table) {
+		close_table(state);
+	}
 	// close an open blockquote — finishes its nested inner document (an
 	// unclosed fence inside ends at the quote's EOF, which is this) and
 	// cascades recursively through deeper quotes
@@ -558,6 +565,12 @@ const process_loop = (state: MdzStreamParserState, forced: boolean): void => {
 			continue;
 		}
 
+		// open table — each line is a body row or ends the table
+		if (state.table) {
+			if (!process_table_line(state, forced)) return; // need more input
+			continue;
+		}
+
 		const char_code = state.buffer.charCodeAt(state.pos);
 
 		// newline handling
@@ -632,6 +645,10 @@ const process_loop = (state: MdzStreamParserState, forced: boolean): void => {
 				const r = try_blockquote_start(state, forced);
 				if (r === 'consumed') continue;
 				if (!forced && r === 'need_more') return;
+			} else if (char_code === PIPE) {
+				const r = try_table_start(state, forced);
+				if (r === 'consumed') continue;
+				if (!forced && r === 'need_more') return;
 			}
 		}
 

package/src/lib/mdz_stream_parser_block.ts

@@ -17,7 +17,7 @@ import {
 	NEWLINE,
 	SPACE,
 	is_line_whitespace,
-} from './mdz_helpers.js';
+} from './mdz_helpers.ts';
 import {
 	type CodeblockState,
 	type MdzStreamParserState,
@@ -28,7 +28,7 @@ import {
 	flush_text,
 	offset,
 	push_stack_entry,
-} from './mdz_stream_parser_state.js';
+} from './mdz_stream_parser_state.ts';
 
 /**
  * Try to parse a heading at column 0.

package/src/lib/mdz_stream_parser_inline.ts

@@ -14,7 +14,7 @@ import {
 	is_word_char,
 	mdz_blockquote_line_has_content,
 	mdz_match_blockquote_prefix,
-} from './mdz_helpers.js';
+} from './mdz_helpers.ts';
 import {
 	type MdzStreamParserState,
 	accumulate_text,
@@ -28,9 +28,9 @@ import {
 	pop_stack_entry,
 	push_stack_entry,
 	revert_above,
-} from './mdz_stream_parser_state.js';
-import {match_fence_opener, match_list_marker} from './mdz_stream_parser_list.js';
-import {consume_delimiter_as_text} from './mdz_stream_parser_text.js';
+} from './mdz_stream_parser_state.ts';
+import {match_fence_opener, match_list_marker} from './mdz_stream_parser_list.ts';
+import {consume_delimiter_as_text} from './mdz_stream_parser_text.ts';
 
 /**
  * Whether `buffer` contains a paragraph break — a newline followed (across
@@ -339,17 +339,24 @@ export const scan_closer_boundary = (
 /** @nodocs */
 export const try_code = (state: MdzStreamParserState, forced = false): boolean => {
 	const start = state.pos;
-	let i = start + 1; // past opening backtick
+	const search_start = start + 1; // past opening backtick
 
 	// compute search boundary: don't scan past open formatting delimiters
-	const search_limit = code_search_limit(state, start + 1);
+	const search_limit = code_search_limit(state, search_start);
 
-	// scan for closing backtick (must be before newline and search boundary)
-	while (i < state.buffer.length && i < search_limit) {
-		const c = state.buffer.charCodeAt(i);
-		if (c === BACKTICK) {
+	// find the first closing backtick or newline (memoized — a candidate that
+	// holds under open formatting re-enters here on every feed, and a raw scan
+	// would re-cover the same growing region each time, going quadratic on a
+	// single long line)
+	const close_pos = buffer_index_of(state, '`', search_start);
+	const newline_pos = buffer_index_of(state, '\n', search_start);
+	const terminator =
+		close_pos !== -1 && (newline_pos === -1 || close_pos < newline_pos) ? close_pos : newline_pos;
+
+	if (terminator !== -1 && terminator < search_limit) {
+		if (terminator === close_pos) {
 			// found close
-			const content = state.buffer.slice(start + 1, i);
+			const content = state.buffer.slice(search_start, terminator);
 			if (content.length === 0) {
 				// empty code — treat as text
 				consume_delimiter_as_text(state, '``');
@@ -364,20 +371,17 @@ export const try_code = (state: MdzStreamParserState, forced = false): boolean =
 				content,
 				text_type: 'Code',
 				start: offset(state, start),
-				end: offset(state, i + 1),
+				end: offset(state, terminator + 1),
 			});
 			state.active_text_id = null;
-			state.pos = i + 1;
-			state.column += i + 1 - start;
+			state.pos = terminator + 1;
+			state.column += terminator + 1 - start;
 			state.prev_char = BACKTICK;
 			return true;
 		}
-		if (c === NEWLINE) {
-			// inline code can't span lines — treat opening backtick as text
-			consume_delimiter_as_text(state, '`');
-			return true;
-		}
-		i++;
+		// newline before any close — inline code can't span lines
+		consume_delimiter_as_text(state, '`');
+		return true;
 	}
 
 	// hit search limit (open formatting delimiter boundary) — treat backtick as
@@ -386,7 +390,7 @@ export const try_code = (state: MdzStreamParserState, forced = false): boolean =
 	// processed), the one-shot parse rejects that italic and scans unbounded —
 	// see the `MdzStreamParser` TSDoc and the parity suite's expected-divergence
 	// battery
-	if (i < state.buffer.length && i >= search_limit) {
+	if (search_limit < state.buffer.length) {
 		consume_delimiter_as_text(state, '`');
 		return true;
 	}

package/src/lib/mdz_stream_parser_link.ts

@@ -5,12 +5,12 @@
  * @module
  */
 
-import type {MdzNodeTypeContainer} from './mdz_opcodes.js';
+import type {MdzNodeTypeContainer} from './mdz_opcodes.ts';
 import {
 	A_UPPER,
 	LEFT_BRACKET,
 	LEFT_PAREN,
-	NEWLINE,
+	MAX_INLINE_NESTING_DEPTH,
 	RIGHT_ANGLE,
 	RIGHT_BRACKET,
 	RIGHT_PAREN,
@@ -22,12 +22,13 @@ import {
 	is_valid_path_char,
 	mdz_is_safe_reference,
 	mdz_is_url,
-} from './mdz_helpers.js';
+} from './mdz_helpers.ts';
 import {
 	type MdzStreamParserState,
 	type TryResult,
 	accumulate_text,
 	alloc_id,
+	buffer_index_of,
 	emit,
 	ensure_paragraph,
 	flush_text,
@@ -36,12 +37,21 @@ import {
 	pop_stack_entry,
 	push_stack_entry,
 	revert_above,
-} from './mdz_stream_parser_state.js';
+} from './mdz_stream_parser_state.ts';
+import {consume_delimiter_as_text} from './mdz_stream_parser_text.ts';
+import {mdz_debug_work} from './mdz_debug_work.ts';
 
 // -- Links --
 
 /** @nodocs */
 export const try_link_open = (state: MdzStreamParserState): boolean => {
+	// nesting-depth cap: too deep to open — render `[` literal, matching the
+	// sync lexer's `#tokenize_markdown_link` cap (keeps the two parsers agreeing
+	// on where deep `[` stops nesting)
+	if (state.open_link_tag_depth >= MAX_INLINE_NESTING_DEPTH) {
+		consume_delimiter_as_text(state, '[');
+		return true;
+	}
 	flush_text(state);
 	ensure_paragraph(state);
 
@@ -97,65 +107,69 @@ export const try_complete_link = (state: MdzStreamParserState, link_stack_idx: n
 		return true;
 	}
 
-	// found ]( — scan for )
-	let i = bracket_pos + 2; // past ](
-	while (i < state.buffer.length) {
-		const c = state.buffer.charCodeAt(i);
-		if (c === RIGHT_PAREN) {
-			const reference = state.buffer.slice(bracket_pos + 2, i);
-			// validate reference: non-empty and only valid path chars
-			if (!reference.trim()) {
-				abort_link_to_text(state, link_stack_idx, bracket_pos);
-				return true;
-			}
-			let valid = true;
-			for (let j = 0; j < reference.length; j++) {
-				if (!is_valid_path_char(reference.charCodeAt(j))) {
-					valid = false;
-					break;
-				}
-			}
-			if (!valid) {
-				abort_link_to_text(state, link_stack_idx, bracket_pos);
-				return true;
-			}
-			// Reject unsafe protocols (javascript:, data:, etc.) at parse time.
-			// `is_valid_path_char` permits `:` so we have to filter explicitly.
-			if (!mdz_is_safe_reference(reference)) {
-				abort_link_to_text(state, link_stack_idx, bracket_pos);
-				return true;
-			}
-
-			// success — close the link with reference
-			flush_text(state);
-			revert_above(state, link_stack_idx);
-			const entry = pop_stack_entry(state);
-			const link_type = mdz_is_url(reference) ? 'external' : 'internal';
-			emit(state, {
-				type: 'close',
-				id: entry.id,
-				end: offset(state, i + 1),
-				reference,
-				link_type,
-			});
-			state.active_text_id = null;
-			state.pos = i + 1;
-			state.column += i + 1 - bracket_pos;
-			state.prev_char = RIGHT_PAREN;
+	// found ]( — find the reference's terminator: `)` completes, a space or
+	// newline aborts. Memoized searches (see `buffer_index_of`) — a link whose
+	// URL arrives incrementally re-enters here on every feed with `pos` still
+	// at the `]`, and a raw scan would re-cover the growing reference each
+	// time, going quadratic on long streamed URLs.
+	const ref_start = bracket_pos + 2;
+	const close_pos = buffer_index_of(state, ')', ref_start);
+	const space_pos = buffer_index_of(state, ' ', ref_start);
+	const newline_pos = buffer_index_of(state, '\n', ref_start);
+	let invalid_pos = space_pos;
+	if (newline_pos !== -1 && (invalid_pos === -1 || newline_pos < invalid_pos)) {
+		invalid_pos = newline_pos;
+	}
+
+	if (close_pos === -1) {
+		// didn't find ) — need more input or invalid
+		if (invalid_pos === -1) return false; // need more input
+		// found invalid char before any ) — revert
+		abort_link_to_text(state, link_stack_idx, bracket_pos);
+		return true;
+	}
+	if (invalid_pos !== -1 && invalid_pos < close_pos) {
+		// invalid character in URL (simplified check)
+		abort_link_to_text(state, link_stack_idx, bracket_pos);
+		return true;
+	}
+
+	const i = close_pos;
+	const reference = state.buffer.slice(ref_start, i);
+	// validate reference: non-empty and only valid path chars
+	if (!reference.trim()) {
+		abort_link_to_text(state, link_stack_idx, bracket_pos);
+		return true;
+	}
+	for (let j = 0; j < reference.length; j++) {
+		if (!is_valid_path_char(reference.charCodeAt(j))) {
+			abort_link_to_text(state, link_stack_idx, bracket_pos);
 			return true;
 		}
-		if (c === NEWLINE || c === SPACE) {
-			// invalid character in URL (simplified check)
-			break;
-		}
-		i++;
+	}
+	// Reject unsafe protocols (javascript:, data:, etc.) at parse time.
+	// `is_valid_path_char` permits `:` so we have to filter explicitly.
+	if (!mdz_is_safe_reference(reference)) {
+		abort_link_to_text(state, link_stack_idx, bracket_pos);
+		return true;
 	}
 
-	// didn't find ) — need more input or invalid
-	if (i >= state.buffer.length) return false; // need more input
-
-	// found invalid char before ) — revert
-	abort_link_to_text(state, link_stack_idx, bracket_pos);
+	// success — close the link with reference
+	flush_text(state);
+	revert_above(state, link_stack_idx);
+	const entry = pop_stack_entry(state);
+	const link_type = mdz_is_url(reference) ? 'external' : 'internal';
+	emit(state, {
+		type: 'close',
+		id: entry.id,
+		end: offset(state, i + 1),
+		reference,
+		link_type,
+	});
+	state.active_text_id = null;
+	state.pos = i + 1;
+	state.column += i + 1 - bracket_pos;
+	state.prev_char = RIGHT_PAREN;
 	return true;
 };
 
@@ -176,7 +190,19 @@ export const try_tag_open = (state: MdzStreamParserState): TryResult => {
 		return 'not_match';
 	}
 
+	// nesting-depth cap: `<letter…` is a tag open, but we're too deep to nest —
+	// render `<` literal, matching the sync lexer's `#tokenize_tag` cap. Checked
+	// after the closer/hold cases above (a `</…>` closer still closes; a `<` at
+	// the buffer end still holds), so only genuine opens are suppressed.
+	if (state.open_link_tag_depth >= MAX_INLINE_NESTING_DEPTH) {
+		consume_delimiter_as_text(state, '<');
+		return 'consumed';
+	}
+
 	// collect tag name
+	// TODO the name/whitespace scans rescan from `start` on every held retry
+	// of an unclosed tag's open tail — needs a hold watermark; a length cap would
+	// diverge from the sync parser on pathological multi-KB "tag names"
 	const name_start = i;
 	while (i < state.buffer.length && is_tag_name_char(state.buffer.charCodeAt(i))) {
 		i++;
@@ -255,9 +281,16 @@ export const try_close_tag = (state: MdzStreamParserState): TryResult => {
 	const name = state.buffer.slice(name_start, i);
 	i++; // past >
 
+	// O(1) bail when no tag of this name is open — without it, every `</name>`
+	// candidate walks the whole inline stack, which grows one frame per leaked
+	// unclosed `<Tag>` and goes quadratic on tag-dense input (the same shape
+	// `open_counts` guards for the delimiter inlines)
+	if (!state.open_tag_counts?.get(name)) return 'not_match';
+
 	// find matching open tag on stack
 	let found_idx = -1;
 	for (let j = state.stack.length - 1; j >= 0; j--) {
+		if (mdz_debug_work.enabled) mdz_debug_work.total++; // frames walked (mismatched-tags guard)
 		const entry = state.stack[j]!;
 		if (
 			(entry.node_type === 'Element' || entry.node_type === 'Component') &&

package/src/lib/mdz_stream_parser_list.ts

@@ -21,13 +21,15 @@ import {
 	NEWLINE,
 	NINE,
 	PERIOD,
+	PIPE,
 	RIGHT_ANGLE,
 	SPACE,
 	ZERO,
 	is_line_whitespace,
 	mdz_match_blockquote_prefix,
 	mdz_blockquote_line_has_content,
-} from './mdz_helpers.js';
+} from './mdz_helpers.ts';
+import {match_table_in_item_head, open_table_in_item} from './mdz_stream_parser_table.ts';
 import {
 	type MdzStreamParserState,
 	type TryResult,
@@ -43,7 +45,7 @@ import {
 	flush_text,
 	offset,
 	push_stack_entry,
-} from './mdz_stream_parser_state.js';
+} from './mdz_stream_parser_state.ts';
 
 /**
  * A matched list marker: `- ` (unordered) or `N. ` (ordered, 1-9 digits).
@@ -416,10 +418,16 @@ export const process_list_line = (
 
 	if (marker !== null && marker.empty) {
 		// empty item: only at an indent exactly matching an open level of the
-		// same marker type — anything else degrades below
-		const level_idx = state.list_levels.findIndex(
-			(l) => l.indent === indent && l.ordered === marker.ordered,
-		);
+		// same marker type — anything else degrades below (manual loop like
+		// `list_continuation_target` — no per-line closure allocation)
+		let level_idx = -1;
+		for (let li = 0; li < state.list_levels.length; li++) {
+			const l = state.list_levels[li]!;
+			if (l.indent === indent && l.ordered === marker.ordered) {
+				level_idx = li;
+				break;
+			}
+		}
 		if (level_idx !== -1) {
 			close_list_run(state);
 			while (state.list_levels.length - 1 > level_idx) {
@@ -495,6 +503,26 @@ export const process_list_line = (
 		}
 	}
 
+	// a deeper pipe-row line whose next line is a delimiter becomes a table
+	// child of the item it would otherwise continue (the fence/quote precedent,
+	// a two-line lookahead held until the delimiter line arrives)
+	if (indent > 0 && buffer.charCodeAt(j) === PIPE) {
+		const target_idx = list_continuation_target(state, indent);
+		if (target_idx !== -1) {
+			const marker_indent = state.list_levels[target_idx]!.indent;
+			const head = match_table_in_item_head(state, j, marker_indent, forced);
+			if (head === 'pending') return false;
+			if (head !== null) {
+				close_list_run(state);
+				while (state.list_levels.length - 1 > target_idx) {
+					close_list_level(state);
+				}
+				open_table_in_item(state, j, marker_indent, head);
+				return true;
+			}
+		}
+	}
+
 	if (run_current && indent > deepest.indent) {
 		// continuation line: soft break, structural indent stripped
 		return continue_run(state, nl_pos, j);