@fuzdev/fuz_ui 0.204.0 → 0.205.1

package/src/lib/mdz_stream_parser.ts

@@ -1,415 +0,0 @@
-/**
- * Streaming opcode parser for mdz.
- *
- * Fed chunks of text (e.g., from LLM output), emits opcodes as rendering
- * instructions. Makes optimistic assumptions about ambiguous syntax and
- * emits `revert` opcodes to correct when wrong. Never re-parses.
- *
- * The design was independently arrived at but shares goals with
- * {@link https://bsky.app/profile/pngwn.at/post/3mi527zntb22n @pngwn.at}'s
- * Penguin-Flavoured Markdown (PFM): restrict the syntax so streaming is tractable,
- * render optimistically and correct when wrong, emit serializable opcodes
- * to avoid re-parsing, and keep the opcodes target-agnostic so any renderer
- * can consume them. mdz diverges in one respect: the Svelte consumer
- * (`MdzStreamState`) does build a reactive tree from opcodes — the platform
- * dictates this — but mutations are fine-grained via `$state`, not diffed.
- *
- * The parser is split across sibling modules: this file holds the public
- * `MdzStreamParser` class and the `process_loop` / `process_inline`
- * orchestrators. Per-category handlers (block / inline / link / url / text)
- * live in `mdz_stream_parser_*.ts` as free functions taking the shared
- * `MdzStreamParserState` as first argument.
- *
- * Usage:
- * ```ts
- * const parser = new MdzStreamParser();
- * parser.feed('hello **bold');
- * const ops1 = parser.take_opcodes(); // open Paragraph, text "hello ", open Bold, text "bold"
- * parser.feed('** world');
- * const ops2 = parser.take_opcodes(); // close Bold, text " world"
- * parser.finish();
- * const ops3 = parser.take_opcodes(); // close Paragraph
- * ```
- *
- * @module
- */
-
-import type {MdzOpcode} from './mdz_opcodes.js';
-import {
-	BACKTICK,
-	ASTERISK,
-	UNDERSCORE,
-	TILDE,
-	NEWLINE,
-	HYPHEN,
-	HASH,
-	LEFT_ANGLE,
-	SLASH,
-	LEFT_BRACKET,
-	RIGHT_BRACKET,
-	H_LOWER,
-	H_UPPER,
-	PERIOD,
-	is_word_char,
-} from './mdz_helpers.js';
-import {
-	type MdzStreamParserState,
-	accumulate_text,
-	close_codeblock_at_eof,
-	close_heading,
-	close_paragraph,
-	create_state,
-	ensure_paragraph,
-	find_open,
-	flush_text,
-	handle_paragraph_break,
-	offset,
-} from './mdz_stream_parser_state.js';
-import {
-	process_codeblock,
-	process_codeblock_forced,
-	try_codeblock_open,
-	try_heading,
-	try_hr,
-} from './mdz_stream_parser_block.js';
-import {
-	check_close_word_boundary,
-	close_bold,
-	close_single_delimiter,
-	process_code_content,
-	try_bold,
-	try_code,
-	try_italic,
-	try_strikethrough,
-} from './mdz_stream_parser_inline.js';
-import {
-	try_close_tag,
-	try_complete_link,
-	try_link_open,
-	try_tag_open,
-} from './mdz_stream_parser_link.js';
-import {
-	complete_pending_url,
-	process_url_content,
-	start_speculative_url,
-	try_auto_path_absolute,
-	try_auto_path_relative,
-	try_auto_url_forced,
-} from './mdz_stream_parser_url.js';
-import {consume_text_run} from './mdz_stream_parser_text.js';
-
-/**
- * Streaming opcode parser for mdz content.
- * Feed chunks via `feed()`, retrieve opcodes via `take_opcodes()`, call `finish()` at end.
- *
- * The opcode sequence is not deterministic across chunk boundaries — the same input
- * fed in different chunk sizes may produce different `text`/`append_text` splits and
- * different optimistic/revert sequences. The final tree (via `mdz_opcodes_to_nodes`)
- * matches the one-shot result for all input except one case: italic (`_..._`) where
- * the opening and closing delimiters straddle a chunk boundary. Italic is
- * non-optimistic — it requires a confirmed closer in the buffer — so when the
- * closer arrives only in a later chunk, the opening `_` has already been emitted
- * as text and italic cannot apply retroactively. See `try_italic` in
- * `mdz_stream_parser_inline.ts`.
- */
-export class MdzStreamParser {
-	#state: MdzStreamParserState = create_state();
-
-	/**
-	 * Feed a chunk of text to the parser.
-	 * Opcodes are accumulated and retrieved via `take_opcodes()`.
-	 */
-	feed(chunk: string): void {
-		const state = this.#state;
-		state.buffer += chunk;
-		process_loop(state, false);
-		// flush any accumulated text so the renderer shows it immediately
-		flush_text(state);
-		// drain processed bytes from the buffer
-		if (state.pos > 0) {
-			state.base_offset += state.pos;
-			state.buffer = state.buffer.slice(state.pos);
-			state.pos = 0;
-		}
-	}
-
-	/**
-	 * Signal end of input. Resolves all pending state: closes open blocks,
-	 * reverts unclosed optimistic opens, trims trailing newlines.
-	 *
-	 * Trailing-newline trimming is handled in one place: `trim_trailing_newline()`
-	 * called at the top of `close_paragraph()` and `close_codeblock_at_eof()`,
-	 * before either function reverts its inner stack. The trim sees the
-	 * just-flushed text node's `last_text_id` (or a still-accumulated `\n`) and
-	 * emits a `trim_text` opcode. Revert opcodes only fire after.
-	 *
-	 * Optimistic-container revert is handled by `close_paragraph` and
-	 * `close_heading` (each pops its own inner stack), so no separate
-	 * `revert_all_optimistic` is needed — optimistic containers can only exist
-	 * inside an open Paragraph or Heading (parser invariant).
-	 */
-	finish(): void {
-		const state = this.#state;
-		// process any remaining buffer — `forced=true` treats ambiguous trailing
-		// bytes as concrete content rather than waiting for more input
-		if (state.buffer.length > 0) {
-			process_loop(state, true);
-		}
-		// finalize pending URL if any (text-first auto-links)
-		if (state.pending_url !== null) {
-			flush_text(state);
-			complete_pending_url(state);
-		}
-		flush_text(state);
-		// close heading if open — internally reverts its inner optimistic stack
-		if (state.in_heading) {
-			close_heading(state);
-		}
-		// close paragraph if open — internally trims trailing \n, reverts its
-		// inner optimistic stack, then pops
-		close_paragraph(state);
-		// close codeblock if open (unclosed at EOF) — internally trims trailing \n
-		if (state.codeblock) {
-			close_codeblock_at_eof(state);
-		}
-	}
-
-	/**
-	 * Drain and return all accumulated opcodes.
-	 * Destructive — empties the internal queue. The returned array is owned by the caller.
-	 */
-	take_opcodes(): Array<MdzOpcode> {
-		const ops = this.#state.opcodes;
-		this.#state.opcodes = [];
-		return ops;
-	}
-}
-
-// -- Processing loop --
-
-/**
- * Core processing loop. When `forced` is false (normal streaming), returns
- * when more input is needed. When `forced` is true (EOF), treats buffer end
- * as content end and never waits for more input.
- */
-const process_loop = (state: MdzStreamParserState, forced: boolean): void => {
-	// absorb leading newlines at start of input or left over from a block element
-	// close in a prior chunk. Only clear the flag once a non-newline char arrives —
-	// otherwise an all-newline chunk would prematurely drop the flag and let the
-	// next chunk's leading content stick to a `\n` text node.
-	if (state.skip_leading_newlines) {
-		while (state.pos < state.buffer.length && state.buffer.charCodeAt(state.pos) === NEWLINE) {
-			state.pos++;
-		}
-		if (state.pos < state.buffer.length) {
-			state.skip_leading_newlines = false;
-		}
-	}
-	while (state.pos < state.buffer.length) {
-		// codeblock mode
-		if (state.codeblock) {
-			if (forced) {
-				process_codeblock_forced(state);
-				return;
-			}
-			if (!process_codeblock(state)) return; // need more input
-			continue;
-		}
-
-		// optimistic inline code mode — raw scanning for close/revert
-		if (state.in_code) {
-			process_code_content(state);
-			continue;
-		}
-
-		// text-first URL/path scanning mode
-		if (state.pending_url !== null) {
-			process_url_content(state, forced);
-			continue;
-		}
-
-		const char_code = state.buffer.charCodeAt(state.pos);
-
-		// newline handling
-		if (char_code === NEWLINE) {
-			if (state.in_heading) {
-				// newline ends heading
-				flush_text(state);
-				close_heading(state);
-				state.pos++;
-				// absorb consecutive newlines after heading
-				while (state.pos < state.buffer.length && state.buffer.charCodeAt(state.pos) === NEWLINE) {
-					state.pos++;
-				}
-				if (state.pos >= state.buffer.length) {
-					state.skip_leading_newlines = true;
-				}
-				state.column = 0;
-				state.prev_char = NEWLINE;
-				continue;
-			}
-			// hold trailing \n when streaming (could be start of \n\n)
-			if (!forced && state.pos + 1 >= state.buffer.length) return;
-			// check for paragraph break (\n\n)
-			if (
-				state.pos + 1 < state.buffer.length &&
-				state.buffer.charCodeAt(state.pos + 1) === NEWLINE
-			) {
-				handle_paragraph_break(state);
-				continue;
-			}
-			// single newline: accumulate as text
-			accumulate_text(state, '\n', offset(state));
-			state.pos++;
-			state.column = 0;
-			state.prev_char = NEWLINE;
-			continue;
-		}
-
-		// block elements at column 0
-		if (state.column === 0 && !state.in_heading) {
-			if (char_code === HASH) {
-				const r = try_heading(state);
-				if (r === 'consumed') continue;
-				if (!forced && r === 'need_more') return;
-			} else if (char_code === HYPHEN) {
-				const r = try_hr(state, forced);
-				if (r === 'consumed') continue;
-				if (!forced && r === 'need_more') return;
-			} else if (char_code === BACKTICK) {
-				const r = try_codeblock_open(state);
-				if (r === 'consumed') continue;
-				if (!forced && r === 'need_more') return;
-			}
-		}
-
-		// inline processing
-		if (!process_inline(state, forced)) return; // need more input
-	}
-};
-
-// -- Inline processing --
-
-/**
- * Process one inline element. Returns false if more input is needed.
- * When `forced` is true (EOF processing), skips optimistic opening constructs
- * (no new bold/code/link/tag opens) but still handles italic (non-optimistic,
- * safe to try), all closing constructs, and auto-links.
- * Never returns false in forced mode.
- */
-const process_inline = (state: MdzStreamParserState, forced: boolean): boolean => {
-	const char_code = state.buffer.charCodeAt(state.pos);
-
-	// check for closing delimiters first (if matching open exists)
-	if (
-		char_code === ASTERISK &&
-		state.pos + 1 < state.buffer.length &&
-		state.buffer.charCodeAt(state.pos + 1) === ASTERISK
-	) {
-		const bold_idx = find_open(state, 'Bold');
-		if (bold_idx !== -1) {
-			return close_bold(state, bold_idx);
-		}
-		if (!forced) return try_bold(state);
-	}
-
-	if (char_code === ASTERISK && !forced) {
-		// could be start of ** — need next char to decide
-		if (state.pos + 1 >= state.buffer.length) return false; // hold for next chunk
-		// single asterisk is text (next char is not *)
-		ensure_paragraph(state);
-		accumulate_text(state, '*', offset(state));
-		state.prev_char = ASTERISK;
-		state.column++;
-		state.pos++;
-		return true;
-	}
-
-	if (char_code === UNDERSCORE) {
-		const open_idx = find_open(state, 'Italic');
-		if (open_idx !== -1 && check_close_word_boundary(state)) {
-			return close_single_delimiter(state, open_idx);
-		}
-		// Italic is non-optimistic — only opens when closer is confirmed in
-		// the buffer, so it's safe (and necessary) to try in forced mode too.
-		return try_italic(state, forced);
-	}
-
-	if (char_code === TILDE) {
-		const open_idx = find_open(state, 'Strikethrough');
-		if (open_idx !== -1 && check_close_word_boundary(state)) {
-			return close_single_delimiter(state, open_idx);
-		}
-		if (!forced) return try_strikethrough(state);
-	}
-
-	if (char_code === BACKTICK && !forced) {
-		return try_code(state);
-	}
-
-	if (char_code === LEFT_BRACKET && !forced) {
-		return try_link_open(state);
-	}
-
-	if (char_code === RIGHT_BRACKET && !forced) {
-		const link_idx = find_open(state, 'Link');
-		if (link_idx !== -1) {
-			return try_complete_link(state, link_idx);
-		}
-		// no open link — treat as text
-		ensure_paragraph(state);
-		accumulate_text(state, ']', offset(state));
-		state.prev_char = RIGHT_BRACKET;
-		state.column++;
-		state.pos++;
-		return true;
-	}
-
-	if (char_code === LEFT_ANGLE) {
-		// closing tags are handled even in forced mode
-		if (state.pos + 1 < state.buffer.length && state.buffer.charCodeAt(state.pos + 1) === SLASH) {
-			const result = try_close_tag(state);
-			if (result === 'consumed') return true;
-			// in forced mode, need_more falls through to text
-			if (!forced && result === 'need_more') return false;
-			// not_match — fall through
-		}
-		if (!forced) {
-			const tag_result = try_tag_open(state);
-			if (tag_result === 'consumed') return true;
-			if (tag_result === 'need_more') return false;
-			// not_match — fall through
-		}
-		// not a valid tag — fall through to text
-	}
-
-	// auto-detected URLs (text-first speculative prefix matching).
-	// Fires on `h` and `H` — scheme matching is case-insensitive (RFC 3986).
-	if (char_code === H_LOWER || char_code === H_UPPER) {
-		if (forced) {
-			const result = try_auto_url_forced(state);
-			if (result === 'consumed') return true;
-			if (result === 'need_more') return false;
-		} else if (state.prev_char === -1 || !is_word_char(state.prev_char)) {
-			// word boundary — start speculative URL prefix matching
-			start_speculative_url(state);
-			return true;
-		}
-	}
-
-	// auto-detected paths
-	if (char_code === SLASH) {
-		const result = try_auto_path_absolute(state, forced);
-		if (result === 'consumed') return true;
-		if (result === 'need_more') return false;
-	}
-	if (char_code === PERIOD) {
-		const result = try_auto_path_relative(state, forced);
-		if (result === 'consumed') return true;
-		if (result === 'need_more') return false;
-	}
-
-	// plain text
-	consume_text_run(state);
-	return true;
-};