@fuzdev/fuz_ui 0.198.1 → 0.200.0

package/dist/mdz_stream_parser.js

@@ -0,0 +1,354 @@
+/**
+ * Streaming opcode parser for mdz.
+ *
+ * Fed chunks of text (e.g., from LLM output), emits opcodes as rendering
+ * instructions. Makes optimistic assumptions about ambiguous syntax and
+ * emits `revert` opcodes to correct when wrong. Never re-parses.
+ *
+ * The design was independently arrived at but shares goals with
+ * {@link https://bsky.app/profile/pngwn.at/post/3mi527zntb22n @pngwn.at}'s
+ * Penguin-Flavoured Markdown (PFM): restrict the syntax so streaming is tractable,
+ * render optimistically and correct when wrong, emit serializable opcodes
+ * to avoid re-parsing, and keep the opcodes target-agnostic so any renderer
+ * can consume them. mdz diverges in one respect: the Svelte consumer
+ * (`MdzStreamState`) does build a reactive tree from opcodes — the platform
+ * dictates this — but mutations are fine-grained via `$state`, not diffed.
+ *
+ * The parser is split across sibling modules: this file holds the public
+ * `MdzStreamParser` class and the `process_loop` / `process_inline`
+ * orchestrators. Per-category handlers (block / inline / link / url / text)
+ * live in `mdz_stream_parser_*.ts` as free functions taking the shared
+ * `MdzStreamParserState` as first argument.
+ *
+ * Usage:
+ * ```ts
+ * const parser = new MdzStreamParser();
+ * parser.feed('hello **bold');
+ * const ops1 = parser.take_opcodes(); // open Paragraph, text "hello ", open Bold, text "bold"
+ * parser.feed('** world');
+ * const ops2 = parser.take_opcodes(); // close Bold, text " world"
+ * parser.finish();
+ * const ops3 = parser.take_opcodes(); // close Paragraph
+ * ```
+ *
+ * @module
+ */
+import { BACKTICK, ASTERISK, UNDERSCORE, TILDE, NEWLINE, HYPHEN, HASH, LEFT_ANGLE, SLASH, LEFT_BRACKET, RIGHT_BRACKET, H_LOWER, H_UPPER, PERIOD, is_word_char, } from './mdz_helpers.js';
+import { accumulate_text, close_codeblock_at_eof, close_heading, close_paragraph, create_state, ensure_paragraph, find_open, flush_text, handle_paragraph_break, offset, } from './mdz_stream_parser_state.js';
+import { process_codeblock, process_codeblock_forced, try_codeblock_open, try_heading, try_hr, } from './mdz_stream_parser_block.js';
+import { check_close_word_boundary, close_bold, close_single_delimiter, process_code_content, try_bold, try_code, try_italic, try_strikethrough, } from './mdz_stream_parser_inline.js';
+import { try_close_tag, try_complete_link, try_link_open, try_tag_open, } from './mdz_stream_parser_link.js';
+import { complete_pending_url, process_url_content, start_speculative_url, try_auto_path_absolute, try_auto_path_relative, try_auto_url_forced, } from './mdz_stream_parser_url.js';
+import { consume_text_run } from './mdz_stream_parser_text.js';
+/**
+ * Streaming opcode parser for mdz content.
+ * Feed chunks via `feed()`, retrieve opcodes via `take_opcodes()`, call `finish()` at end.
+ *
+ * The opcode sequence is not deterministic across chunk boundaries — the same input
+ * fed in different chunk sizes may produce different `text`/`append_text` splits and
+ * different optimistic/revert sequences. The final tree (via `mdz_opcodes_to_nodes`)
+ * matches the one-shot result for all input except one case: italic (`_..._`) where
+ * the opening and closing delimiters straddle a chunk boundary. Italic is
+ * non-optimistic — it requires a confirmed closer in the buffer — so when the
+ * closer arrives only in a later chunk, the opening `_` has already been emitted
+ * as text and italic cannot apply retroactively. See `try_italic` in
+ * `mdz_stream_parser_inline.ts`.
+ */
+export class MdzStreamParser {
+    #state = create_state();
+    /**
+     * Feed a chunk of text to the parser.
+     * Opcodes are accumulated and retrieved via `take_opcodes()`.
+     */
+    feed(chunk) {
+        const state = this.#state;
+        state.buffer += chunk;
+        process_loop(state, false);
+        // flush any accumulated text so the renderer shows it immediately
+        flush_text(state);
+        // drain processed bytes from the buffer
+        if (state.pos > 0) {
+            state.base_offset += state.pos;
+            state.buffer = state.buffer.slice(state.pos);
+            state.pos = 0;
+        }
+    }
+    /**
+     * Signal end of input. Resolves all pending state: closes open blocks,
+     * reverts unclosed optimistic opens, trims trailing newlines.
+     *
+     * Trailing-newline trimming is handled in one place: `trim_trailing_newline()`
+     * called at the top of `close_paragraph()` and `close_codeblock_at_eof()`,
+     * before either function reverts its inner stack. The trim sees the
+     * just-flushed text node's `last_text_id` (or a still-accumulated `\n`) and
+     * emits a `trim_text` opcode. Revert opcodes only fire after.
+     *
+     * Optimistic-container revert is handled by `close_paragraph` and
+     * `close_heading` (each pops its own inner stack), so no separate
+     * `revert_all_optimistic` is needed — optimistic containers can only exist
+     * inside an open Paragraph or Heading (parser invariant).
+     */
+    finish() {
+        const state = this.#state;
+        // process any remaining buffer — `forced=true` treats ambiguous trailing
+        // bytes as concrete content rather than waiting for more input
+        if (state.buffer.length > 0) {
+            process_loop(state, true);
+        }
+        // finalize pending URL if any (text-first auto-links)
+        if (state.pending_url !== null) {
+            flush_text(state);
+            complete_pending_url(state);
+        }
+        flush_text(state);
+        // close heading if open — internally reverts its inner optimistic stack
+        if (state.in_heading) {
+            close_heading(state);
+        }
+        // close paragraph if open — internally trims trailing \n, reverts its
+        // inner optimistic stack, then pops
+        close_paragraph(state);
+        // close codeblock if open (unclosed at EOF) — internally trims trailing \n
+        if (state.codeblock) {
+            close_codeblock_at_eof(state);
+        }
+    }
+    /**
+     * Drain and return all accumulated opcodes.
+     * Destructive — empties the internal queue. The returned array is owned by the caller.
+     */
+    take_opcodes() {
+        const ops = this.#state.opcodes;
+        this.#state.opcodes = [];
+        return ops;
+    }
+}
+// -- Processing loop --
+/**
+ * Core processing loop. When `forced` is false (normal streaming), returns
+ * when more input is needed. When `forced` is true (EOF), treats buffer end
+ * as content end and never waits for more input.
+ */
+const process_loop = (state, forced) => {
+    // absorb leading newlines at start of input or left over from a block element
+    // close in a prior chunk. Only clear the flag once a non-newline char arrives —
+    // otherwise an all-newline chunk would prematurely drop the flag and let the
+    // next chunk's leading content stick to a `\n` text node.
+    if (state.skip_leading_newlines) {
+        while (state.pos < state.buffer.length && state.buffer.charCodeAt(state.pos) === NEWLINE) {
+            state.pos++;
+        }
+        if (state.pos < state.buffer.length) {
+            state.skip_leading_newlines = false;
+        }
+    }
+    while (state.pos < state.buffer.length) {
+        // codeblock mode
+        if (state.codeblock) {
+            if (forced) {
+                process_codeblock_forced(state);
+                return;
+            }
+            if (!process_codeblock(state))
+                return; // need more input
+            continue;
+        }
+        // optimistic inline code mode — raw scanning for close/revert
+        if (state.in_code) {
+            process_code_content(state);
+            continue;
+        }
+        // text-first URL/path scanning mode
+        if (state.pending_url !== null) {
+            process_url_content(state, forced);
+            continue;
+        }
+        const char_code = state.buffer.charCodeAt(state.pos);
+        // newline handling
+        if (char_code === NEWLINE) {
+            if (state.in_heading) {
+                // newline ends heading
+                flush_text(state);
+                close_heading(state);
+                state.pos++;
+                // absorb consecutive newlines after heading
+                while (state.pos < state.buffer.length && state.buffer.charCodeAt(state.pos) === NEWLINE) {
+                    state.pos++;
+                }
+                if (state.pos >= state.buffer.length) {
+                    state.skip_leading_newlines = true;
+                }
+                state.column = 0;
+                state.prev_char = NEWLINE;
+                continue;
+            }
+            // hold trailing \n when streaming (could be start of \n\n)
+            if (!forced && state.pos + 1 >= state.buffer.length)
+                return;
+            // check for paragraph break (\n\n)
+            if (state.pos + 1 < state.buffer.length &&
+                state.buffer.charCodeAt(state.pos + 1) === NEWLINE) {
+                handle_paragraph_break(state);
+                continue;
+            }
+            // single newline: accumulate as text
+            accumulate_text(state, '\n', offset(state));
+            state.pos++;
+            state.column = 0;
+            state.prev_char = NEWLINE;
+            continue;
+        }
+        // block elements at column 0
+        if (state.column === 0 && !state.in_heading) {
+            if (char_code === HASH) {
+                const r = try_heading(state);
+                if (r === 'consumed')
+                    continue;
+                if (!forced && r === 'need_more')
+                    return;
+            }
+            else if (char_code === HYPHEN) {
+                const r = try_hr(state, forced);
+                if (r === 'consumed')
+                    continue;
+                if (!forced && r === 'need_more')
+                    return;
+            }
+            else if (char_code === BACKTICK) {
+                const r = try_codeblock_open(state);
+                if (r === 'consumed')
+                    continue;
+                if (!forced && r === 'need_more')
+                    return;
+            }
+        }
+        // inline processing
+        if (!process_inline(state, forced))
+            return; // need more input
+    }
+};
+// -- Inline processing --
+/**
+ * Process one inline element. Returns false if more input is needed.
+ * When `forced` is true (EOF processing), skips optimistic opening constructs
+ * (no new bold/code/link/tag opens) but still handles italic (non-optimistic,
+ * safe to try), all closing constructs, and auto-links.
+ * Never returns false in forced mode.
+ */
+const process_inline = (state, forced) => {
+    const char_code = state.buffer.charCodeAt(state.pos);
+    // check for closing delimiters first (if matching open exists)
+    if (char_code === ASTERISK &&
+        state.pos + 1 < state.buffer.length &&
+        state.buffer.charCodeAt(state.pos + 1) === ASTERISK) {
+        const bold_idx = find_open(state, 'Bold');
+        if (bold_idx !== -1) {
+            return close_bold(state, bold_idx);
+        }
+        if (!forced)
+            return try_bold(state);
+    }
+    if (char_code === ASTERISK && !forced) {
+        // could be start of ** — need next char to decide
+        if (state.pos + 1 >= state.buffer.length)
+            return false; // hold for next chunk
+        // single asterisk is text (next char is not *)
+        ensure_paragraph(state);
+        accumulate_text(state, '*', offset(state));
+        state.prev_char = ASTERISK;
+        state.column++;
+        state.pos++;
+        return true;
+    }
+    if (char_code === UNDERSCORE) {
+        const open_idx = find_open(state, 'Italic');
+        if (open_idx !== -1 && check_close_word_boundary(state)) {
+            return close_single_delimiter(state, open_idx);
+        }
+        // Italic is non-optimistic — only opens when closer is confirmed in
+        // the buffer, so it's safe (and necessary) to try in forced mode too.
+        return try_italic(state, forced);
+    }
+    if (char_code === TILDE) {
+        const open_idx = find_open(state, 'Strikethrough');
+        if (open_idx !== -1 && check_close_word_boundary(state)) {
+            return close_single_delimiter(state, open_idx);
+        }
+        if (!forced)
+            return try_strikethrough(state);
+    }
+    if (char_code === BACKTICK && !forced) {
+        return try_code(state);
+    }
+    if (char_code === LEFT_BRACKET && !forced) {
+        return try_link_open(state);
+    }
+    if (char_code === RIGHT_BRACKET && !forced) {
+        const link_idx = find_open(state, 'Link');
+        if (link_idx !== -1) {
+            return try_complete_link(state, link_idx);
+        }
+        // no open link — treat as text
+        ensure_paragraph(state);
+        accumulate_text(state, ']', offset(state));
+        state.prev_char = RIGHT_BRACKET;
+        state.column++;
+        state.pos++;
+        return true;
+    }
+    if (char_code === LEFT_ANGLE) {
+        // closing tags are handled even in forced mode
+        if (state.pos + 1 < state.buffer.length && state.buffer.charCodeAt(state.pos + 1) === SLASH) {
+            const result = try_close_tag(state);
+            if (result === 'consumed')
+                return true;
+            // in forced mode, need_more falls through to text
+            if (!forced && result === 'need_more')
+                return false;
+            // not_match — fall through
+        }
+        if (!forced) {
+            const tag_result = try_tag_open(state);
+            if (tag_result === 'consumed')
+                return true;
+            if (tag_result === 'need_more')
+                return false;
+            // not_match — fall through
+        }
+        // not a valid tag — fall through to text
+    }
+    // auto-detected URLs (text-first speculative prefix matching).
+    // Fires on `h` and `H` — scheme matching is case-insensitive (RFC 3986).
+    if (char_code === H_LOWER || char_code === H_UPPER) {
+        if (forced) {
+            const result = try_auto_url_forced(state);
+            if (result === 'consumed')
+                return true;
+            if (result === 'need_more')
+                return false;
+        }
+        else if (state.prev_char === -1 || !is_word_char(state.prev_char)) {
+            // word boundary — start speculative URL prefix matching
+            start_speculative_url(state);
+            return true;
+        }
+    }
+    // auto-detected paths
+    if (char_code === SLASH) {
+        const result = try_auto_path_absolute(state, forced);
+        if (result === 'consumed')
+            return true;
+        if (result === 'need_more')
+            return false;
+    }
+    if (char_code === PERIOD) {
+        const result = try_auto_path_relative(state, forced);
+        if (result === 'consumed')
+            return true;
+        if (result === 'need_more')
+            return false;
+    }
+    // plain text
+    consume_text_run(state);
+    return true;
+};

package/dist/mdz_stream_parser_block.d.ts

@@ -0,0 +1,76 @@
+/**
+ * Block-element handlers for the streaming mdz parser.
+ *
+ * Each `try_*` returns {@link TryResult} so the orchestrator in
+ * {@link ./mdz_stream_parser.ts} can fall through or wait for more input.
+ *
+ * @module
+ */
+import { type CodeblockState, type MdzStreamParserState, type TryResult } from './mdz_stream_parser_state.js';
+/**
+ * Try to parse a heading at column 0.
+ * Returns true if consumed, false if needs more input, null if definitely not a heading.
+ */
+export declare const try_heading: (state: MdzStreamParserState) => TryResult;
+/**
+ * Try to parse a horizontal rule at column 0.
+ * Returns true if consumed, false if needs more input, null if definitely not an HR.
+ * When `forced` is true, treats buffer end as EOF (no more input coming).
+ */
+export declare const try_hr: (state: MdzStreamParserState, forced?: boolean) => TryResult;
+/**
+ * Try to open a code block at column 0.
+ *
+ * Opens the codeblock optimistically when possible. When the current paragraph
+ * has no prior content, opens immediately on `\`\`\`lang\n` without lookahead,
+ * enabling streaming. When the paragraph has content (text before the fence on
+ * a prior line), falls back to lookahead via `find_closing_fence` to avoid
+ * splitting the paragraph on revert.
+ */
+export declare const try_codeblock_open: (state: MdzStreamParserState) => TryResult;
+/**
+ * Scan buffer from `start` for a valid closing fence with `backtick_count` backticks.
+ * Used as a fallback when the current paragraph has prior content (to avoid
+ * splitting the paragraph on revert). Returns 'found' if a valid closing fence
+ * exists, 'not_found' if the buffer ends before we can determine, 'invalid' if
+ * we can see the full remaining content and there's no valid closing fence.
+ */
+export declare const find_closing_fence: (state: MdzStreamParserState, start: number, backtick_count: number) => "found" | "not_found" | "invalid";
+/**
+ * Emit codeblock content as `text` (first chunk) or `append_text` (subsequent
+ * chunks), updating `cb.text_id` on first emit. `content_start` is the local
+ * buffer position where the content slice began — only used to compute the
+ * global byte offset for the first `text` opcode.
+ */
+export declare const emit_codeblock_text: (state: MdzStreamParserState, cb: CodeblockState, content: string, content_start: number) => void;
+/**
+ * Drain the remaining buffer as codeblock content at EOF, looking for a closing
+ * fence whose terminator is EOF rather than `\n`. Matches `mdz_parse`'s
+ * `#match_code_block`, which allows a fence at the end of the document.
+ *
+ * If a fence is found, the codeblock closes normally. Otherwise the remaining
+ * buffer becomes content and `close_codeblock_at_eof` (in state.ts) handles
+ * the revert-and-wrap dance after process completes.
+ */
+export declare const process_codeblock_forced: (state: MdzStreamParserState) => void;
+/**
+ * Process bytes while in codeblock mode.
+ * Returns true if processing should continue, false if more input needed.
+ */
+export declare const process_codeblock: (state: MdzStreamParserState) => boolean;
+/**
+ * Check if the buffer at current position has a codeblock closing fence.
+ * Returns the position after the fence (including trailing newline) on success,
+ * `-1` when definitely not a fence, or `-2` when more input is needed to decide.
+ *
+ * When `forced` is true, EOF after the fence counts as a valid terminator
+ * (matches `mdz_parse`'s handling of code blocks at end of input).
+ */
+export declare const match_codeblock_close: (state: MdzStreamParserState, backtick_count: number, forced?: boolean) => number;
+/**
+ * Revert an empty codeblock (closing fence found but no content was emitted).
+ * Wraps the opening fence delimiter in a new paragraph and accumulates the
+ * closing fence text for normal paragraph processing.
+ */
+export declare const revert_empty_codeblock: (state: MdzStreamParserState, cb: CodeblockState) => boolean;
+//# sourceMappingURL=mdz_stream_parser_block.d.ts.map

package/dist/mdz_stream_parser_block.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"mdz_stream_parser_block.d.ts","sourceRoot":"../src/lib/","sources":["../src/lib/mdz_stream_parser_block.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAYH,OAAO,EACN,KAAK,cAAc,EACnB,KAAK,oBAAoB,EACzB,KAAK,SAAS,EAQd,MAAM,8BAA8B,CAAC;AAEtC;;;GAGG;AACH,eAAO,MAAM,WAAW,GAAI,OAAO,oBAAoB,KAAG,SAgDzD,CAAC;AAEF;;;;GAIG;AACH,eAAO,MAAM,MAAM,GAAI,OAAO,oBAAoB,EAAE,gBAAc,KAAG,SAqDpE,CAAC;AAEF;;;;;;;;GAQG;AACH,eAAO,MAAM,kBAAkB,GAAI,OAAO,oBAAoB,KAAG,SA4DhE,CAAC;AAEF;;;;;;GAMG;AACH,eAAO,MAAM,kBAAkB,GAC9B,OAAO,oBAAoB,EAC3B,OAAO,MAAM,EACb,gBAAgB,MAAM,KACpB,OAAO,GAAG,WAAW,GAAG,SA0C1B,CAAC;AAEF;;;;;GAKG;AACH,eAAO,MAAM,mBAAmB,GAC/B,OAAO,oBAAoB,EAC3B,IAAI,cAAc,EAClB,SAAS,MAAM,EACf,eAAe,MAAM,KACnB,IAgBF,CAAC;AAEF;;;;;;;;GAQG;AACH,eAAO,MAAM,wBAAwB,GAAI,OAAO,oBAAoB,KAAG,IA8CtE,CAAC;AAEF;;;GAGG;AACH,eAAO,MAAM,iBAAiB,GAAI,OAAO,oBAAoB,KAAG,OAsE/D,CAAC;AAEF;;;;;;;GAOG;AACH,eAAO,MAAM,qBAAqB,GACjC,OAAO,oBAAoB,EAC3B,gBAAgB,MAAM,EACtB,SAAQ,OAAe,KACrB,MA+BF,CAAC;AAEF;;;;GAIG;AACH,eAAO,MAAM,sBAAsB,GAClC,OAAO,oBAAoB,EAC3B,IAAI,cAAc,KAChB,OA4CF,CAAC"}