@sigx/lynx-markdown 0.4.5 → 0.4.7

package/dist/parser/blocks.js

@@ -0,0 +1,274 @@
+/**
+ * Block-level parser: source → `BlockNode[]`.
+ *
+ * Line-oriented with a small open-container model. The key streaming-safety
+ * rule lives here: a blank line only separates blocks at the top level and
+ * *outside* an open fenced code block — a blank line inside ``` does not close
+ * it. An unterminated fence parses as a `codeBlock` with `closed: false` and is
+ * still rendered, so streaming code dumps don't flicker when the closing fence
+ * finally arrives.
+ *
+ * Keys are assigned by the caller through a `KeyGen` so nested blocks (list
+ * items, blockquote contents) get stable, unique keys for reconciliation.
+ */
+import { parseInline } from './inline.js';
+import { indentOf, isBlank, isFenceClose, isThematicBreak, matchEmptyListMarker, matchFence, matchHeading, matchListMarker, matchTableDelimiter, splitTableRow, } from './scanner.js';
+export function makeKeyGen(prefix) {
+    let n = 0;
+    return { next: () => `${prefix}-${n++}` };
+}
+/** Parse a complete (or partial) source string into block nodes. */
+export function parseBlocks(src, keys = makeKeyGen('b')) {
+    const lines = src.split('\n');
+    return parseLines(lines, keys);
+}
+function parseLines(lines, keys) {
+    const blocks = [];
+    let i = 0;
+    while (i < lines.length) {
+        if (isBlank(lines[i])) {
+            i++;
+            continue;
+        }
+        const line = lines[i];
+        // Fenced code block.
+        const fence = matchFence(line);
+        if (fence) {
+            const start = i;
+            const content = [];
+            i++;
+            let closed = false;
+            while (i < lines.length) {
+                if (isFenceClose(lines[i], fence)) {
+                    closed = true;
+                    i++;
+                    break;
+                }
+                // Strip the fence's own indentation from content lines.
+                content.push(stripIndent(lines[i], fence.indent));
+                i++;
+            }
+            blocks.push({
+                type: 'codeBlock',
+                key: keys.next(),
+                raw: lines.slice(start, i).join('\n'),
+                ...(fence.info ? { lang: fence.info.split(/\s+/)[0] } : {}),
+                value: content.join('\n'),
+                closed,
+            });
+            continue;
+        }
+        // ATX heading.
+        const heading = matchHeading(line);
+        if (heading) {
+            blocks.push({
+                type: 'heading',
+                key: keys.next(),
+                raw: line,
+                level: heading.level,
+                children: parseInline(heading.text),
+            });
+            i++;
+            continue;
+        }
+        // Thematic break (checked before list so `* * *` isn't a bullet).
+        if (isThematicBreak(line)) {
+            blocks.push({ type: 'thematicBreak', key: keys.next(), raw: line });
+            i++;
+            continue;
+        }
+        // Blockquote.
+        if (/^ {0,3}>/.test(line)) {
+            const start = i;
+            const inner = [];
+            while (i < lines.length && /^ {0,3}>/.test(lines[i])) {
+                inner.push(lines[i].replace(/^ {0,3}> ?/, ''));
+                i++;
+            }
+            blocks.push({
+                type: 'blockquote',
+                key: keys.next(),
+                raw: lines.slice(start, i).join('\n'),
+                children: parseLines(inner, keys),
+            });
+            continue;
+        }
+        // List.
+        const marker = matchListMarker(line) ?? matchEmptyListMarker(line);
+        if (marker) {
+            const result = parseList(lines, i, marker, keys);
+            blocks.push(result.block);
+            i = result.next;
+            continue;
+        }
+        // GFM table: a header line whose successor is a delimiter row.
+        if (line.indexOf('|') !== -1 && i + 1 < lines.length) {
+            const align = matchTableDelimiter(lines[i + 1]);
+            if (align) {
+                const result = parseTable(lines, i, align, keys);
+                blocks.push(result.block);
+                i = result.next;
+                continue;
+            }
+        }
+        // Paragraph: consecutive non-blank lines until a block interrupts.
+        const start = i;
+        const para = [];
+        while (i < lines.length && !isBlank(lines[i]) && !interrupts(lines, i)) {
+            para.push(lines[i]);
+            i++;
+        }
+        blocks.push({
+            type: 'paragraph',
+            key: keys.next(),
+            raw: lines.slice(start, i).join('\n'),
+            children: parseInline(para.join('\n')),
+        });
+    }
+    return blocks;
+}
+/** True when the line at `i` starts a block that interrupts a paragraph. */
+function interrupts(lines, i) {
+    const line = lines[i];
+    if (matchFence(line))
+        return true;
+    if (matchHeading(line))
+        return true;
+    if (isThematicBreak(line))
+        return true;
+    if (/^ {0,3}>/.test(line))
+        return true;
+    // An unordered/ordered-1 list marker interrupts; other ordered starts don't.
+    const m = matchListMarker(line);
+    if (m && (!m.ordered || m.start === 1))
+        return true;
+    if (line.indexOf('|') !== -1 && i + 1 < lines.length && matchTableDelimiter(lines[i + 1])) {
+        return true;
+    }
+    return false;
+}
+function stripIndent(line, n) {
+    let i = 0;
+    while (i < n && line[i] === ' ')
+        i++;
+    return line.slice(i);
+}
+// ---------------------------------------------------------------------------
+// Lists
+// ---------------------------------------------------------------------------
+function sameList(a, b) {
+    return a.ordered === b.ordered && a.delimiter === b.delimiter;
+}
+function parseList(lines, start, first, keys) {
+    const items = [];
+    let i = start;
+    let tight = true;
+    let sawBlankBetween = false;
+    while (i < lines.length) {
+        const m = matchListMarker(lines[i]) ?? matchEmptyListMarker(lines[i]);
+        if (!m || !sameList(m, first) || m.markerIndent >= first.contentIndent)
+            break;
+        const itemLines = [m.content];
+        const contentIndent = m.contentIndent;
+        i++;
+        let pendingBlank = false;
+        while (i < lines.length) {
+            const cur = lines[i];
+            if (isBlank(cur)) {
+                pendingBlank = true;
+                itemLines.push('');
+                i++;
+                continue;
+            }
+            // A new marker for this list ends the current item.
+            const nm = matchListMarker(cur) ?? matchEmptyListMarker(cur);
+            if (nm && sameList(nm, first) && nm.markerIndent < contentIndent) {
+                if (pendingBlank)
+                    sawBlankBetween = true;
+                break;
+            }
+            // Indented continuation (nested content, lazy paragraph, sub-list).
+            if (indentOf(cur) >= contentIndent) {
+                if (pendingBlank) {
+                    tight = false;
+                    pendingBlank = false;
+                }
+                itemLines.push(cur.slice(contentIndent));
+                i++;
+                continue;
+            }
+            // A blank line followed by an unindented, non-marker line ends the list.
+            if (pendingBlank) {
+                sawBlankBetween = true;
+                break;
+            }
+            // Lazy paragraph continuation (no indent, not a new block).
+            if (!interrupts(lines, i)) {
+                itemLines.push(cur);
+                i++;
+                continue;
+            }
+            break;
+        }
+        // Trim a trailing blank line captured at the item boundary.
+        while (itemLines.length && itemLines[itemLines.length - 1] === '')
+            itemLines.pop();
+        const { checked, lines: bodyLines } = extractTask(itemLines);
+        items.push({
+            key: keys.next(),
+            checked,
+            children: parseLines(bodyLines, keys),
+        });
+    }
+    if (sawBlankBetween)
+        tight = false;
+    return {
+        block: {
+            type: 'list',
+            key: keys.next(),
+            raw: lines.slice(start, i).join('\n'),
+            ordered: first.ordered,
+            start: first.start,
+            tight,
+            items,
+        },
+        next: i,
+    };
+}
+/** Detect and strip a GFM task-list marker (`[ ]` / `[x]`) from an item. */
+function extractTask(itemLines) {
+    if (itemLines.length === 0)
+        return { checked: null, lines: itemLines };
+    const m = /^\[([ xX])\]\s+(.*)$/.exec(itemLines[0]);
+    if (!m)
+        return { checked: null, lines: itemLines };
+    const rest = [m[2], ...itemLines.slice(1)];
+    return { checked: m[1].toLowerCase() === 'x', lines: rest };
+}
+// ---------------------------------------------------------------------------
+// Tables
+// ---------------------------------------------------------------------------
+function parseTable(lines, start, align, keys) {
+    const header = splitTableRow(lines[start]).map((c) => ({ children: parseInline(c) }));
+    let i = start + 2; // skip header + delimiter
+    const rows = [];
+    while (i < lines.length && !isBlank(lines[i]) && lines[i].indexOf('|') !== -1 && !interrupts(lines, i)) {
+        const cells = splitTableRow(lines[i]).map((c) => ({ children: parseInline(c) }));
+        // Normalize cell count to the header width.
+        while (cells.length < header.length)
+            cells.push({ children: [] });
+        rows.push(cells.slice(0, header.length));
+        i++;
+    }
+    return {
+        block: {
+            type: 'table',
+            key: keys.next(),
+            raw: lines.slice(start, i).join('\n'),
+            align,
+            header,
+            rows,
+        },
+        next: i,
+    };
+}

package/dist/parser/incremental.d.ts

@@ -0,0 +1,26 @@
+/**
+ * Incremental parsing engine for streaming markdown.
+ *
+ * The win for AI chat output: as the source string grows token-by-token, we
+ * never re-parse or re-render finalized content, and completed blocks keep a
+ * stable identity so the reconciler never remounts (no flicker / reflow).
+ *
+ * Core invariant: for an append-only stream, only the **last** top-level block
+ * can still change — appended text always extends the final block. So when a
+ * parse of the live tail yields N blocks, the first N−1 are finalized: they are
+ * cached by reference and never rebuilt, and the source cursor advances to the
+ * start of the last block. Each subsequent chunk re-parses only that trailing
+ * block region (≈ O(last block), not O(document)).
+ *
+ * Keys are the absolute block index (`b-<i>`), so a block keeps the same key
+ * when it transitions from live → finalized: the reconciler patches in place
+ * rather than remounting.
+ */
+import type { BlockNode } from '../ast.js';
+export interface IncrementalEngine {
+    /** Parse `src`, reusing finalized blocks from prior calls where possible. */
+    parse(src: string): BlockNode[];
+    /** Drop all cached state (e.g. when the source is replaced, not appended). */
+    reset(): void;
+}
+export declare function createIncrementalEngine(): IncrementalEngine;

package/dist/parser/incremental.js

@@ -0,0 +1,65 @@
+/**
+ * Incremental parsing engine for streaming markdown.
+ *
+ * The win for AI chat output: as the source string grows token-by-token, we
+ * never re-parse or re-render finalized content, and completed blocks keep a
+ * stable identity so the reconciler never remounts (no flicker / reflow).
+ *
+ * Core invariant: for an append-only stream, only the **last** top-level block
+ * can still change — appended text always extends the final block. So when a
+ * parse of the live tail yields N blocks, the first N−1 are finalized: they are
+ * cached by reference and never rebuilt, and the source cursor advances to the
+ * start of the last block. Each subsequent chunk re-parses only that trailing
+ * block region (≈ O(last block), not O(document)).
+ *
+ * Keys are the absolute block index (`b-<i>`), so a block keeps the same key
+ * when it transitions from live → finalized: the reconciler patches in place
+ * rather than remounting.
+ */
+import { parseBlocks, makeKeyGen } from './blocks.js';
+import { normalize } from './scanner.js';
+export function createIncrementalEngine() {
+    /** Finalized blocks, frozen and reused by reference across chunks. */
+    let cached = [];
+    /** Source length (in normalized chars) already folded into `cached`. */
+    let cut = 0;
+    /** The normalized source prefix corresponding to `cached` (`src.slice(0, cut)`). */
+    let stableSource = '';
+    const reset = () => {
+        cached = [];
+        cut = 0;
+        stableSource = '';
+    };
+    const assignKey = (block, index) => {
+        block.key = `b-${index}`;
+    };
+    const parse = (src) => {
+        const norm = normalize(src ?? '');
+        // Non-append edit (regenerate, replace): the cached prefix no longer
+        // matches → full re-parse.
+        if (!norm.startsWith(stableSource))
+            reset();
+        const tail = norm.slice(cut);
+        const blocks = parseBlocks(tail, makeKeyGen(`p${cut}`));
+        if (blocks.length === 0) {
+            // Tail is empty / all-blank: nothing live, return finalized prefix.
+            return cached.slice();
+        }
+        // Finalize every block except the last (the only one that can still grow).
+        if (blocks.length > 1) {
+            const finalized = blocks.slice(0, -1);
+            finalized.forEach((b, i) => assignKey(b, cached.length + i));
+            cached = cached.concat(finalized);
+            const last = blocks[blocks.length - 1];
+            const idx = tail.lastIndexOf(last.raw);
+            cut += idx < 0 ? 0 : idx;
+            stableSource = norm.slice(0, cut);
+            assignKey(last, cached.length);
+            return cached.concat(last);
+        }
+        const live = blocks[0];
+        assignKey(live, cached.length);
+        return cached.concat(live);
+    };
+    return { parse, reset };
+}

package/dist/parser/inline.d.ts

@@ -0,0 +1,16 @@
+/**
+ * Inline tokenizer: a markdown text run → `InlineNode[]` (nested spans).
+ *
+ * Precedence (highest first): backtick code spans, backslash escapes, images,
+ * links, angle/bare autolinks, then emphasis (`**`/`__` strong, `*`/`_` em,
+ * `~~` del) resolved with a delimiter-run stack, then hard breaks, then text.
+ *
+ * Robustness for streaming: any unmatched construct at the tail (a lone `**`,
+ * a half-open `[text](`, an unterminated code span) degrades to literal text.
+ * The function never throws.
+ */
+import type { InlineNode } from '../ast.js';
+/** Parse an inline text run into a node array. */
+export declare function parseInline(input: string): InlineNode[];
+/** Flatten inline nodes to their visible text (for image alt). */
+export declare function stripFormatting(text: string): string;