@vectojs/markdown 0.7.0 → 0.9.0

package/dist/incrementalLex.d.ts

@@ -0,0 +1,172 @@
+/**
+ * Incremental block lexing for a growing (append-only) Markdown source.
+ *
+ * ## The problem this solves
+ *
+ * `marked` has no incremental lexing API, so the obvious streaming strategy is
+ * to re-lex the whole accumulated document on every chunk. That is O(n²) over a
+ * stream, and measured in `comparisons/stream-markdown-smd` it dominates
+ * everything else: a 25 070-char document delivered in 784 chunks cost 434 ms in
+ * Chrome 150 while lexing the finished document **once** cost 0.975 ms. The
+ * parser is linear; the strategy was quadratic.
+ *
+ * This module keeps a **stable block boundary** — a character offset before
+ * which the token list can no longer change — and re-lexes only the text after
+ * it, splicing the result onto the already-stable token prefix. The cost per
+ * chunk becomes O(unstable tail) instead of O(document).
+ *
+ * ## The correctness contract
+ *
+ * {@link lexFull} and {@link lexAppend} must return a token list **deeply
+ * identical** to `marked.lexer(source)` for the same source. Speed is
+ * secondary: a boundary chosen one line too early silently corrupts the token
+ * stream, which is a far worse failure than being slow. `incrementalLex.test.ts`
+ * enforces this by streaming a corpus one character at a time and comparing
+ * against a full lex at **every** intermediate length.
+ *
+ * ## Why the boundary rule is what it is
+ *
+ * The rule: cut immediately after a `space` token that has **at least one token
+ * following it**, and never when a link reference definition exists.
+ *
+ * Three properties of `marked`'s block lexer (18.0.7) make that safe, and each
+ * was measured exhaustively rather than reasoned about:
+ *
+ * 1. **A pushed `space` token always means a blank line.** A lone `\n` is merged
+ *    into the preceding token's `raw` instead of being pushed, so a `space`
+ *    token in the list is a real block separator — never a single line ending.
+ * 2. **For every built-in rule, only the token adjacent to the end of the source
+ *    can still change.** With a token following the `space`, the construct
+ *    before that `space` is committed. This is what rules out the interesting
+ *    failures: an indented code block or a loose list *can* absorb a blank line
+ *    and keep going, and a `paragraph` can still acquire a setext underline —
+ *    but only while it is the last thing in the source. A brute-force sweep over
+ *    14 documents × every prefix length × every cut index found the
+ *    `nFollow >= 1` form safe for every predecessor type (`blockquote`, `code`,
+ *    `heading`, `hr`, `html`, `list`, `paragraph`, `table`) and the
+ *    `nFollow == 0` form unsafe for `code`, `list` and `paragraph` — hence the
+ *    one-token lag.
+ *
+ *    Two things break that property, and both were found by fuzzing rather than
+ *    by reading the rules:
+ *
+ *    **A `list` reaches past a blank line to absorb a following list item.**
+ *    Measured: `'1. ordered\n2. second\n\n\n1.'` is a *single* `list` token with
+ *    `loose: true`, where a splice at the blank line yields
+ *    `[list(loose:false), space, list]`. Three tokens against one, and the
+ *    `loose` flag differs, which changes rendering. So a cut whose next token is
+ *    a `list` is only taken once a *further* token exists after it — at which
+ *    point the list can no longer grow. This is {@link cutIsSettled}.
+ *
+ *    **Our own `blockMath` extension breaks locality in both directions**, which
+ *    is why {@link hasBlockMathOpener} degrades outright:
+ *    - *Forward*: the tokenizer is `/^ {0,3}\$\$([\s\S]+?)\$\$[ \t]*(?:\n|$)/`
+ *      and `[\s\S]+?` crosses blank lines, so an unterminated `$$` reaches
+ *      arbitrarily far ahead. Measured: `'$$\nopen\n\npara\n'` lexes to
+ *      `[paragraph, space, paragraph]`, and appending `'\n$$\n'` collapses all
+ *      three into one `blockMath` token.
+ *    - *Backward*: `blockTokens` clips the text handed to the paragraph
+ *      tokenizer whenever an extension's `startBlock` hook reports a position,
+ *      and sets a flag that merges the **next** paragraph into the clipped one.
+ *      Since `blockMath` supplies `start()`, a `$$` anywhere ahead retroactively
+ *      re-groups paragraphs already emitted. Measured on
+ *      `'Term\n: definition-ish\n| partial | table |\n| --- |\n\nAfter.\n'`
+ *      plus a trailing `'\n$$\nx\n'`: without the extension it is
+ *      `paragraph, paragraph`; with it registered the two become **one** merged
+ *      paragraph. Capping the boundary cannot fix a backwards reach, so any
+ *      line-start `$$` degrades the instance instead. That is only correct
+ *      because `start()` returns `undefined` when no line-start `$$` exists, so
+ *      with none present the clip never fires and the merge is impossible.
+ *
+ *    Any future block-level extension that supplies `start()` or whose tokenizer
+ *    can span a blank line needs the same treatment.
+ * 3. **Link reference definitions break prefix reuse entirely.** `marked`
+ *    collects every `def` while block-lexing and only then resolves reflinks
+ *    across the *whole* document, so a definition arriving late retroactively
+ *    changes inline tokens that are already emitted, and one inside the stable
+ *    prefix is invisible to a suffix lex. Both directions are unfixable by
+ *    boundary placement, so an instance that sees any definition degrades to
+ *    full lexing permanently. The sweep confirms it: `def` was the one
+ *    predecessor type unsafe at `nFollow >= 1` (21/21).
+ *
+ * Degrading is always available and always correct, so a carriage return takes
+ * it: `marked` normalises CR internally, which desyncs every raw-length offset
+ * from the source those offsets are supposed to index.
+ *
+ * ## Why the boundary is verified rather than trusted
+ *
+ * Offsets are derived by summing `raw` lengths, which assumes `raw` strings tile
+ * their source. They usually do, but not always: measured against marked 18.0.7,
+ * a source ending in a bare list marker (`"- a\n- "`) lexes to raw `"- a\n-\n"`,
+ * because the list tokenizer trims the final item and re-adds a newline. So
+ * every advance is **verified** — the text being declared stable must equal the
+ * concatenated `raw` of the tokens covering it — and an advance that fails
+ * verification is simply not taken. That case is transient (the next chunk
+ * completes the item and it tiles again), so declining costs one chunk of window
+ * growth, where degrading would have cost the whole rest of the stream.
+ */
+import { type TokensList } from 'marked';
+/**
+ * Everything needed to extend a lex without redoing it.
+ *
+ * `source` and `tail` are both carried because they serve different masters:
+ * `source` is what a full lex needs if this instance ever degrades (and what the
+ * caller reconciles its own length check against), while `tail` is the unstable
+ * suffix that is actually re-lexed each chunk. Keeping `tail` separately is what
+ * makes the per-chunk string work O(tail) — deriving it as
+ * `source.slice(stableOffset)` each time would force the engine to flatten the
+ * concatenation rope, putting an O(document) memcpy back into the hot path.
+ */
+export interface IncrementalLexCache {
+    /** Full accumulated source these tokens describe. */
+    readonly source: string;
+    /** `source.slice(stableOffset)` — the part still subject to change. */
+    readonly tail: string;
+    /** Complete token list for `source`. */
+    readonly tokens: TokensList;
+    /** Number of leading tokens that can no longer change. */
+    readonly stableCount: number;
+    /** Character offset in `source` at which the stable prefix ends. */
+    readonly stableOffset: number;
+    /** Once set, this instance always full-lexes. Never clears. */
+    readonly degraded: boolean;
+    /** Why it degraded, for tests and diagnostics. `null` while incremental. */
+    readonly degradedReason: DegradeReason | null;
+}
+/** Why an instance gave up on incremental lexing. */
+export type DegradeReason = 
+/** A link reference definition exists; see the module comment. */
+'link-definition'
+/** A carriage return desyncs `raw`-length offsets from source offsets. */
+ | 'carriage-return'
+/** A line-start `$$` lets `blockMath` reach outside its own token. */
+ | 'block-math';
+export interface IncrementalLexResult {
+    /** Deeply identical to `marked.lexer(source)`. */
+    readonly tokens: TokensList;
+    readonly cache: IncrementalLexCache;
+    /**
+     * Characters actually handed to `marked.lexer()`. Equal to `source.length` for
+     * a full lex and to the unstable tail otherwise — so the ratio against the
+     * document length is the direct measure of what the boundary saved.
+     */
+    readonly charsLexed: number;
+    /** Leading tokens taken from the cache rather than re-lexed. */
+    readonly reusedTokens: number;
+}
+/**
+ * Lex `source` from scratch and prepare to extend it incrementally.
+ *
+ * Used for the first request for an instance, and for anything that is not an
+ * append: a `setContent()`, or a resync.
+ */
+export declare function lexFull(source: string): IncrementalLexResult;
+/**
+ * Extend a previous lex with appended text, re-lexing only the unstable tail.
+ *
+ * The caller must guarantee `append` extends exactly `prev.source`. This does not
+ * verify that, because the verification would be an O(document) comparison per
+ * chunk and would defeat the purpose; the worker enforces it structurally by
+ * owning the cache and by checking the caller's `expectedLength` first.
+ */
+export declare function lexAppend(prev: IncrementalLexCache, append: string): IncrementalLexResult;

package/dist/index.d.ts

@@ -1,2 +1,4 @@
 export * from './Markdown';
+export { parseFrontMatterFields, scanFrontMatter } from './frontMatter';
+export type { FrontMatterScan } from './frontMatter';
 export type { IncompleteMarkdownMode, StreamController, StreamControllerOptions, StreamControllerState, StreamPacingOptions, } from './StreamController';