@oh-my-pi/pi-utils 18.4.9 → 18.4.10

package/CHANGELOG.md

@@ -2,6 +2,23 @@
 
 ## [Unreleased]
 
+## [18.4.10] - 2026-10-02
+
+### Added
+
+- Added `startFrom(src, from)` to inline Markdown tokenizer extensions: a start hint that returns the first match at or after `from` (or `undefined`), so long paragraphs stay linear ([#13961](https://github.com/can1357/oh-my-pi/pull/13961) by [@sjawhar](https://github.com/sjawhar)).
+- Added `this.source` and `this.end` for inline Markdown tokenizer extensions: the whole inline source and where the text being lexed ends in it, with one `this` per source that the link labels and emphasis inside it share, so a tokenizer can remember what it already scanned ([#13961](https://github.com/can1357/oh-my-pi/pull/13961) by [@sjawhar](https://github.com/sjawhar)).
+- Added `mathSpanInContext` and `MathSpans` to `math-delimiters`, which find math spans without rescanning a run of unclosed openers ([#13961](https://github.com/can1357/oh-my-pi/pull/13961) by [@sjawhar](https://github.com/sjawhar)).
+
+### Fixed
+
+- Fixed long Markdown paragraphs lexing slowly: a 44 KB paragraph with no blank line now parses in about 3 ms instead of 100 ms ([#13961](https://github.com/can1357/oh-my-pi/pull/13961) by [@sjawhar](https://github.com/sjawhar)).
+- Fixed Markdown paragraphs with many unclosed `[`, `*` or `_`, or with a long address-like word and no dotted domain, lexing slowly: a 40 KB paragraph of each now lexes in 4-24 ms instead of 2-12 s ([#13961](https://github.com/can1357/oh-my-pi/pull/13961) by [@sjawhar](https://github.com/sjawhar)).
+- Fixed Markdown paragraphs of deeply nested emphasis, links or images lexing slowly: 32 KB now lexes in about 50 ms instead of 7 s, and a long word inside every level no longer costs its length once per level, except in nested image labels that hold a backslash escape ([#13961](https://github.com/can1357/oh-my-pi/pull/13961) by [@sjawhar](https://github.com/sjawhar)).
+- Fixed Markdown paragraphs with long or many unclosed runs of backticks, or `<http://` autolinks with no space or `>` after them, lexing slowly: 80 KB of each now lexes in about 50-60 ms instead of seconds (40 KB of one unclosed run took 10 s) ([#13961](https://github.com/can1357/oh-my-pi/pull/13961) by [@sjawhar](https://github.com/sjawhar)).
+- Fixed Markdown paragraphs of nested brackets, URLs with long trailing punctuation, or unclosed HTML tags or comments lexing slowly: 80 KB of each now lexes in under 40 ms instead of 4-30 s ([#13961](https://github.com/can1357/oh-my-pi/pull/13961) by [@sjawhar](https://github.com/sjawhar)).
+- Fixed deeply nested Markdown links and emphasis overflowing the stack early: in a fresh process links now nest about three times as deep before a stack overflow, and emphasis twice as deep ([#13961](https://github.com/can1357/oh-my-pi/pull/13961) by [@sjawhar](https://github.com/sjawhar)).
+
 ## [18.4.9] - 2026-10-01
 
 ### Added

package/dist/types/marked/core.d.ts

@@ -184,16 +184,40 @@ export type TokensList = Token[] & {
 /** Context supplied to extension tokenizers. */
 export interface TokenizerThis {
     lexer: Lexer;
+    /**
+     * For an inline tokenizer, the whole inline source, such as a paragraph's text. The `src` it receives is the part
+     * of it before `end` that starts at `end - src.length`: inside a link label or the text of emphasis, `end` is where
+     * that text ends, right before its closer, and a token there must not reach past it. Inline tokenizers get one
+     * context object per inline source, the same for every call while the lexer works through it and the labels and
+     * emphasis inside it, so a tokenizer can keep state for it in a `WeakMap`.
+     */
+    source?: string;
+    /** For an inline tokenizer, where the text the lexer works through ends in `source` (see `source`). */
+    end?: number;
 }
 /** A tokenizer extension callback. */
 export type TokenizerExtensionFunction = (this: TokenizerThis, src: string, tokens: Token[] | TokensList) => Tokens.Generic | undefined;
 /** A tokenizer extension start hint. */
 export type TokenizerStartFunction = (this: TokenizerThis, src: string) => number | void;
+/**
+ * An offset form of a start hint: the first index at or after `from` in `src` where the extension's tokenizer could
+ * match, or `undefined` (not -1) when there is none; anything else below `from` throws. The test at each index must
+ * read `src` only from that index on, so the answer for `from` is also the answer for every later offset up to it,
+ * and must still pass where `src` goes on past its end: for a link label or the text of emphasis, the lexer asks on
+ * the inline source it lies in, then asks again on the label or text at the index it got, if that lies inside.
+ */
+export type TokenizerStartFromFunction = (this: TokenizerThis, src: string, from: number) => number | undefined;
 /** An inline or block tokenizer extension. */
 export interface TokenizerExtension {
     name: string;
     level: "block" | "inline";
     start?: TokenizerStartFunction;
+    /**
+     * Replaces `start` for inline text when present. Inline lexing calls it on the whole inline source and reuses
+     * the answer until lexing passes it, inside link labels and emphasis too, instead of calling `start` on the rest
+     * of the source at every text step, so a hint that stops at its answer keeps a long paragraph linear.
+     */
+    startFrom?: TokenizerStartFromFunction;
     tokenizer: TokenizerExtensionFunction;
     childTokens?: string[];
 }

package/dist/types/marked/inline-closers.d.ts

@@ -0,0 +1,119 @@
+/**
+ * Where brackets, emphasis, code spans, autolinks and inline HTML close in inline Markdown. The indexes are built
+ * over one root inline source, each on first use, and shared by every source lexed inside it (a link label, the text
+ * of emphasis), so lexing nested sources builds no index of its own and keeps none alive per nesting level.
+ */
+export declare const PUNCTUATION: RegExp;
+/** Whether an odd run of backslashes right before `index` escapes the character there. */
+export declare function escapedAt(src: string, index: number): boolean;
+export declare function canOpenDelimiter(src: string, index: number, width: number, marker: string, previous?: string): boolean;
+export declare function canCloseDelimiter(src: string, index: number, marker: string): boolean;
+/**
+ * Where a bracket opened in the root source closes. A search starts right after the opening bracket, and a bracket
+ * never begins an escape, so the escape pairing from there is the pairing from the start of the root source. The
+ * closer is then the first unescaped closing bracket at or after the start whose depth (openers minus closers
+ * before it, from the start of the root source) equals the depth at the start.
+ */
+declare class BracketDepths {
+    #private;
+    constructor(src: string, open: number, close: number);
+    /** The offset of the bracket closing the one just before `start`, or -1. */
+    closeAfter(start: number): number;
+    /** Whether an unescaped closing bracket lies at or after `from`, right after a bracket, and before `to`. */
+    closesWithin(from: number, to: number): boolean;
+}
+/**
+ * The delimiters the emphasis rule's closer walk visits for one marker and width in the root source, typed the way
+ * the walk types them: a delimiter that can close is a closer, else one that can open is an opener. For width 1 they
+ * are every unescaped marker. For width 2 they are marker pairs taken two at a time from the start of each run of
+ * markers (after it when an odd run of backslashes escapes the start), which is where a walk that reaches the run
+ * from before it enters. The closer for an opener is then the first closer after it whose depth (openers minus
+ * closers before it) equals the depth where its walk begins, the first point where the walk's nesting count would
+ * drop below zero.
+ */
+declare class EmphasisDelimiters {
+    #private;
+    constructor(src: string, marker: string, width: number);
+    /**
+     * Where the emphasis opened by the delimiter at `opener` closes in a source that ends at `end`, or -1 when the
+     * closer the walk meets first does not lie wholly before `end`. The walk reads nothing before its start. Past
+     * `end` it reads only for the source's last delimiter, which cannot open there and closes as it would at the end
+     * of the source: the sources inline lexing nests end right before a marker or "]", and the closing test reads
+     * either punctuation there like the end.
+     */
+    closeFor(opener: number, end: number): number;
+}
+/**
+ * The runs of backticks in the root source, for the code span rule. A span opened by `width` backticks closes at the
+ * first `width` backticks after its opening run that no backslash escapes: the start of a later run of at least
+ * `width`, or, in a run whose first backtick a backslash escapes, the backticks from its `width`th on, since the
+ * search for a closer goes on `width` past an escaped one.
+ */
+declare class CodeRuns {
+    #private;
+    constructor(src: string);
+    /** Where the run of backticks holding the backtick at `at` ends. */
+    endOf(at: number): number;
+    /** Where the closer of a code span opened by `width` backticks that end at `from` starts, or -1. */
+    closerAfter(from: number, width: number): number;
+}
+/** The closer indexes of one root inline source, each built on first use. */
+export declare class CloserIndexes {
+    #private;
+    constructor(src: string);
+    get square(): BracketDepths;
+    get round(): BracketDepths;
+    emphasis(marker: string, width: number): EmphasisDelimiters;
+    /**
+     * For each offset, where the HTML rule's scan for a tag's end from there stops: the first ">" outside quotes,
+     * where a quote runs to the next quote of its kind, or -1 at the end of the source.
+     */
+    get tagEnds(): Int32Array;
+    /** Every offset where "-->" starts, ascending. */
+    get commentEnds(): number[];
+    get codeRuns(): CodeRuns;
+    /**
+     * The first " " or ">" at or after `from`, or `Infinity`. Asked at offsets that never decrease, as the lexer's
+     * position does, so each search starts past the last one's answer.
+     */
+    spaceOrAngleFrom(from: number): number;
+    /** Every offset of a backslash before ASCII punctuation, ascending: where unescaping drops a backslash. */
+    get escapes(): number[];
+}
+/**
+ * The link, emphasis, code span, autolink and inline HTML rules' closer lookups for one inline source, which is the
+ * root source of `indexes` or a part of it that ends at `end`. Offsets are relative to `rest`, a suffix of the
+ * source. A closer the root source places at or past `end` does not exist in the source.
+ */
+export declare class InlineClosers {
+    #private;
+    /** Where the source ends in the root source. */
+    readonly end: number;
+    constructor(indexes: CloserIndexes, end: number);
+    /** The closers of a part of `rest` that ends at `end`, lexed as its own source. */
+    nested(rest: string, end: number): InlineClosers;
+    /** Where the "[" just before `start` closes, or -1. */
+    closeSquare(rest: string, start: number): number;
+    /** Where the "(" just before `start` closes, or -1. */
+    closeRound(rest: string, start: number): number;
+    /** Whether `rest.slice(from, to)`, which starts right after a "[", holds an unescaped "]". */
+    squareCloserWithin(rest: string, from: number, to: number): boolean;
+    /** Whether `rest.slice(from, to)` holds a backslash before ASCII punctuation, which unescaping drops. */
+    escapeWithin(rest: string, from: number, to: number): boolean;
+    /** Where the emphasis opened by the `width` markers at the start of `rest` closes, or -1. */
+    closeEmphasis(rest: string, marker: string, width: number): number;
+    /** How many backticks start `rest`, which starts with one. */
+    codeOpenerWidth(rest: string): number;
+    /** Where the code span opened by the `width` backticks at the start of `rest` closes, or -1. */
+    closeCode(rest: string, width: number): number;
+    /**
+     * Where the URL autolink opened at the start of `rest` ends, its scheme ending at `from`: the first " " or ">"
+     * after the scheme if it is a ">" with text before it, or -1. One past the end of `rest` reads as no ">".
+     */
+    closeUrlAutolink(rest: string, from: number): number;
+    /** Where the HTML tag opened by the "<" at the start of `rest` ends: the first ">" after it outside quotes, or -1. */
+    closeTag(rest: string): number;
+    /** Where the first "-->" after the "<!--" at the start of `rest` starts, or -1. */
+    closeComment(rest: string): number;
+}
+export {};

package/dist/types/marked/inline-text.d.ts

@@ -0,0 +1,70 @@
+/**
+ * The plain-text step of inline lexing over one inline source: where text can end ({@link TextStops}, which also
+ * tells the bare-URL rule where it can match) and how it is appended ({@link TextRun}).
+ */
+import type { Lexer, Token, TokenizerExtension, TokenizerStartFromFunction } from "./core.js";
+/**
+ * The places where plain text can end in one root inline source, as offsets into it, shared by every source lexed
+ * inside it (a link label, the text of emphasis). Each stop is the first offset at or after a search offset where a
+ * test passes, and each test reads the source only from its own offset on, so a found stop stays the answer for every
+ * later search offset up to it. A stop is searched for again only after lexing passes it, starting past it, so all
+ * the text steps of a paragraph, at every nesting level, read it a bounded number of times instead of once per step.
+ * Search offsets never decrease: the lexer asks at its position for the bare-URL rule and one past it for the text
+ * step, its position only grows, and a nested source is lexed from its start to its end before the lexer goes on
+ * past it.
+ *
+ * A nested source is the part of the root source before `end`, and a stop that holds there is a stop of the root
+ * source whose match ends by `end`. The first root stop after an offset tells whether there is one: when its match
+ * runs past `end`, no stop that ends by `end` follows it, for the reason each search gives.
+ */
+declare class RootTextStops {
+    #private;
+    constructor(src: string);
+    /** The first character that can start another token at or after `from`. */
+    tokenCharFrom(from: number): number;
+    /** The first bare-URL scheme at or after `from` that ends by `end`. */
+    schemeFrom(from: number, end: number): number;
+    /** The "@" of the run {@link mailFrom} last found. */
+    get mailAt(): number;
+    /** The first offset at or after `from` that starts a match of `/[A-Za-z0-9._+-]+@/` whose "@" is before `end`. */
+    mailFrom(from: number, end: number): number;
+    /** The first offset at or after `from` that starts a match of `/(?: {2,}|\\)\n/` whose "\n" is before `end`. */
+    hardBreakFrom(from: number, end: number): number;
+    /** Where `extension`'s `startFrom` hint points at or after `from`, asked again only once lexing passes it. */
+    startFrom(extension: TokenizerExtension, startFrom: TokenizerStartFromFunction, lexer: Lexer, from: number): number;
+}
+/**
+ * The places where one inline source's plain text can end, as offsets into that source: its root source's stops
+ * that hold before the source's end. The source is a root source, or a part of one lexed as its own source (a link
+ * label, the text of emphasis), which ends right before the delimiter that closes it.
+ */
+export declare class TextStops {
+    #private;
+    /** The stops of `src`, a root source, or else the part of `root`'s source that ends at `end`. */
+    constructor(src: string, lexer: Lexer, root?: RootTextStops, end?: number);
+    /** The stops of `src`, the part of this source's root source that ends at `end`, lexed as its own source. */
+    nested(src: string, end: number): TextStops;
+    /**
+     * Whether the bare-URL rule can match at `pos`: its first alternative needs a scheme there, and its second a
+     * run of e-mail local characters from `pos` to an "@" (the only "@" such a run can reach) with a dotted domain
+     * right after that "@", which is the same test for every offset of the run.
+     */
+    bareUrlCanStart(pos: number): boolean;
+    /**
+     * Length of the plain text at the start of `rest`, a suffix of the source: the distance to the nearest later
+     * offset where another token could start or an inline extension's start hint points.
+     */
+    textLength(rest: string): number;
+}
+/**
+ * Appends plain text to one inline source's token list, merging into a text token before it as marked does. A text
+ * token this run created grows by taking a longer slice of the source instead of by concatenation, so its raw stays
+ * a flat string: reading its last character (the emphasis rule's look-behind) does not copy the whole token.
+ */
+export declare class TextRun {
+    #private;
+    constructor(src: string);
+    /** Appends the source text from `start` to `end`. */
+    append(tokens: Token[], start: number, end: number): void;
+}
+export {};

package/dist/types/math-delimiters.d.ts

@@ -41,5 +41,27 @@ export declare function mathOpenerAt(source: string, at: number): MathOpener | u
  * though a backslash precedes it.
  */
 export declare function mathSpanAt(source: string, at: number, from?: number): MathSpan | undefined;
+/**
+ * The math spans of one source, found as {@link mathSpanAt} finds them with the escape scan stopping at the opener
+ * (the caller has consumed the escapes before it). A closer scan also answers for the openers of its kind that it
+ * passed, which meet the same closer or none, so asking at every opener of a long run reads the run once.
+ */
+export declare class MathSpans {
+    #private;
+    constructor(source: string);
+    /** The span opened at `at` that ends by `end`, or `undefined`. */
+    spanAt(at: number, end?: number): MathSpan | undefined;
+}
+/**
+ * The span opened at the start of `src`, with `end` counted from there, for a marked inline tokenizer: `src` is the
+ * part of `context.source` before `context.end` that the tokenizer received, and the scans are remembered per
+ * context, which marked keeps for one inline source and the link labels and emphasis inside it. Only a span that ends
+ * by `context.end` counts. Inside a label or emphasis, `context.end` holds its closer (`]`, `*` or `_`), which is not
+ * a digit, so a `$` right before it closes there as it would at the end of the text.
+ */
+export declare function mathSpanInContext(context: {
+    source?: string;
+    end?: number;
+}, src: string): MathSpan | undefined;
 /** The own-line display block starting at offset 0, or `undefined`. */
 export declare function mathBlockAt(source: string): MathBlock | undefined;

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@oh-my-pi/pi-utils",
-	"version": "18.4.9",
+	"version": "18.4.10",
 	"description": "Shared utilities for pi packages",
 	"keywords": [
 		"cli",
@@ -57,7 +57,7 @@
 		"fmt": "oxfmt --no-error-on-unmatched-pattern 'src/**/*.{ts,tsx}' '{test,bench,examples,scripts}/**/*.ts' '*.ts'"
 	},
 	"dependencies": {
-		"@oh-my-pi/pi-natives": "18.4.9"
+		"@oh-my-pi/pi-natives": "18.4.10"
 	},
 	"devDependencies": {
 		"@types/bun": "^1.3.14"