@cube-drone/marquee-parser 0.3.0 → 0.5.0

package/README.md

@@ -15,6 +15,27 @@ const doc = parse("# hello *world*\n");
 version (`UnsupportedVersionError`). Malformed constructs become `invalid_directive` nodes
 with spec'd reasons; confusing inline input degrades to literal text. Nothing is ever eaten.
 
+**For editor tooling** there's `parseWithPositions(source)`, returning
+`{ doc, spans, source }`: the same AST byte-for-byte (positions live in a `WeakMap`
+side-table, never on the nodes, so serialization and the wire contract are untouched), with
+each node mapped to its `[start, end)` extent in **UTF-16 code units** over the *normalized*
+source (`\r\n` → `\n`; the returned `source` is the string offsets refer to — hand it, not
+your raw input, to anything consuming the spans). A span covers its construct
+opener-through-closer, markers included; the interiors of canonicalized text (merged
+literals, resolved escapes) are covered but not subdivided. Built for CodeMirror
+decorations; deliberately outside the cross-implementation conformance corpus (see SPEC.md,
+"Source positions").
+
+Three sharp edges, all consequences of "a span is a *source* extent":
+
+- `slice(span) !== node.value` wherever the source is denser than the text — `a \*b\* c`
+  slices nine characters for a seven-character text node.
+- A construct that crosses a line inside a container **contains that container's prefix**:
+  in `> *a\n> b*`, the emphasis span slices `*a\n> b*`, `> ` and all. Range *marks* are fine;
+  a *replace* decoration over such a range would swallow the quote marker, so split
+  multi-line ranges per line before replacing.
+- Blank lines between blocks belong to no node: coverage has gaps, by design.
+
 You probably want [`@cube-drone/marquee-markup`](https://www.npmjs.com/package/@cube-drone/marquee-markup)
 (batteries included: parse + render + CLI) unless you're building a renderer or tool of your
 own — in which case the AST contract is specified in

package/dist/blocks.d.ts

@@ -3,4 +3,13 @@ export declare const MAX_LIST_DEPTH = 16;
 export declare const MAX_QUOTE_DEPTH = 16;
 export declare const MAX_DIRECTIVE_DEPTH = 8;
 export declare const MAX_TARGET_BYTES = 4096;
-export declare function parseBlocks(body: string): Node[];
+/** A node's source extent: [start, end) in UTF-16 code units over the
+ * front-door-normalized source (shebang line included in the count). */
+export interface Span {
+    start: number;
+    end: number;
+}
+export interface PosTracker {
+    spans: WeakMap<Node, Span>;
+}
+export declare function parseBlocks(body: string, T?: PosTracker | null, baseOffset?: number): Node[];

package/dist/blocks.js

@@ -1,5 +1,10 @@
 // Line-oriented block parser: a faithful port of rust/parser/src/blocks.rs.
 // Where this file and that file disagree, one of them has a conformance bug.
+//
+// Positions (SPEC.md, "Source positions"): when a PosTracker rides along,
+// every node gets a [start, end) span - UTF-16 offsets into the normalized
+// source - recorded in the side-table. The AST itself never changes; the
+// default path pays nothing.
 import { isName, nameLen, parseAttrs, utf8Len } from "./attrs.js";
 import { parseInlines } from "./inlines.js";
 export const MAX_LIST_DEPTH = 16;
@@ -9,22 +14,68 @@ export const MAX_TARGET_BYTES = 4096;
 function peek(cur) {
     return cur.lines[cur.pos];
 }
-export function parseBlocks(body) {
+export function parseBlocks(body, T = null, baseOffset = 0) {
     const lines = body.split("\n");
+    let starts = null;
+    if (T !== null) {
+        starts = [];
+        let off = baseOffset;
+        for (const line of lines) {
+            starts.push(off);
+            off += line.length + 1;
+        }
+    }
     // A final newline terminates the last line rather than creating an empty
     // one (otherwise every trailing newline leaks a blank into fences).
     if (lines.length > 0 && lines[lines.length - 1] === "") {
         lines.pop();
+        starts?.pop();
     }
-    const cur = { lines, pos: 0 };
-    return parseContainer(cur, { dirDepth: 0, listDepth: 0, quoteDepth: 0 }, null);
+    const cur = { lines, starts, pos: 0 };
+    return parseContainer(cur, { dirDepth: 0, listDepth: 0, quoteDepth: 0 }, null, T);
+}
+function lineStartOf(cur, i) {
+    return cur.starts[i];
+}
+function lineEndOf(cur, i) {
+    return cur.starts[i] + cur.lines[i].length;
+}
+function mark(T, cur, node, startLine, endLine) {
+    if (T !== null && cur.starts !== null) {
+        T.spans.set(node, { start: lineStartOf(cur, startLine), end: lineEndOf(cur, endLine) });
+    }
+    return node;
+}
+/** Build the logical inline text (segments joined by real source newlines)
+ * plus the code-point -> source-offset map the inline parser records from. */
+function inlineCtx(T, segs) {
+    const text = segs.map((s) => s.text).join("\n");
+    if (T === null) {
+        return { text, P: null };
+    }
+    const map = [];
+    let end = 0;
+    for (let s = 0; s < segs.length; s += 1) {
+        if (s > 0) {
+            map.push(end); // the joining "\n" is the previous line's real newline
+        }
+        const seg = segs[s];
+        let off = seg.start;
+        for (const ch of seg.text) {
+            map.push(off);
+            off += ch.length;
+        }
+        end = off;
+    }
+    map.push(end);
+    return { text, P: { map, spans: T.spans } };
 }
 /**
  * Parse blocks until the container ends. `openDir` is the name of the
  * directive whose body this is (null at document level and inside
  * blockquotes / list items, which are fresh containers).
  */
-function parseContainer(cur, ctx, openDir) {
+function parseContainer(cur, ctx, openDir, T) {
     const out = [];
     for (;;) {
         const line = peek(cur);
@@ -37,6 +88,7 @@ function parseContainer(cur, ctx, openDir) {
         }
         // Directive open / close / error lines.
         if (line.startsWith(":::")) {
+            const li = cur.pos;
             cur.pos += 1;
             const dir = classifyDirective(line);
             if (dir.kind === "close") {
@@ -46,52 +98,66 @@ function parseContainer(cur, ctx, openDir) {
                 if (openDir !== null && dir.name === openDir) {
                     return out;
                 }
-                out.push(invalid(openDir !== null ? "mismatched_close" : "stray_close"));
+                out.push(mark(T, cur, invalid(openDir !== null ? "mismatched_close" : "stray_close"), li, li));
             }
             else if (dir.kind === "open") {
                 if (ctx.dirDepth >= MAX_DIRECTIVE_DEPTH) {
-                    out.push(invalid("depth_exceeded"));
+                    out.push(mark(T, cur, invalid("depth_exceeded"), li, li));
                     continue;
                 }
                 const attrs = parseAttrs(dir.attrsSrc);
                 if (!attrs.ok) {
-                    out.push(invalid(attrs.reason));
+                    out.push(mark(T, cur, invalid(attrs.reason), li, li));
                 }
                 else {
                     const children = dir.leaf
                         ? []
-                        : parseContainer(cur, { ...ctx, dirDepth: ctx.dirDepth + 1 }, dir.name);
-                    out.push({ type: "directive", name: dir.name, attrs: attrs.attrs, children });
+                        : parseContainer(cur, { ...ctx, dirDepth: ctx.dirDepth + 1 }, dir.name, T);
+                    // A container's close line (or EOF auto-close) has been consumed
+                    // by the child parse: the span runs to the last consumed line.
+                    const endLi = dir.leaf ? li : Math.max(li, cur.pos - 1);
+                    out.push(mark(T, cur, { type: "directive", name: dir.name, attrs: attrs.attrs, children }, li, endLi));
                 }
             }
             else {
-                out.push(invalid(dir.reason));
+                out.push(mark(T, cur, invalid(dir.reason), li, li));
             }
             continue;
         }
         // Comment block: consecutive `%%` lines, raw content.
         if (line.startsWith("%%")) {
+            const li = cur.pos;
             const texts = [];
             for (let l = peek(cur); l !== undefined && l.startsWith("%%"); l = peek(cur)) {
                 const rest = l.slice(2);
                 texts.push(rest.startsWith(" ") ? rest.slice(1) : rest);
                 cur.pos += 1;
             }
-            out.push({ type: "comment", text: texts.join("\n") });
+            out.push(mark(T, cur, { type: "comment", text: texts.join("\n") }, li, cur.pos - 1));
             continue;
         }
         const heading = headingLine(line);
         if (heading !== null) {
+            const li = cur.pos;
             cur.pos += 1;
-            out.push({
-                type: "heading",
-                level: heading.level,
-                children: parseInlines(trimSpaceTab(heading.content)),
-            });
+            const content = trimSpaceTab(heading.content);
+            let P = null;
+            let text = content;
+            if (T !== null && cur.starts !== null) {
+                // Content offset: past the #s and their space, plus leading trim
+                // (all ASCII, so char counts are UTF-16 counts).
+                const lead = heading.content.length - heading.content.replace(/^[ \t]+/, "").length;
+                const start = lineStartOf(cur, li) + heading.level + 1 + lead;
+                const built = inlineCtx(T, [{ text: content, start }]);
+                text = built.text;
+                P = built.P;
+            }
+            out.push(mark(T, cur, { type: "heading", level: heading.level, children: parseInlines(text, P) }, li, li));
             continue;
         }
         const fence = fenceOpen(line);
         if (fence !== null) {
+            const li = cur.pos;
             cur.pos += 1;
             const content = [];
             for (let l = peek(cur); l !== undefined; l = peek(cur)) {
@@ -102,38 +168,44 @@ function parseContainer(cur, ctx, openDir) {
                 content.push(l);
             }
             const text = content.join("\n");
-            out.push(fence.info === null
+            out.push(mark(T, cur, fence.info === null
                 ? { type: "code_block", text }
-                : { type: "code_block", info: fence.info, text });
+                : { type: "code_block", info: fence.info, text }, li, cur.pos - 1));
             continue;
         }
         if (line.startsWith(">") && ctx.quoteDepth < MAX_QUOTE_DEPTH) {
+            const li = cur.pos;
             const innerLines = [];
+            const innerStarts = T !== null && cur.starts !== null ? [] : null;
             for (let l = peek(cur); l !== undefined && l.startsWith(">"); l = peek(cur)) {
                 const rest = l.slice(1);
-                innerLines.push(rest.startsWith(" ") ? rest.slice(1) : rest);
+                const stripped = rest.startsWith(" ") ? 2 : 1;
+                innerLines.push(l.slice(stripped));
+                innerStarts?.push(lineStartOf(cur, cur.pos) + stripped);
                 cur.pos += 1;
             }
-            const sub = { lines: innerLines, pos: 0 };
-            out.push({
+            const sub = { lines: innerLines, starts: innerStarts, pos: 0 };
+            out.push(mark(T, cur, {
                 type: "blockquote",
-                children: parseContainer(sub, { ...ctx, quoteDepth: ctx.quoteDepth + 1 }, null),
-            });
+                children: parseContainer(sub, { ...ctx, quoteDepth: ctx.quoteDepth + 1 }, null, T),
+            }, li, cur.pos - 1));
             continue;
         }
         if (line.replace(/[ \t]+$/, "") === "---") {
+            const li = cur.pos;
             cur.pos += 1;
-            out.push({ type: "thematic_break" });
+            out.push(mark(T, cur, { type: "thematic_break" }, li, li));
             continue;
         }
         if (ctx.listDepth < MAX_LIST_DEPTH) {
             const m = marker(line);
             if (m !== null) {
-                out.push(parseList(cur, ctx, m));
+                out.push(parseList(cur, ctx, m, T));
                 continue;
             }
         }
         // Paragraph: plain lines until blank / container end / block start.
+        const li = cur.pos;
         const plines = [line];
         cur.pos += 1;
         for (let l = peek(cur); l !== undefined; l = peek(cur)) {
@@ -146,11 +218,19 @@ function parseContainer(cur, ctx, openDir) {
         if (plines.length === 1) {
             const target = turbolink(plines[0]);
             if (target !== null) {
-                out.push({ type: "turbolink", target });
+                out.push(mark(T, cur, { type: "turbolink", target }, li, li));
                 continue;
             }
         }
-        out.push({ type: "paragraph", children: parseInlines(plines.join("\n")) });
+        let P = null;
+        let text = plines.join("\n");
+        if (T !== null && cur.starts !== null) {
+            const segs = plines.map((t, idx) => ({ text: t, start: lineStartOf(cur, li + idx) }));
+            const built = inlineCtx(T, segs);
+            text = built.text;
+            P = built.P;
+        }
+        out.push(mark(T, cur, { type: "paragraph", children: parseInlines(text, P) }, li, cur.pos - 1));
     }
     return out;
 }
@@ -220,11 +300,18 @@ function marker(line) {
 }
 /** One list of one kind. A same-column marker of the other kind ends this
  * list (the dispatcher immediately starts the next one). */
-function parseList(cur, ctx, first) {
+function parseList(cur, ctx, first, T) {
+    const tracking = T !== null && cur.starts !== null;
     const ordered = first.ordered;
     const items = [];
+    const listLi = cur.pos;
+    let itemStart = tracking ? lineStartOf(cur, cur.pos) + first.indent : 0;
     let buf = [peek(cur).slice(first.contentIdx)];
+    let bufStarts = tracking ? [lineStartOf(cur, cur.pos) + first.contentIdx] : null;
     cur.pos += 1;
+    const finish = () => {
+        items.push(finishItem(buf, bufStarts, ctx, T, itemStart));
+    };
     for (;;) {
         const line = peek(cur);
         if (line === undefined) {
@@ -252,6 +339,7 @@ function parseList(cur, ctx, first) {
                 break;
             }
             buf.push("");
+            bufStarts?.push(tracking ? lineStartOf(cur, cur.pos) : 0);
             cur.pos += 1;
             continue;
         }
@@ -259,28 +347,46 @@ function parseList(cur, ctx, first) {
             // Content (or a deeper marker) inside the current item; strip the
             // content column. Off-grid extra spaces ride along (floor rule).
             buf.push(line.slice(2));
+            bufStarts?.push(tracking ? lineStartOf(cur, cur.pos) + 2 : 0);
             cur.pos += 1;
             continue;
         }
         // Column 0 or 1 (floors to 0).
         const m = marker(line);
         if (m !== null && m.ordered === ordered) {
-            items.push(finishItem(buf, ctx));
+            finish();
+            itemStart = tracking ? lineStartOf(cur, cur.pos) + m.indent : 0;
             buf = [line.slice(m.contentIdx)];
+            bufStarts = tracking ? [lineStartOf(cur, cur.pos) + m.contentIdx] : null;
             cur.pos += 1;
             continue;
         }
         break; // column-0 block, or a kind switch
     }
-    items.push(finishItem(buf, ctx));
-    return { type: "list", ordered, children: items };
+    finish();
+    const list = { type: "list", ordered, children: items };
+    if (tracking) {
+        T.spans.set(list, {
+            start: lineStartOf(cur, listLi) + first.indent,
+            end: lineEndOf(cur, cur.pos - 1),
+        });
+    }
+    return list;
 }
-function finishItem(buf, ctx) {
-    const sub = { lines: buf, pos: 0 };
-    return {
+function finishItem(buf, bufStarts, ctx, T, itemStart) {
+    const sub = { lines: buf, starts: bufStarts, pos: 0 };
+    const item = {
         type: "list_item",
-        children: parseContainer(sub, { ...ctx, listDepth: ctx.listDepth + 1 }, null),
+        children: parseContainer(sub, { ...ctx, listDepth: ctx.listDepth + 1 }, null, T),
     };
+    if (T !== null && bufStarts !== null) {
+        let end = itemStart;
+        for (let i = 0; i < buf.length; i += 1) {
+            end = Math.max(end, bufStarts[i] + buf[i].length);
+        }
+        T.spans.set(item, { start: itemStart, end });
+    }
+    return item;
 }
 function classifyDirective(line) {
     const rest = line.slice(3).replace(/[ \t]+$/, "");

package/dist/index.d.ts

@@ -1,7 +1,27 @@
+import { type Span } from "./blocks.ts";
 import type { Node } from "./ast.ts";
 export type { Attrs, Node, Reason } from "./ast.ts";
+export type { Span } from "./blocks.ts";
 export declare class UnsupportedVersionError extends Error {
     readonly version: number;
     constructor(version: number);
 }
 export declare function parse(input: string): Node;
+/** parse() plus source positions - the editor-tooling extra, deliberately
+ * OUTSIDE the AST wire contract (SPEC.md, "Source positions"):
+ *
+ * - `doc` is byte-identical to what parse() returns: nodes carry no
+ *   position fields, so serialization, vectors, and goldens are untouched.
+ * - `spans` maps each node object to its [start, end) source extent, in
+ *   UTF-16 code units (what JavaScript strings and CodeMirror speak) over
+ *   `source` - the front-door-NORMALIZED input (\r\n -> \n; the shebang
+ *   line, when present, is included in the offset space).
+ * - Node-level extents are the guarantee (a span covers its opener through
+ *   its closer); the interiors of canonicalized text (merged literals,
+ *   resolved escapes) are covered but not subdivided.
+ */
+export declare function parseWithPositions(input: string): {
+    doc: Node;
+    spans: WeakMap<Node, Span>;
+    source: string;
+};

package/dist/index.js

@@ -22,6 +22,34 @@ export function parse(input) {
     }
     return { type: "document", version, children: parseBlocks(body) };
 }
+/** parse() plus source positions - the editor-tooling extra, deliberately
+ * OUTSIDE the AST wire contract (SPEC.md, "Source positions"):
+ *
+ * - `doc` is byte-identical to what parse() returns: nodes carry no
+ *   position fields, so serialization, vectors, and goldens are untouched.
+ * - `spans` maps each node object to its [start, end) source extent, in
+ *   UTF-16 code units (what JavaScript strings and CodeMirror speak) over
+ *   `source` - the front-door-NORMALIZED input (\r\n -> \n; the shebang
+ *   line, when present, is included in the offset space).
+ * - Node-level extents are the guarantee (a span covers its opener through
+ *   its closer); the interiors of canonicalized text (merged literals,
+ *   resolved escapes) are covered but not subdivided.
+ */
+export function parseWithPositions(input) {
+    const normalized = input.replaceAll("\r\n", "\n").replaceAll("\r", "\n");
+    const [version, body] = stripVersion(normalized);
+    if (version !== 0) {
+        throw new UnsupportedVersionError(version);
+    }
+    const tracker = { spans: new WeakMap() };
+    const doc = {
+        type: "document",
+        version,
+        children: parseBlocks(body, tracker, normalized.length - body.length),
+    };
+    tracker.spans.set(doc, { start: 0, end: normalized.length });
+    return { doc, spans: tracker.spans, source: normalized };
+}
 /** The in-band version declaration: line 1, exactly `#!marquee <integer>`.
  * The default is version 0, forever. */
 function stripVersion(s) {

package/dist/inlines.d.ts

@@ -1,4 +1,11 @@
 import type { Node } from "./ast.ts";
+import { type Span } from "./blocks.ts";
 export declare const MAX_INLINE_DEPTH = 16;
 export declare const MAX_EMOJI_SLUG_BYTES = 64;
-export declare function parseInlines(text: string): Node[];
+/** Positions context: map[cpIndex] = source UTF-16 offset of that code
+ * point in the logical text (one extra entry for the end). */
+export interface PosCtx {
+    map: number[];
+    spans: WeakMap<Node, Span>;
+}
+export declare function parseInlines(text: string, P?: PosCtx | null): Node[];