@cube-drone/marquee-parser 0.3.0 → 0.5.0

package/src/blocks.ts

@@ -1,15 +1,31 @@
 // Line-oriented block parser: a faithful port of rust/parser/src/blocks.rs.
 // Where this file and that file disagree, one of them has a conformance bug.
+//
+// Positions (SPEC.md, "Source positions"): when a PosTracker rides along,
+// every node gets a [start, end) span - UTF-16 offsets into the normalized
+// source - recorded in the side-table. The AST itself never changes; the
+// default path pays nothing.
 
 import type { Node, Reason } from "./ast.ts";
 import { isName, nameLen, parseAttrs, utf8Len } from "./attrs.ts";
-import { parseInlines } from "./inlines.ts";
+import { parseInlines, type PosCtx } from "./inlines.ts";
 
 export const MAX_LIST_DEPTH = 16;
 export const MAX_QUOTE_DEPTH = 16;
 export const MAX_DIRECTIVE_DEPTH = 8;
 export const MAX_TARGET_BYTES = 4096;
 
+/** A node's source extent: [start, end) in UTF-16 code units over the
+ * front-door-normalized source (shebang line included in the count). */
+export interface Span {
+  start: number;
+  end: number;
+}
+
+export interface PosTracker {
+  spans: WeakMap<Node, Span>;
+}
+
 interface Ctx {
   dirDepth: number;
   listDepth: number;
@@ -18,6 +34,9 @@ interface Ctx {
 
 interface Cursor {
   lines: string[];
+  /** Parallel to `lines`: source offset of each line's first character
+   * (post any container prefix-stripping). Null when positions are off. */
+  starts: number[] | null;
   pos: number;
 }
 
@@ -25,15 +44,68 @@ function peek(cur: Cursor): string | undefined {
   return cur.lines[cur.pos];
 }
 
-export function parseBlocks(body: string): Node[] {
+export function parseBlocks(body: string, T: PosTracker | null = null, baseOffset = 0): Node[] {
   const lines = body.split("\n");
+  let starts: number[] | null = null;
+  if (T !== null) {
+    starts = [];
+    let off = baseOffset;
+    for (const line of lines) {
+      starts.push(off);
+      off += line.length + 1;
+    }
+  }
   // A final newline terminates the last line rather than creating an empty
   // one (otherwise every trailing newline leaks a blank into fences).
   if (lines.length > 0 && lines[lines.length - 1] === "") {
     lines.pop();
+    starts?.pop();
+  }
+  const cur: Cursor = { lines, starts, pos: 0 };
+  return parseContainer(cur, { dirDepth: 0, listDepth: 0, quoteDepth: 0 }, null, T);
+}
+
+function lineStartOf(cur: Cursor, i: number): number {
+  return cur.starts![i]!;
+}
+
+function lineEndOf(cur: Cursor, i: number): number {
+  return cur.starts![i]! + cur.lines[i]!.length;
+}
+
+function mark(T: PosTracker | null, cur: Cursor, node: Node, startLine: number, endLine: number): Node {
+  if (T !== null && cur.starts !== null) {
+    T.spans.set(node, { start: lineStartOf(cur, startLine), end: lineEndOf(cur, endLine) });
+  }
+  return node;
+}
+
+/** Build the logical inline text (segments joined by real source newlines)
+ * plus the code-point -> source-offset map the inline parser records from. */
+function inlineCtx(
+  T: PosTracker | null,
+  segs: Array<{ text: string; start: number }>,
+): { text: string; P: PosCtx | null } {
+  const text = segs.map((s) => s.text).join("\n");
+  if (T === null) {
+    return { text, P: null };
+  }
+  const map: number[] = [];
+  let end = 0;
+  for (let s = 0; s < segs.length; s += 1) {
+    if (s > 0) {
+      map.push(end); // the joining "\n" is the previous line's real newline
+    }
+    const seg = segs[s]!;
+    let off = seg.start;
+    for (const ch of seg.text) {
+      map.push(off);
+      off += ch.length;
+    }
+    end = off;
   }
-  const cur: Cursor = { lines, pos: 0 };
-  return parseContainer(cur, { dirDepth: 0, listDepth: 0, quoteDepth: 0 }, null);
+  map.push(end);
+  return { text, P: { map, spans: T.spans } };
 }
 
 /**
@@ -41,7 +113,7 @@ export function parseBlocks(body: string): Node[] {
  * directive whose body this is (null at document level and inside
  * blockquotes / list items, which are fresh containers).
  */
-function parseContainer(cur: Cursor, ctx: Ctx, openDir: string | null): Node[] {
+function parseContainer(cur: Cursor, ctx: Ctx, openDir: string | null, T: PosTracker | null): Node[] {
   const out: Node[] = [];
   for (;;) {
     const line = peek(cur);
@@ -55,6 +127,7 @@ function parseContainer(cur: Cursor, ctx: Ctx, openDir: string | null): Node[] {
 
     // Directive open / close / error lines.
     if (line.startsWith(":::")) {
+      const li = cur.pos;
       cur.pos += 1;
       const dir = classifyDirective(line);
       if (dir.kind === "close") {
@@ -64,52 +137,76 @@ function parseContainer(cur: Cursor, ctx: Ctx, openDir: string | null): Node[] {
         if (openDir !== null && dir.name === openDir) {
           return out;
         }
-        out.push(invalid(openDir !== null ? "mismatched_close" : "stray_close"));
+        out.push(mark(T, cur, invalid(openDir !== null ? "mismatched_close" : "stray_close"), li, li));
       } else if (dir.kind === "open") {
         if (ctx.dirDepth >= MAX_DIRECTIVE_DEPTH) {
-          out.push(invalid("depth_exceeded"));
+          out.push(mark(T, cur, invalid("depth_exceeded"), li, li));
           continue;
         }
         const attrs = parseAttrs(dir.attrsSrc);
         if (!attrs.ok) {
-          out.push(invalid(attrs.reason));
+          out.push(mark(T, cur, invalid(attrs.reason), li, li));
         } else {
           const children = dir.leaf
             ? []
-            : parseContainer(cur, { ...ctx, dirDepth: ctx.dirDepth + 1 }, dir.name);
-          out.push({ type: "directive", name: dir.name, attrs: attrs.attrs, children });
+            : parseContainer(cur, { ...ctx, dirDepth: ctx.dirDepth + 1 }, dir.name, T);
+          // A container's close line (or EOF auto-close) has been consumed
+          // by the child parse: the span runs to the last consumed line.
+          const endLi = dir.leaf ? li : Math.max(li, cur.pos - 1);
+          out.push(
+            mark(T, cur, { type: "directive", name: dir.name, attrs: attrs.attrs, children }, li, endLi),
+          );
         }
       } else {
-        out.push(invalid(dir.reason));
+        out.push(mark(T, cur, invalid(dir.reason), li, li));
       }
       continue;
     }
 
     // Comment block: consecutive `%%` lines, raw content.
     if (line.startsWith("%%")) {
+      const li = cur.pos;
       const texts: string[] = [];
       for (let l = peek(cur); l !== undefined && l.startsWith("%%"); l = peek(cur)) {
         const rest = l.slice(2);
         texts.push(rest.startsWith(" ") ? rest.slice(1) : rest);
         cur.pos += 1;
       }
-      out.push({ type: "comment", text: texts.join("\n") });
+      out.push(mark(T, cur, { type: "comment", text: texts.join("\n") }, li, cur.pos - 1));
       continue;
     }
 
     const heading = headingLine(line);
     if (heading !== null) {
+      const li = cur.pos;
       cur.pos += 1;
-      out.push({
-        type: "heading",
-        level: heading.level,
-        children: parseInlines(trimSpaceTab(heading.content)),
-      });
+      const content = trimSpaceTab(heading.content);
+      let P: PosCtx | null = null;
+      let text = content;
+      if (T !== null && cur.starts !== null) {
+        // Content offset: past the #s and their space, plus leading trim
+        // (all ASCII, so char counts are UTF-16 counts).
+        const lead = heading.content.length - heading.content.replace(/^[ \t]+/, "").length;
+        const start = lineStartOf(cur, li) + heading.level + 1 + lead;
+        const built = inlineCtx(T, [{ text: content, start }]);
+        text = built.text;
+        P = built.P;
+      }
+      out.push(
+        mark(
+          T,
+          cur,
+          { type: "heading", level: heading.level, children: parseInlines(text, P) },
+          li,
+          li,
+        ),
+      );
       continue;
     }
 
     const fence = fenceOpen(line);
     if (fence !== null) {
+      const li = cur.pos;
       cur.pos += 1;
       const content: string[] = [];
       for (let l = peek(cur); l !== undefined; l = peek(cur)) {
@@ -121,43 +218,63 @@ function parseContainer(cur: Cursor, ctx: Ctx, openDir: string | null): Node[] {
       }
       const text = content.join("\n");
       out.push(
-        fence.info === null
-          ? { type: "code_block", text }
-          : { type: "code_block", info: fence.info, text },
+        mark(
+          T,
+          cur,
+          fence.info === null
+            ? { type: "code_block", text }
+            : { type: "code_block", info: fence.info, text },
+          li,
+          cur.pos - 1,
+        ),
       );
       continue;
     }
 
     if (line.startsWith(">") && ctx.quoteDepth < MAX_QUOTE_DEPTH) {
+      const li = cur.pos;
       const innerLines: string[] = [];
+      const innerStarts: number[] | null = T !== null && cur.starts !== null ? [] : null;
       for (let l = peek(cur); l !== undefined && l.startsWith(">"); l = peek(cur)) {
         const rest = l.slice(1);
-        innerLines.push(rest.startsWith(" ") ? rest.slice(1) : rest);
+        const stripped = rest.startsWith(" ") ? 2 : 1;
+        innerLines.push(l.slice(stripped));
+        innerStarts?.push(lineStartOf(cur, cur.pos) + stripped);
         cur.pos += 1;
       }
-      const sub: Cursor = { lines: innerLines, pos: 0 };
-      out.push({
-        type: "blockquote",
-        children: parseContainer(sub, { ...ctx, quoteDepth: ctx.quoteDepth + 1 }, null),
-      });
+      const sub: Cursor = { lines: innerLines, starts: innerStarts, pos: 0 };
+      out.push(
+        mark(
+          T,
+          cur,
+          {
+            type: "blockquote",
+            children: parseContainer(sub, { ...ctx, quoteDepth: ctx.quoteDepth + 1 }, null, T),
+          },
+          li,
+          cur.pos - 1,
+        ),
+      );
       continue;
     }
 
     if (line.replace(/[ \t]+$/, "") === "---") {
+      const li = cur.pos;
       cur.pos += 1;
-      out.push({ type: "thematic_break" });
+      out.push(mark(T, cur, { type: "thematic_break" }, li, li));
       continue;
     }
 
     if (ctx.listDepth < MAX_LIST_DEPTH) {
       const m = marker(line);
       if (m !== null) {
-        out.push(parseList(cur, ctx, m));
+        out.push(parseList(cur, ctx, m, T));
         continue;
       }
     }
 
     // Paragraph: plain lines until blank / container end / block start.
+    const li = cur.pos;
     const plines = [line];
     cur.pos += 1;
     for (let l = peek(cur); l !== undefined; l = peek(cur)) {
@@ -170,11 +287,19 @@ function parseContainer(cur: Cursor, ctx: Ctx, openDir: string | null): Node[] {
     if (plines.length === 1) {
       const target = turbolink(plines[0]!);
       if (target !== null) {
-        out.push({ type: "turbolink", target });
+        out.push(mark(T, cur, { type: "turbolink", target }, li, li));
         continue;
       }
     }
-    out.push({ type: "paragraph", children: parseInlines(plines.join("\n")) });
+    let P: PosCtx | null = null;
+    let text = plines.join("\n");
+    if (T !== null && cur.starts !== null) {
+      const segs = plines.map((t, idx) => ({ text: t, start: lineStartOf(cur, li + idx) }));
+      const built = inlineCtx(T, segs);
+      text = built.text;
+      P = built.P;
+    }
+    out.push(mark(T, cur, { type: "paragraph", children: parseInlines(text, P) }, li, cur.pos - 1));
   }
   return out;
 }
@@ -262,12 +387,20 @@ function marker(line: string): Marker | null {
 
 /** One list of one kind. A same-column marker of the other kind ends this
  * list (the dispatcher immediately starts the next one). */
-function parseList(cur: Cursor, ctx: Ctx, first: Marker): Node {
+function parseList(cur: Cursor, ctx: Ctx, first: Marker, T: PosTracker | null): Node {
+  const tracking = T !== null && cur.starts !== null;
   const ordered = first.ordered;
   const items: Node[] = [];
+  const listLi = cur.pos;
+  let itemStart = tracking ? lineStartOf(cur, cur.pos) + first.indent : 0;
   let buf: string[] = [peek(cur)!.slice(first.contentIdx)];
+  let bufStarts: number[] | null = tracking ? [lineStartOf(cur, cur.pos) + first.contentIdx] : null;
   cur.pos += 1;
 
+  const finish = (): void => {
+    items.push(finishItem(buf, bufStarts, ctx, T, itemStart));
+  };
+
   for (;;) {
     const line = peek(cur);
     if (line === undefined) {
@@ -294,6 +427,7 @@ function parseList(cur: Cursor, ctx: Ctx, first: Marker): Node {
         break;
       }
       buf.push("");
+      bufStarts?.push(tracking ? lineStartOf(cur, cur.pos) : 0);
       cur.pos += 1;
       continue;
     }
@@ -301,29 +435,53 @@ function parseList(cur: Cursor, ctx: Ctx, first: Marker): Node {
       // Content (or a deeper marker) inside the current item; strip the
       // content column. Off-grid extra spaces ride along (floor rule).
       buf.push(line.slice(2));
+      bufStarts?.push(tracking ? lineStartOf(cur, cur.pos) + 2 : 0);
       cur.pos += 1;
       continue;
     }
     // Column 0 or 1 (floors to 0).
     const m = marker(line);
     if (m !== null && m.ordered === ordered) {
-      items.push(finishItem(buf, ctx));
+      finish();
+      itemStart = tracking ? lineStartOf(cur, cur.pos) + m.indent : 0;
       buf = [line.slice(m.contentIdx)];
+      bufStarts = tracking ? [lineStartOf(cur, cur.pos) + m.contentIdx] : null;
       cur.pos += 1;
       continue;
     }
     break; // column-0 block, or a kind switch
   }
-  items.push(finishItem(buf, ctx));
-  return { type: "list", ordered, children: items };
+  finish();
+  const list: Node = { type: "list", ordered, children: items };
+  if (tracking) {
+    T!.spans.set(list, {
+      start: lineStartOf(cur, listLi) + first.indent,
+      end: lineEndOf(cur, cur.pos - 1),
+    });
+  }
+  return list;
 }
 
-function finishItem(buf: string[], ctx: Ctx): Node {
-  const sub: Cursor = { lines: buf, pos: 0 };
-  return {
+function finishItem(
+  buf: string[],
+  bufStarts: number[] | null,
+  ctx: Ctx,
+  T: PosTracker | null,
+  itemStart: number,
+): Node {
+  const sub: Cursor = { lines: buf, starts: bufStarts, pos: 0 };
+  const item: Node = {
     type: "list_item",
-    children: parseContainer(sub, { ...ctx, listDepth: ctx.listDepth + 1 }, null),
+    children: parseContainer(sub, { ...ctx, listDepth: ctx.listDepth + 1 }, null, T),
   };
+  if (T !== null && bufStarts !== null) {
+    let end = itemStart;
+    for (let i = 0; i < buf.length; i += 1) {
+      end = Math.max(end, bufStarts[i]! + buf[i]!.length);
+    }
+    T.spans.set(item, { start: itemStart, end });
+  }
+  return item;
 }
 
 type DirLine =

package/src/index.ts

@@ -4,10 +4,11 @@
 // unknown dialect version (SPEC.md, "Conformance"), surfaced as a thrown
 // error rather than a guessed parse.
 
-import { parseBlocks } from "./blocks.ts";
+import { parseBlocks, type PosTracker, type Span } from "./blocks.ts";
 import type { Node } from "./ast.ts";
 
 export type { Attrs, Node, Reason } from "./ast.ts";
+export type { Span } from "./blocks.ts";
 
 export class UnsupportedVersionError extends Error {
   readonly version: number;
@@ -31,6 +32,39 @@ export function parse(input: string): Node {
   return { type: "document", version, children: parseBlocks(body) };
 }
 
+/** parse() plus source positions - the editor-tooling extra, deliberately
+ * OUTSIDE the AST wire contract (SPEC.md, "Source positions"):
+ *
+ * - `doc` is byte-identical to what parse() returns: nodes carry no
+ *   position fields, so serialization, vectors, and goldens are untouched.
+ * - `spans` maps each node object to its [start, end) source extent, in
+ *   UTF-16 code units (what JavaScript strings and CodeMirror speak) over
+ *   `source` - the front-door-NORMALIZED input (\r\n -> \n; the shebang
+ *   line, when present, is included in the offset space).
+ * - Node-level extents are the guarantee (a span covers its opener through
+ *   its closer); the interiors of canonicalized text (merged literals,
+ *   resolved escapes) are covered but not subdivided.
+ */
+export function parseWithPositions(input: string): {
+  doc: Node;
+  spans: WeakMap<Node, Span>;
+  source: string;
+} {
+  const normalized = input.replaceAll("\r\n", "\n").replaceAll("\r", "\n");
+  const [version, body] = stripVersion(normalized);
+  if (version !== 0) {
+    throw new UnsupportedVersionError(version);
+  }
+  const tracker: PosTracker = { spans: new WeakMap() };
+  const doc: Node = {
+    type: "document",
+    version,
+    children: parseBlocks(body, tracker, normalized.length - body.length),
+  };
+  tracker.spans.set(doc, { start: 0, end: normalized.length });
+  return { doc, spans: tracker.spans, source: normalized };
+}
+
 /** The in-band version declaration: line 1, exactly `#!marquee <integer>`.
  * The default is version 0, forever. */
 function stripVersion(s: string): [number, string] {