@cueloop/schema 0.1.0-alpha.7 → 0.1.0-alpha.71

package/src/inline.ts

@@ -0,0 +1,240 @@
+/**
+ * Inline markdown tokenizer: turns one block's text into styled runs while
+ * preserving exact source offsets, so the review surface can conceal the markup
+ * markers yet keep quote anchors char-precise.
+ *
+ * Each run is either VISIBLE text - carrying `start`, its offset in the input -
+ * or a CONCEALED marker (`**`, `` ` ``, `[`, `](url)`, ...) with `start: null`.
+ * A null run occupies no source offset, exactly like a word-diff removal, so the
+ * renderer's offset-mapping (overlayMarks / renderedOffsetFor / workRangeForRendered)
+ * paints highlights across the visible letters and skips the markers unchanged.
+ *
+ * Pure and dependency-free: the parse tree never escapes this module - the only
+ * output is the flat run list. Covers the plan subset: **strong**, *em*, `code`,
+ * ~~strike~~, [label](url), and backslash escapes. Unbalanced markup falls back
+ * to literal text.
+ */
+
+/** What a visible run represents; a marker run is concealed markup, no emphasis. */
+export type InlineRole = "text" | "strong" | "em" | "code" | "strike" | "link" | "marker";
+
+export interface InlineRun {
+  text: string;
+  role: InlineRole;
+  /** Offset of this run in the input text; null for a concealed marker. */
+  start: number | null;
+  /** Link target, present only on `link` runs. */
+  href?: string;
+}
+
+/** An emphasis span opened by a delimiter at some index, once its close is found. */
+interface Span {
+  /** Length of the opening marker (e.g. 2 for `**`). */
+  openLength: number;
+  /** Length of the closing marker. */
+  closeLength: number;
+  /** Index in `text` where the closing marker starts. */
+  closeIndex: number;
+  /** Emphasis role applied to the span's (recursively parsed) content. */
+  role: "strong" | "em" | "strike";
+}
+
+const PUNCTUATION = new Set(["*", "_", "`", "~", "[", "]", "(", ")", "\\"]);
+
+/** Tokenize one block's text into visible + concealed-marker runs, in order. */
+export function inlineRuns(text: string): InlineRun[] {
+  const runs: InlineRun[] = [];
+
+  tokenize(text, 0, "text", runs);
+
+  return runs;
+}
+
+/**
+ * Walk `source` (whose first character sits at absolute offset `base`), emitting
+ * plain text as `role` runs and recursing into emphasis spans. Links and code
+ * spans are handled inline because their content is not further emphasized the
+ * same way.
+ */
+function tokenize(source: string, base: number, role: InlineRole, out: InlineRun[]): void {
+  let plainStart = 0;
+  let cursor = 0;
+  const flushPlain = (end: number): void => {
+    if (end > plainStart) {
+      out.push({ text: source.slice(plainStart, end), role, start: base + plainStart });
+    }
+  };
+
+  while (cursor < source.length) {
+    const char = source[cursor]!;
+
+    if (char === "\\" && cursor + 1 < source.length && PUNCTUATION.has(source[cursor + 1]!)) {
+      flushPlain(cursor);
+      out.push({ text: "\\", role: "marker", start: null });
+      out.push({ text: source[cursor + 1]!, role, start: base + cursor + 1 });
+      cursor += 2;
+      plainStart = cursor;
+      continue;
+    }
+
+    const link = matchLink(source, cursor);
+
+    if (link) {
+      flushPlain(cursor);
+      out.push({ text: source.slice(cursor, link.labelStart), role: "marker", start: null });
+      out.push({
+        text: source.slice(link.labelStart, link.labelEnd),
+        role: "link",
+        start: base + link.labelStart,
+        href: source.slice(link.hrefStart, link.hrefEnd),
+      });
+      out.push({ text: source.slice(link.labelEnd, link.end), role: "marker", start: null });
+      cursor = link.end;
+      plainStart = cursor;
+      continue;
+    }
+
+    const code = matchCode(source, cursor);
+
+    if (code) {
+      flushPlain(cursor);
+      out.push({ text: source.slice(cursor, code.contentStart), role: "marker", start: null });
+      out.push({
+        text: source.slice(code.contentStart, code.contentEnd),
+        role: "code",
+        start: base + code.contentStart,
+      });
+      out.push({ text: source.slice(code.contentEnd, code.end), role: "marker", start: null });
+      cursor = code.end;
+      plainStart = cursor;
+      continue;
+    }
+
+    const span = matchEmphasis(source, cursor);
+
+    if (span) {
+      flushPlain(cursor);
+      out.push({
+        text: source.slice(cursor, cursor + span.openLength),
+        role: "marker",
+        start: null,
+      });
+      const contentStart = cursor + span.openLength;
+
+      tokenize(source.slice(contentStart, span.closeIndex), base + contentStart, span.role, out);
+      out.push({
+        text: source.slice(span.closeIndex, span.closeIndex + span.closeLength),
+        role: "marker",
+        start: null,
+      });
+      cursor = span.closeIndex + span.closeLength;
+      plainStart = cursor;
+      continue;
+    }
+
+    cursor++;
+  }
+  flushPlain(source.length);
+}
+
+/** A `[label](url)` link at `index`, or null. Empty label or url does not match. */
+function matchLink(
+  text: string,
+  index: number,
+): {
+  labelStart: number;
+  labelEnd: number;
+  hrefStart: number;
+  hrefEnd: number;
+  end: number;
+} | null {
+  if (text[index] !== "[") return null;
+  const labelStart = index + 1;
+  const labelEnd = text.indexOf("]", labelStart);
+
+  if (labelEnd < 0 || labelEnd === labelStart) return null;
+  if (text[labelEnd + 1] !== "(") return null;
+  const hrefStart = labelEnd + 2;
+  // the destination may carry balanced parentheses (wiki URLs); the span ends
+  // at the first unmatched closer
+  let depth = 0;
+
+  for (let scan = hrefStart; scan < text.length; scan++) {
+    const character = text[scan];
+
+    if (character === "(") depth++;
+    else if (character === ")") {
+      if (depth > 0) {
+        depth--;
+        continue;
+      }
+      if (scan === hrefStart) return null;
+
+      return { labelStart, labelEnd, hrefStart, hrefEnd: scan, end: scan + 1 };
+    }
+  }
+
+  return null;
+}
+
+/** A `` `code` `` span at `index` (n backticks .. n backticks), or null. */
+function matchCode(
+  text: string,
+  index: number,
+): { contentStart: number; contentEnd: number; end: number } | null {
+  if (text[index] !== "`") return null;
+  let fence = 0;
+
+  while (text[index + fence] === "`") fence++;
+  const contentStart = index + fence;
+  const closer = "`".repeat(fence);
+  const contentEnd = text.indexOf(closer, contentStart);
+
+  if (contentEnd < 0 || contentEnd === contentStart) return null;
+
+  return { contentStart, contentEnd, end: contentEnd + fence };
+}
+
+/** A `**strong**`, `*em*`, or `~~strike~~` span at `index`, or null. */
+function matchEmphasis(text: string, index: number): Span | null {
+  const two = text.slice(index, index + 2);
+
+  if (two === "**") return closingSpan(text, index, 2, "strong");
+  if (two === "~~") return closingSpan(text, index, 2, "strike");
+  if (text[index] === "*") return closingSpan(text, index, 1, "em");
+
+  return null;
+}
+
+/** Find the matching close for a marker of `length` at `index`; null if unbalanced. */
+function closingSpan(
+  text: string,
+  index: number,
+  length: number,
+  role: "strong" | "em" | "strike",
+): Span | null {
+  const marker = text.slice(index, index + length);
+  const contentStart = index + length;
+  const closeIndex =
+    length === 1 ? findLoneStar(text, contentStart) : text.indexOf(marker, contentStart);
+
+  if (closeIndex < 0 || closeIndex === contentStart) return null; // unbalanced or empty
+
+  return { openLength: length, closeLength: length, closeIndex, role };
+}
+
+/** Index of the next standalone `*` (not part of a `**` run), or -1. */
+function findLoneStar(text: string, from: number): number {
+  let index = from;
+
+  while (index < text.length) {
+    if (text[index] === "*") {
+      if (text[index + 1] !== "*") return index;
+      while (text[index] === "*") index++; // skip the whole `**` run
+      continue;
+    }
+    index++;
+  }
+
+  return -1;
+}

package/src/markdown.test.ts

@@ -1,5 +1,5 @@
 import { describe, expect, test } from "bun:test";
-import { parseBlocks, blockToMd, sectionOf } from "./markdown";
+import { parseBlocks, blockToMd, sectionOf, stripLeadingBlockMarker } from "./markdown";
 
 const SAMPLE = `# Title
 
@@ -29,8 +29,11 @@ const y = 2;
 
 describe("parseBlocks", () => {
   test("parses every block kind with source line ranges", () => {
+    // Act
     const blocks = parseBlocks(SAMPLE);
-    expect(blocks.map((b) => b.kind)).toEqual([
+
+    // Assert
+    expect(blocks.map((block) => block.kind)).toEqual([
       "h1",
       "h2",
       "p",
@@ -43,7 +46,8 @@ describe("parseBlocks", () => {
       "quote",
       "hr",
     ]);
-    const code = blocks.find((b) => b.kind === "code")!;
+    const code = blocks.find((block) => block.kind === "code")!;
+
     expect(code.lang).toBe("ts");
     expect(code.text).toBe("const x = 1;\nconst y = 2;");
     // fence lines included in the source range
@@ -51,50 +55,101 @@ describe("parseBlocks", () => {
   });
 
   test("multi-line paragraph keeps its text and range", () => {
+    // Arrange
     const blocks = parseBlocks(SAMPLE);
-    const p = blocks.find((b) => b.kind === "p")!;
-    expect(p.text).toBe("A paragraph that\nspans two lines.");
-    expect(p.lineEnd - p.lineStart).toBe(1);
+    const paragraph = blocks.find((block) => block.kind === "p")!;
+
+    // Assert
+    expect(paragraph.text).toBe("A paragraph that\nspans two lines.");
+    expect(paragraph.lineEnd - paragraph.lineStart).toBe(1);
   });
 
   test("line ranges index into the source", () => {
+    // Arrange
     const lines = SAMPLE.split("\n");
-    for (const b of parseBlocks(SAMPLE)) {
-      if (b.kind === "code") {
-        expect(lines[b.lineStart]!.startsWith("```")).toBe(true);
-      } else if (b.kind === "h2") {
-        expect(lines[b.lineStart]!.startsWith("## ")).toBe(true);
+
+    // Assert
+    for (const block of parseBlocks(SAMPLE)) {
+      if (block.kind === "code") {
+        expect(lines[block.lineStart]!.startsWith("```")).toBe(true);
+      } else if (block.kind === "h2") {
+        expect(lines[block.lineStart]!.startsWith("## ")).toBe(true);
       }
     }
   });
 
   test("round-trips through blockToMd", () => {
+    // Arrange
     const blocks = parseBlocks(SAMPLE);
-    let oli = 0;
+    let orderedItemCount = 0;
     const rebuilt = blocks
-      .map((b) => {
-        oli = b.kind === "oli" ? oli + 1 : 0;
-        return blockToMd(b, oli || 1);
+      .map((block) => {
+        orderedItemCount = block.kind === "oli" ? orderedItemCount + 1 : 0;
+
+        return blockToMd(block, orderedItemCount || 1);
       })
       .join("\n\n");
+
+    // Act
     // re-parsing the rebuild yields the same kinds and texts
     const again = parseBlocks(rebuilt);
-    expect(again.map((b) => [b.kind, b.text])).toEqual(blocks.map((b) => [b.kind, b.text]));
+
+    // Assert
+    expect(again.map((block) => [block.kind, block.text])).toEqual(
+      blocks.map((block) => [block.kind, block.text]),
+    );
   });
 
   test("unknown constructs degrade to paragraphs, no content lost", () => {
+    // Arrange
     const md = "| a | b |\n|---|---|\n| 1 | 2 |";
+
+    // Act
     const blocks = parseBlocks(md);
-    expect(blocks.map((b) => b.text).join("\n")).toContain("| a | b |");
+
+    // Assert
+    expect(blocks.map((block) => block.text).join("\n")).toContain("| a | b |");
   });
 });
 
 describe("sectionOf", () => {
   test("returns the nearest preceding heading", () => {
+    // Arrange
     const blocks = parseBlocks(SAMPLE);
-    const oliIdx = blocks.findIndex((b) => b.kind === "oli");
-    expect(sectionOf(blocks, oliIdx)).toBe("Steps");
-    const pIdx = blocks.findIndex((b) => b.kind === "p");
-    expect(sectionOf(blocks, pIdx)).toBe("Context");
+
+    // Assert
+    const orderedItemIndex = blocks.findIndex((block) => block.kind === "oli");
+
+    expect(sectionOf(blocks, orderedItemIndex)).toBe("Steps");
+    const paragraphIndex = blocks.findIndex((block) => block.kind === "p");
+
+    expect(sectionOf(blocks, paragraphIndex)).toBe("Context");
+  });
+});
+
+describe("stripLeadingBlockMarker", () => {
+  test("strips the marker the parser strips for every marked block kind", () => {
+    // Assert
+    expect(stripLeadingBlockMarker("# Heading")).toBe("Heading");
+    expect(stripLeadingBlockMarker("### Heading")).toBe("Heading");
+    expect(stripLeadingBlockMarker("- bullet item")).toBe("bullet item");
+    expect(stripLeadingBlockMarker("3. ordered item")).toBe("ordered item");
+    expect(stripLeadingBlockMarker("> quoted line")).toBe("quoted line");
+  });
+
+  test("agrees with the text the parser produces", () => {
+    // Arrange
+    const source = "- This is a dummy plan - confirm the item";
+
+    // Act
+    const [listBlock] = parseBlocks(source);
+
+    // Assert
+    expect(stripLeadingBlockMarker(source)).toBe(listBlock!.text);
+  });
+
+  test("leaves an unmarked line untouched", () => {
+    // Assert
+    expect(stripLeadingBlockMarker("a plain paragraph")).toBe("a plain paragraph");
   });
 });

package/src/markdown.ts

@@ -1,6 +1,6 @@
 /**
- * Markdown block model (map #22): parse source into addressable blocks that
- * record their source line ranges. Rendered→source mapping is block+line
+ * Markdown block model: parse source into addressable blocks that record
+ * their source line ranges. Rendered-to-source mapping is block+line
  * granular; the quote carries sub-block precision. Round-trip safe for the
  * subset cueloop plans use; unknown constructs fall back to paragraph blocks
  * so no content is ever lost.
@@ -19,113 +19,164 @@ export interface Block {
   lineEnd: number;
 }
 
-function isMarkerLine(l: string): boolean {
+/**
+ * The leading block markers the parser strips from a line: headings (h1-h3),
+ * bullet list, ordered list, and blockquote. Single source of truth so the
+ * anchor resolver strips exactly the same set and the two never drift.
+ */
+export const LEADING_BLOCK_MARKER = /^(?:#{1,3} |- |\d+\. |> )/;
+
+/** Remove one leading markdown block marker, or return the line unchanged. */
+export function stripLeadingBlockMarker(line: string): string {
+  return line.replace(LEADING_BLOCK_MARKER, "");
+}
+
+function isMarkerLine(line: string): boolean {
   return (
-    l.startsWith("```") ||
-    l.startsWith("# ") ||
-    l.startsWith("## ") ||
-    l.startsWith("### ") ||
-    l.startsWith("> ") ||
-    /^- /.test(l) ||
-    /^\d+\. /.test(l) ||
-    /^(---|\*\*\*|___)\s*$/.test(l)
+    line.startsWith("```") || LEADING_BLOCK_MARKER.test(line) || /^(---|\*\*\*|___)\s*$/.test(line)
   );
 }
 
-export function parseBlocks(md: string): Block[] {
-  const lines = md.split("\n");
+export function parseBlocks(markdown: string): Block[] {
+  const lines = markdown.split("\n");
   const blocks: Block[] = [];
-  let i = 0;
-  while (i < lines.length) {
-    const l = lines[i]!;
-    if (l.trim() === "") {
-      i++;
+  let lineIndex = 0;
+
+  while (lineIndex < lines.length) {
+    const line = lines[lineIndex]!;
+
+    if (line.trim() === "") {
+      lineIndex++;
       continue;
     }
-    if (l.startsWith("```")) {
-      const start = i;
-      const lang = l.slice(3).trim() || undefined;
-      i++;
+    if (line.startsWith("```")) {
+      const start = lineIndex;
+      const lang = line.slice(3).trim() || undefined;
+
+      lineIndex++;
       const body: string[] = [];
-      while (i < lines.length && !lines[i]!.startsWith("```")) {
-        body.push(lines[i]!);
-        i++;
+
+      while (lineIndex < lines.length && !lines[lineIndex]!.startsWith("```")) {
+        body.push(lines[lineIndex]!);
+        lineIndex++;
       }
-      const end = Math.min(i, lines.length - 1);
-      i++;
+      const end = Math.min(lineIndex, lines.length - 1);
+
+      lineIndex++;
       blocks.push({ kind: "code", text: body.join("\n"), lang, lineStart: start, lineEnd: end });
-    } else if (l.startsWith("### ")) {
-      blocks.push({ kind: "h3", text: l.slice(4), lineStart: i, lineEnd: i });
-      i++;
-    } else if (l.startsWith("## ")) {
-      blocks.push({ kind: "h2", text: l.slice(3), lineStart: i, lineEnd: i });
-      i++;
-    } else if (l.startsWith("# ")) {
-      blocks.push({ kind: "h1", text: l.slice(2), lineStart: i, lineEnd: i });
-      i++;
-    } else if (/^(---|\*\*\*|___)\s*$/.test(l)) {
-      blocks.push({ kind: "hr", text: "", lineStart: i, lineEnd: i });
-      i++;
-    } else if (l.startsWith("> ")) {
-      const start = i;
+    } else if (line.startsWith("### ")) {
+      blocks.push({
+        kind: "h3",
+        text: stripLeadingBlockMarker(line),
+        lineStart: lineIndex,
+        lineEnd: lineIndex,
+      });
+      lineIndex++;
+    } else if (line.startsWith("## ")) {
+      blocks.push({
+        kind: "h2",
+        text: stripLeadingBlockMarker(line),
+        lineStart: lineIndex,
+        lineEnd: lineIndex,
+      });
+      lineIndex++;
+    } else if (line.startsWith("# ")) {
+      blocks.push({
+        kind: "h1",
+        text: stripLeadingBlockMarker(line),
+        lineStart: lineIndex,
+        lineEnd: lineIndex,
+      });
+      lineIndex++;
+    } else if (/^(---|\*\*\*|___)\s*$/.test(line)) {
+      blocks.push({ kind: "hr", text: "", lineStart: lineIndex, lineEnd: lineIndex });
+      lineIndex++;
+    } else if (line.startsWith("> ")) {
+      const start = lineIndex;
       const body: string[] = [];
-      while (i < lines.length && lines[i]!.startsWith("> ")) {
-        body.push(lines[i]!.slice(2));
-        i++;
+
+      while (lineIndex < lines.length && lines[lineIndex]!.startsWith("> ")) {
+        body.push(stripLeadingBlockMarker(lines[lineIndex]!));
+        lineIndex++;
       }
-      blocks.push({ kind: "quote", text: body.join("\n"), lineStart: start, lineEnd: i - 1 });
-    } else if (/^- /.test(l)) {
-      blocks.push({ kind: "li", text: l.slice(2), lineStart: i, lineEnd: i });
-      i++;
-    } else if (/^\d+\. /.test(l)) {
-      blocks.push({ kind: "oli", text: l.replace(/^\d+\. /, ""), lineStart: i, lineEnd: i });
-      i++;
+      blocks.push({
+        kind: "quote",
+        text: body.join("\n"),
+        lineStart: start,
+        lineEnd: lineIndex - 1,
+      });
+    } else if (line.startsWith("- ")) {
+      blocks.push({
+        kind: "li",
+        text: stripLeadingBlockMarker(line),
+        lineStart: lineIndex,
+        lineEnd: lineIndex,
+      });
+      lineIndex++;
+    } else if (/^\d+\. /.test(line)) {
+      blocks.push({
+        kind: "oli",
+        text: stripLeadingBlockMarker(line),
+        lineStart: lineIndex,
+        lineEnd: lineIndex,
+      });
+      lineIndex++;
     } else {
-      const start = i;
+      const start = lineIndex;
       const body: string[] = [];
-      while (i < lines.length && lines[i]!.trim() !== "" && !isMarkerLine(lines[i]!)) {
-        body.push(lines[i]!);
-        i++;
+
+      while (
+        lineIndex < lines.length &&
+        lines[lineIndex]!.trim() !== "" &&
+        !isMarkerLine(lines[lineIndex]!)
+      ) {
+        body.push(lines[lineIndex]!);
+        lineIndex++;
       }
-      blocks.push({ kind: "p", text: body.join("\n"), lineStart: start, lineEnd: i - 1 });
+      blocks.push({ kind: "p", text: body.join("\n"), lineStart: start, lineEnd: lineIndex - 1 });
     }
   }
+
   return blocks;
 }
 
 /** Serialize one block back to its markdown chunk. */
-export function blockToMd(b: Block, ordinal = 1): string {
-  switch (b.kind) {
+export function blockToMd(block: Block, ordinal = 1): string {
+  switch (block.kind) {
     case "h1":
-      return "# " + b.text;
+      return "# " + block.text;
     case "h2":
-      return "## " + b.text;
+      return "## " + block.text;
     case "h3":
-      return "### " + b.text;
+      return "### " + block.text;
     case "li":
-      return "- " + b.text;
+      return "- " + block.text;
     case "oli":
-      return `${ordinal}. ` + b.text;
+      return `${ordinal}. ` + block.text;
     case "quote":
-      return b.text
+      return block.text
         .split("\n")
-        .map((l) => "> " + l)
+        .map((line) => "> " + line)
         .join("\n");
     case "code":
-      return "```" + (b.lang ?? "") + "\n" + b.text + "\n```";
+      return "```" + (block.lang ?? "") + "\n" + block.text + "\n```";
     case "hr":
       return "---";
     default:
-      return b.text;
+      return block.text;
   }
 }
 
 /** The section (nearest preceding heading) a block belongs to. */
 export function sectionOf(blocks: Block[], index: number): string {
-  let sec = "";
-  for (let i = 0; i <= index && i < blocks.length; i++) {
-    const b = blocks[i]!;
-    if (b.kind === "h1" || b.kind === "h2" || b.kind === "h3") sec = b.text;
+  let sectionTitle = "";
+
+  for (let blockIndex = 0; blockIndex <= index && blockIndex < blocks.length; blockIndex++) {
+    const block = blocks[blockIndex]!;
+
+    if (block.kind === "h1" || block.kind === "h2" || block.kind === "h3")
+      sectionTitle = block.text;
   }
-  return sec;
+
+  return sectionTitle;
 }