@cueloop/schema 0.1.0-alpha.5 → 0.1.0-alpha.50

package/src/inline.test.ts

@@ -0,0 +1,191 @@
+import { describe, expect, test } from "bun:test";
+import { inlineRuns, type InlineRun } from "./inline";
+
+/** Every run concatenated back in order must equal the source (nothing dropped). */
+function reconstruct(runs: InlineRun[]): string {
+  return runs.map((run) => run.text).join("");
+}
+
+/** Each visible run's text must sit at its claimed source offset. */
+function offsetsAreExact(source: string, runs: InlineRun[]): boolean {
+  return runs.every(
+    (run) =>
+      run.start === null || source.slice(run.start, run.start + run.text.length) === run.text,
+  );
+}
+
+/** The visible (non-marker) runs, which are what an anchor can resolve against. */
+function visible(runs: InlineRun[]): InlineRun[] {
+  return runs.filter((run) => run.start !== null);
+}
+
+describe("inlineRuns", () => {
+  test("plain text is a single positioned run", () => {
+    // Arrange
+    const source = "just plain words";
+
+    // Act
+    const runs = inlineRuns(source);
+
+    // Assert
+    expect(runs).toEqual([{ text: "just plain words", role: "text", start: 0 }]);
+  });
+
+  test("strong emphasis: markers concealed, content positioned", () => {
+    // Arrange
+    const source = "a **bold** b";
+
+    // Act
+    const runs = inlineRuns(source);
+
+    // Assert
+    expect(runs).toEqual([
+      { text: "a ", role: "text", start: 0 },
+      { text: "**", role: "marker", start: null },
+      { text: "bold", role: "strong", start: 4 },
+      { text: "**", role: "marker", start: null },
+      { text: " b", role: "text", start: 10 },
+    ]);
+    expect(reconstruct(runs)).toBe(source);
+    expect(offsetsAreExact(source, runs)).toBe(true);
+  });
+
+  test("em, code, and strike each conceal their markers and keep offsets", () => {
+    // Assert
+    for (const [source, role, inner] of [
+      ["x *em* y", "em", "em"],
+      ["x `code` y", "code", "code"],
+      ["x ~~gone~~ y", "strike", "gone"],
+    ] as const) {
+      const runs = inlineRuns(source);
+      const content = runs.find((run) => run.role === role)!;
+      expect(content.text).toBe(inner);
+      expect(source.slice(content.start!, content.start! + inner.length)).toBe(inner);
+      expect(reconstruct(runs)).toBe(source);
+      expect(runs.filter((run) => run.role === "marker").every((run) => run.start === null)).toBe(
+        true,
+      );
+    }
+  });
+
+  test("code spans do not parse markup inside them", () => {
+    // Act
+    const runs = inlineRuns("call `a**b**c` now");
+
+    // Assert - the backtick content is one literal code run, no strong inside
+    const code = runs.find((run) => run.role === "code")!;
+    expect(code.text).toBe("a**b**c");
+    expect(runs.some((run) => run.role === "strong")).toBe(false);
+  });
+
+  test("links: label positioned with href, brackets and url concealed", () => {
+    // Arrange
+    const source = "see [the docs](https://example.com) here";
+
+    // Act
+    const runs = inlineRuns(source);
+    const label = runs.find((run) => run.role === "link")!;
+
+    // Assert
+    expect(label).toEqual({
+      text: "the docs",
+      role: "link",
+      start: 5,
+      href: "https://example.com",
+    });
+    expect(reconstruct(runs)).toBe(source);
+    // the url text is a concealed marker, never a visible run
+    expect(visible(runs).some((run) => run.text.includes("https"))).toBe(false);
+  });
+
+  test("links: a destination with balanced parentheses stays whole", () => {
+    // Arrange
+    const source = "see [foo](https://en.wikipedia.org/wiki/Foo_(bar)) here";
+
+    // Act
+    const runs = inlineRuns(source);
+    const label = runs.find((run) => run.role === "link")!;
+
+    // Assert
+    expect(label.href).toBe("https://en.wikipedia.org/wiki/Foo_(bar)");
+    expect(reconstruct(runs)).toBe(source);
+  });
+
+  test("links: an unclosed destination never becomes a link", () => {
+    // Arrange
+    const source = "see [foo](https://example.com/(open here";
+
+    // Act
+    const runs = inlineRuns(source);
+
+    // Assert
+    expect(runs.some((run) => run.role === "link")).toBe(false);
+    expect(reconstruct(runs)).toBe(source);
+  });
+
+  test("nested emphasis: inner marker wins, both markers concealed", () => {
+    // Act
+    const runs = inlineRuns("**a *b* c**");
+
+    // Assert
+    expect(visible(runs)).toEqual([
+      { text: "a ", role: "strong", start: 2 },
+      { text: "b", role: "em", start: 5 },
+      { text: " c", role: "strong", start: 7 },
+    ]);
+    expect(reconstruct(runs)).toBe("**a *b* c**");
+  });
+
+  test("a lone em close is not stolen by an inner ** run", () => {
+    // Act - the em must span the whole thing, with **b** strong nested inside
+    const runs = inlineRuns("*a **b** c*");
+
+    // Assert
+    expect(runs.find((run) => run.role === "em" && run.text === "a ")).toBeDefined();
+    expect(runs.find((run) => run.role === "strong" && run.text === "b")).toBeDefined();
+    expect(reconstruct(runs)).toBe("*a **b** c*");
+  });
+
+  test("unbalanced markers fall back to literal text", () => {
+    // Assert
+    for (const source of ["**not closed", "a * b", "`open code", "[label](noclose"]) {
+      const runs = inlineRuns(source);
+      expect(reconstruct(runs)).toBe(source);
+      expect(runs.every((run) => run.role === "text" || run.role === "marker")).toBe(true);
+      expect(offsetsAreExact(source, runs)).toBe(true);
+    }
+  });
+
+  test("empty spans do not match", () => {
+    // Act
+    const runs = inlineRuns("a **** b `` c [](x)");
+
+    // Assert - nothing emphasized; all reconstructs
+    expect(reconstruct(runs)).toBe("a **** b `` c [](x)");
+    expect(runs.some((run) => ["strong", "em", "code", "link"].includes(run.role))).toBe(false);
+  });
+
+  test("backslash escapes render the punctuation literally, marker concealed", () => {
+    // Act
+    const runs = inlineRuns("not \\*emph\\* here");
+
+    // Assert - no em; the escaped stars are positioned text, backslashes concealed
+    expect(runs.some((run) => run.role === "em")).toBe(false);
+    expect(reconstruct(runs)).toBe("not \\*emph\\* here");
+    expect(offsetsAreExact("not \\*emph\\* here", runs)).toBe(true);
+    const star = runs.find((run) => run.text === "*" && run.start !== null)!;
+    expect("not \\*emph\\* here"[star.start!]).toBe("*");
+  });
+
+  test("reconstruct + exact offsets hold on a mixed line", () => {
+    // Arrange
+    const source = "The **daemon** persists `sessions` to [disk](d) ~~fast~~";
+
+    // Act
+    const runs = inlineRuns(source);
+
+    // Assert
+    expect(reconstruct(runs)).toBe(source);
+    expect(offsetsAreExact(source, runs)).toBe(true);
+  });
+});

package/src/inline.ts

@@ -0,0 +1,220 @@
+/**
+ * Inline markdown tokenizer: turns one block's text into styled runs while
+ * preserving exact source offsets, so the review surface can conceal the markup
+ * markers yet keep quote anchors char-precise.
+ *
+ * Each run is either VISIBLE text - carrying `start`, its offset in the input -
+ * or a CONCEALED marker (`**`, `` ` ``, `[`, `](url)`, ...) with `start: null`.
+ * A null run occupies no source offset, exactly like a word-diff removal, so the
+ * renderer's offset-mapping (overlayMarks / renderedOffsetFor / workRangeForRendered)
+ * paints highlights across the visible letters and skips the markers unchanged.
+ *
+ * Pure and dependency-free: the parse tree never escapes this module - the only
+ * output is the flat run list. Covers the plan subset: **strong**, *em*, `code`,
+ * ~~strike~~, [label](url), and backslash escapes. Unbalanced markup falls back
+ * to literal text.
+ */
+
+/** What a visible run represents; a marker run is concealed markup, no emphasis. */
+export type InlineRole = "text" | "strong" | "em" | "code" | "strike" | "link" | "marker";
+
+export interface InlineRun {
+  text: string;
+  role: InlineRole;
+  /** Offset of this run in the input text; null for a concealed marker. */
+  start: number | null;
+  /** Link target, present only on `link` runs. */
+  href?: string;
+}
+
+/** An emphasis span opened by a delimiter at some index, once its close is found. */
+interface Span {
+  /** Length of the opening marker (e.g. 2 for `**`). */
+  openLength: number;
+  /** Length of the closing marker. */
+  closeLength: number;
+  /** Index in `text` where the closing marker starts. */
+  closeIndex: number;
+  /** Emphasis role applied to the span's (recursively parsed) content. */
+  role: "strong" | "em" | "strike";
+}
+
+const PUNCTUATION = new Set(["*", "_", "`", "~", "[", "]", "(", ")", "\\"]);
+
+/** Tokenize one block's text into visible + concealed-marker runs, in order. */
+export function inlineRuns(text: string): InlineRun[] {
+  const runs: InlineRun[] = [];
+  tokenize(text, 0, "text", runs);
+  return runs;
+}
+
+/**
+ * Walk `source` (whose first character sits at absolute offset `base`), emitting
+ * plain text as `role` runs and recursing into emphasis spans. Links and code
+ * spans are handled inline because their content is not further emphasized the
+ * same way.
+ */
+function tokenize(source: string, base: number, role: InlineRole, out: InlineRun[]): void {
+  let plainStart = 0;
+  let cursor = 0;
+  const flushPlain = (end: number): void => {
+    if (end > plainStart) {
+      out.push({ text: source.slice(plainStart, end), role, start: base + plainStart });
+    }
+  };
+
+  while (cursor < source.length) {
+    const char = source[cursor]!;
+
+    if (char === "\\" && cursor + 1 < source.length && PUNCTUATION.has(source[cursor + 1]!)) {
+      flushPlain(cursor);
+      out.push({ text: "\\", role: "marker", start: null });
+      out.push({ text: source[cursor + 1]!, role, start: base + cursor + 1 });
+      cursor += 2;
+      plainStart = cursor;
+      continue;
+    }
+
+    const link = matchLink(source, cursor);
+    if (link) {
+      flushPlain(cursor);
+      out.push({ text: source.slice(cursor, link.labelStart), role: "marker", start: null });
+      out.push({
+        text: source.slice(link.labelStart, link.labelEnd),
+        role: "link",
+        start: base + link.labelStart,
+        href: source.slice(link.hrefStart, link.hrefEnd),
+      });
+      out.push({ text: source.slice(link.labelEnd, link.end), role: "marker", start: null });
+      cursor = link.end;
+      plainStart = cursor;
+      continue;
+    }
+
+    const code = matchCode(source, cursor);
+    if (code) {
+      flushPlain(cursor);
+      out.push({ text: source.slice(cursor, code.contentStart), role: "marker", start: null });
+      out.push({
+        text: source.slice(code.contentStart, code.contentEnd),
+        role: "code",
+        start: base + code.contentStart,
+      });
+      out.push({ text: source.slice(code.contentEnd, code.end), role: "marker", start: null });
+      cursor = code.end;
+      plainStart = cursor;
+      continue;
+    }
+
+    const span = matchEmphasis(source, cursor);
+    if (span) {
+      flushPlain(cursor);
+      out.push({
+        text: source.slice(cursor, cursor + span.openLength),
+        role: "marker",
+        start: null,
+      });
+      const contentStart = cursor + span.openLength;
+      tokenize(source.slice(contentStart, span.closeIndex), base + contentStart, span.role, out);
+      out.push({
+        text: source.slice(span.closeIndex, span.closeIndex + span.closeLength),
+        role: "marker",
+        start: null,
+      });
+      cursor = span.closeIndex + span.closeLength;
+      plainStart = cursor;
+      continue;
+    }
+
+    cursor++;
+  }
+  flushPlain(source.length);
+}
+
+/** A `[label](url)` link at `index`, or null. Empty label or url does not match. */
+function matchLink(
+  text: string,
+  index: number,
+): {
+  labelStart: number;
+  labelEnd: number;
+  hrefStart: number;
+  hrefEnd: number;
+  end: number;
+} | null {
+  if (text[index] !== "[") return null;
+  const labelStart = index + 1;
+  const labelEnd = text.indexOf("]", labelStart);
+  if (labelEnd < 0 || labelEnd === labelStart) return null;
+  if (text[labelEnd + 1] !== "(") return null;
+  const hrefStart = labelEnd + 2;
+  // the destination may carry balanced parentheses (wiki URLs); the span ends
+  // at the first unmatched closer
+  let depth = 0;
+  for (let scan = hrefStart; scan < text.length; scan++) {
+    const character = text[scan];
+    if (character === "(") depth++;
+    else if (character === ")") {
+      if (depth > 0) {
+        depth--;
+        continue;
+      }
+      if (scan === hrefStart) return null;
+      return { labelStart, labelEnd, hrefStart, hrefEnd: scan, end: scan + 1 };
+    }
+  }
+  return null;
+}
+
+/** A `` `code` `` span at `index` (n backticks .. n backticks), or null. */
+function matchCode(
+  text: string,
+  index: number,
+): { contentStart: number; contentEnd: number; end: number } | null {
+  if (text[index] !== "`") return null;
+  let fence = 0;
+  while (text[index + fence] === "`") fence++;
+  const contentStart = index + fence;
+  const closer = "`".repeat(fence);
+  const contentEnd = text.indexOf(closer, contentStart);
+  if (contentEnd < 0 || contentEnd === contentStart) return null;
+  return { contentStart, contentEnd, end: contentEnd + fence };
+}
+
+/** A `**strong**`, `*em*`, or `~~strike~~` span at `index`, or null. */
+function matchEmphasis(text: string, index: number): Span | null {
+  const two = text.slice(index, index + 2);
+  if (two === "**") return closingSpan(text, index, 2, "strong");
+  if (two === "~~") return closingSpan(text, index, 2, "strike");
+  if (text[index] === "*") return closingSpan(text, index, 1, "em");
+  return null;
+}
+
+/** Find the matching close for a marker of `length` at `index`; null if unbalanced. */
+function closingSpan(
+  text: string,
+  index: number,
+  length: number,
+  role: "strong" | "em" | "strike",
+): Span | null {
+  const marker = text.slice(index, index + length);
+  const contentStart = index + length;
+  const closeIndex =
+    length === 1 ? findLoneStar(text, contentStart) : text.indexOf(marker, contentStart);
+  if (closeIndex < 0 || closeIndex === contentStart) return null; // unbalanced or empty
+  return { openLength: length, closeLength: length, closeIndex, role };
+}
+
+/** Index of the next standalone `*` (not part of a `**` run), or -1. */
+function findLoneStar(text: string, from: number): number {
+  let index = from;
+  while (index < text.length) {
+    if (text[index] === "*") {
+      if (text[index + 1] !== "*") return index;
+      while (text[index] === "*") index++; // skip the whole `**` run
+      continue;
+    }
+    index++;
+  }
+  return -1;
+}

package/src/markdown.test.ts

@@ -1,5 +1,5 @@
 import { describe, expect, test } from "bun:test";
-import { parseBlocks, blockToMd, sectionOf } from "./markdown";
+import { parseBlocks, blockToMd, sectionOf, stripLeadingBlockMarker } from "./markdown";
 
 const SAMPLE = `# Title
 
@@ -29,8 +29,11 @@ const y = 2;
 
 describe("parseBlocks", () => {
   test("parses every block kind with source line ranges", () => {
+    // Act
     const blocks = parseBlocks(SAMPLE);
-    expect(blocks.map((b) => b.kind)).toEqual([
+
+    // Assert
+    expect(blocks.map((block) => block.kind)).toEqual([
       "h1",
       "h2",
       "p",
@@ -43,7 +46,7 @@ describe("parseBlocks", () => {
       "quote",
       "hr",
     ]);
-    const code = blocks.find((b) => b.kind === "code")!;
+    const code = blocks.find((block) => block.kind === "code")!;
     expect(code.lang).toBe("ts");
     expect(code.text).toBe("const x = 1;\nconst y = 2;");
     // fence lines included in the source range
@@ -51,50 +54,98 @@ describe("parseBlocks", () => {
   });
 
   test("multi-line paragraph keeps its text and range", () => {
+    // Arrange
     const blocks = parseBlocks(SAMPLE);
-    const p = blocks.find((b) => b.kind === "p")!;
-    expect(p.text).toBe("A paragraph that\nspans two lines.");
-    expect(p.lineEnd - p.lineStart).toBe(1);
+    const paragraph = blocks.find((block) => block.kind === "p")!;
+
+    // Assert
+    expect(paragraph.text).toBe("A paragraph that\nspans two lines.");
+    expect(paragraph.lineEnd - paragraph.lineStart).toBe(1);
   });
 
   test("line ranges index into the source", () => {
+    // Arrange
     const lines = SAMPLE.split("\n");
-    for (const b of parseBlocks(SAMPLE)) {
-      if (b.kind === "code") {
-        expect(lines[b.lineStart]!.startsWith("```")).toBe(true);
-      } else if (b.kind === "h2") {
-        expect(lines[b.lineStart]!.startsWith("## ")).toBe(true);
+
+    // Assert
+    for (const block of parseBlocks(SAMPLE)) {
+      if (block.kind === "code") {
+        expect(lines[block.lineStart]!.startsWith("```")).toBe(true);
+      } else if (block.kind === "h2") {
+        expect(lines[block.lineStart]!.startsWith("## ")).toBe(true);
       }
     }
   });
 
   test("round-trips through blockToMd", () => {
+    // Arrange
     const blocks = parseBlocks(SAMPLE);
-    let oli = 0;
+    let orderedItemCount = 0;
     const rebuilt = blocks
-      .map((b) => {
-        oli = b.kind === "oli" ? oli + 1 : 0;
-        return blockToMd(b, oli || 1);
+      .map((block) => {
+        orderedItemCount = block.kind === "oli" ? orderedItemCount + 1 : 0;
+        return blockToMd(block, orderedItemCount || 1);
       })
       .join("\n\n");
+
+    // Act
     // re-parsing the rebuild yields the same kinds and texts
     const again = parseBlocks(rebuilt);
-    expect(again.map((b) => [b.kind, b.text])).toEqual(blocks.map((b) => [b.kind, b.text]));
+
+    // Assert
+    expect(again.map((block) => [block.kind, block.text])).toEqual(
+      blocks.map((block) => [block.kind, block.text]),
+    );
   });
 
   test("unknown constructs degrade to paragraphs, no content lost", () => {
+    // Arrange
     const md = "| a | b |\n|---|---|\n| 1 | 2 |";
+
+    // Act
     const blocks = parseBlocks(md);
-    expect(blocks.map((b) => b.text).join("\n")).toContain("| a | b |");
+
+    // Assert
+    expect(blocks.map((block) => block.text).join("\n")).toContain("| a | b |");
   });
 });
 
 describe("sectionOf", () => {
   test("returns the nearest preceding heading", () => {
+    // Arrange
     const blocks = parseBlocks(SAMPLE);
-    const oliIdx = blocks.findIndex((b) => b.kind === "oli");
-    expect(sectionOf(blocks, oliIdx)).toBe("Steps");
-    const pIdx = blocks.findIndex((b) => b.kind === "p");
-    expect(sectionOf(blocks, pIdx)).toBe("Context");
+
+    // Assert
+    const orderedItemIndex = blocks.findIndex((block) => block.kind === "oli");
+    expect(sectionOf(blocks, orderedItemIndex)).toBe("Steps");
+    const paragraphIndex = blocks.findIndex((block) => block.kind === "p");
+    expect(sectionOf(blocks, paragraphIndex)).toBe("Context");
+  });
+});
+
+describe("stripLeadingBlockMarker", () => {
+  test("strips the marker the parser strips for every marked block kind", () => {
+    // Assert
+    expect(stripLeadingBlockMarker("# Heading")).toBe("Heading");
+    expect(stripLeadingBlockMarker("### Heading")).toBe("Heading");
+    expect(stripLeadingBlockMarker("- bullet item")).toBe("bullet item");
+    expect(stripLeadingBlockMarker("3. ordered item")).toBe("ordered item");
+    expect(stripLeadingBlockMarker("> quoted line")).toBe("quoted line");
+  });
+
+  test("agrees with the text the parser produces", () => {
+    // Arrange
+    const source = "- This is a dummy plan - confirm the item";
+
+    // Act
+    const [listBlock] = parseBlocks(source);
+
+    // Assert
+    expect(stripLeadingBlockMarker(source)).toBe(listBlock!.text);
+  });
+
+  test("leaves an unmarked line untouched", () => {
+    // Assert
+    expect(stripLeadingBlockMarker("a plain paragraph")).toBe("a plain paragraph");
   });
 });