@ai-matrx/rich-editor 0.0.0 → 0.1.1

package/dist/core/markdown-parse.js

@@ -0,0 +1,533 @@
+import { splitRowSegments } from "@ai-matrx/rich-content/utils/table-source";
+import { Lexer, Tokenizer } from "@ai-matrx/rich-content/utils/gfm-lexer";
+import { findTableEnd, tableStartsAt } from "@ai-matrx/rich-content/display/markdown-classification/processors/utils/gfm-table-lines";
+import { isPageBreakLine } from "@ai-matrx/print/directives";
+import {
+  hasPrivateUseCharacter,
+  restorePlaceholders,
+  splitPlaceholders,
+  withPlaceholders
+} from "./placeholders.js";
+import {
+  createSerializeContext,
+  serializeBlock
+} from "./markdown-serialize.js";
+const LEXER_OPTIONS = { gfm: true, breaks: false, pedantic: false };
+class RuleTableTokenizer extends Tokenizer {
+  // The rule judges the stored bytes: an inline island (`</artifact>`, a variable)
+  // stands in the lexed text as a placeholder, so each line is restored first.
+  constructor(islands, segmentText) {
+    super();
+    this.islands = islands;
+    const stored = segmentText.split("\n").map((line) => restorePlaceholders(line, islands));
+    const lazy = /* @__PURE__ */ new Set();
+    const real = /* @__PURE__ */ new Set();
+    for (let i = 0; i + 1 < stored.length; i += 1) {
+      const key = headerKey(stored, i);
+      if (tableStartsAt(stored, i)) real.add(key);
+      else if (tableStartsAt(stored.slice(i).map((line) => line.trimStart()), 0)) lazy.add(key);
+    }
+    for (const key of real) lazy.delete(key);
+    this.lazyHeaders = lazy;
+  }
+  islands;
+  /**
+   * Headers (with their delimiter row) the rule refuses IN CONTEXT but would open
+   * on their own: a table written right under a list item's or quote's text is a
+   * lazy continuation of that text in GFM. marked lexes an item's lines already
+   * dedented, where that context is gone — so the segment's own lines decide,
+   * keyed by the header and delimiter bytes. A key that also opens a real table
+   * in the segment is left to marked (ambiguous, never guessed).
+   */
+  lazyHeaders;
+  table(src) {
+    const lines = src.split("\n");
+    const stored = lines.map((line) => restorePlaceholders(line, this.islands));
+    if (this.lazyHeaders.has(headerKey(stored, 0))) return void 0;
+    if (!tableStartsAt(stored, 0)) return void 0;
+    const end = findTableEnd(stored, 0);
+    return super.table(lines.slice(0, end).join("\n") + (end < lines.length ? "\n" : ""));
+  }
+}
+const headerKey = (lines, i) => `${(lines[i] ?? "").trim()}
+${(lines[i + 1] ?? "").trim()}`;
+const PAGE_BREAK_REASON = "page break";
+const LOCK_REASONS = {
+  html: "raw HTML",
+  code: "an indented code block",
+  def: "a link reference definition"
+};
+function splitTrail(raw) {
+  const match = /\n*$/.exec(raw);
+  const trail = match ? match[0] : "";
+  return { body: raw.slice(0, raw.length - trail.length), trail };
+}
+function pushInline(target, node) {
+  const last = target[target.length - 1];
+  if (node.type === "text" && last?.type === "text" && JSON.stringify(last.marks ?? []) === JSON.stringify(node.marks ?? [])) {
+    last.text = (last.text ?? "") + (node.text ?? "");
+    return;
+  }
+  target.push(node);
+}
+const BR_TAG = /^<br\s*\/?>$/i;
+function withMarks(node, marks) {
+  return marks.length ? { ...node, marks: marks.map((m) => ({ ...m })) } : node;
+}
+function pushText(out, text, marks, state) {
+  for (const piece of splitPlaceholders(text, state.islands)) {
+    if (piece.kind === "text") {
+      if (piece.text) pushInline(out, withMarks({ type: "text", text: piece.text }, marks));
+    } else if (BR_TAG.test(piece.island.raw)) {
+      out.push(withMarks({ type: "hardBreak", attrs: { mdRaw: piece.island.raw } }, marks));
+    } else {
+      out.push(
+        withMarks(
+          {
+            type: "inlineIsland",
+            attrs: { raw: piece.island.raw, islandType: piece.island.islandType }
+          },
+          marks
+        )
+      );
+    }
+  }
+}
+function pushRaw(out, raw, islandType, marks, state) {
+  out.push(
+    withMarks(
+      {
+        type: "inlineIsland",
+        attrs: { raw: restorePlaceholders(raw, state.islands), islandType }
+      },
+      marks
+    )
+  );
+}
+function codeSpanParts(raw) {
+  const ticks = /^`+/.exec(raw)?.[0] ?? "";
+  if (!ticks || !raw.endsWith(ticks) || raw.length <= ticks.length * 2) return null;
+  const inner = raw.slice(ticks.length, raw.length - ticks.length);
+  if (inner.length >= 2 && inner.startsWith(" ") && inner.endsWith(" ") && inner.trim() !== "") {
+    return { open: `${ticks} `, close: ` ${ticks}`, text: inner.slice(1, -1) };
+  }
+  return { open: ticks, close: ticks, text: inner };
+}
+function inlineJSON(tokens, marks, depth, state, out = []) {
+  for (const token of tokens) {
+    switch (token.type) {
+      case "text": {
+        const nested = token.tokens;
+        if (nested && nested.length) inlineJSON(nested, marks, depth, state, out);
+        else pushText(out, token.raw, marks, state);
+        break;
+      }
+      case "escape":
+        pushText(out, token.raw.slice(1), [...marks, { type: "mdEscape" }], state);
+        break;
+      case "strong":
+        inlineJSON(token.tokens, [
+          ...marks,
+          { type: "bold", attrs: { mdMarker: token.raw.slice(0, 2), mdDepth: depth } }
+        ], depth + 1, state, out);
+        break;
+      case "em":
+        inlineJSON(token.tokens, [
+          ...marks,
+          { type: "italic", attrs: { mdMarker: token.raw.slice(0, 1), mdDepth: depth } }
+        ], depth + 1, state, out);
+        break;
+      case "del":
+        inlineJSON(token.tokens, [
+          ...marks,
+          {
+            type: "strike",
+            attrs: { mdMarker: token.raw.startsWith("~~") ? "~~" : "~", mdDepth: depth }
+          }
+        ], depth + 1, state, out);
+        break;
+      case "codespan": {
+        const parts = codeSpanParts(token.raw);
+        if (!parts || !parts.text) {
+          pushRaw(out, token.raw, "md_raw", marks, state);
+          break;
+        }
+        pushText(out, parts.text, [
+          ...marks,
+          { type: "code", attrs: { mdOpen: parts.open, mdClose: parts.close, mdDepth: depth } }
+        ], state);
+        break;
+      }
+      case "link": {
+        const link = token;
+        let form = null;
+        let tail = "";
+        if (link.raw.startsWith(`[${link.text}]`)) {
+          form = "inline";
+          tail = link.raw.slice(link.text.length + 2);
+        } else if (link.raw === `<${link.text}>`) {
+          form = "angle";
+        } else if (link.raw === link.text) {
+          form = "bare";
+        }
+        if (!form) {
+          pushRaw(out, link.raw, "md_raw", marks, state);
+          break;
+        }
+        const href = restorePlaceholders(link.href ?? "", state.islands);
+        inlineJSON(link.tokens ?? [], [
+          ...marks,
+          {
+            type: "link",
+            attrs: {
+              href,
+              title: link.title ?? null,
+              mdForm: form,
+              mdTail: restorePlaceholders(tail, state.islands),
+              mdTailHref: href,
+              mdText: restorePlaceholders(link.text, state.islands),
+              mdDepth: depth
+            }
+          }
+        ], depth + 1, state, out);
+        break;
+      }
+      case "br":
+        out.push(withMarks({ type: "hardBreak", attrs: { mdRaw: token.raw } }, marks));
+        break;
+      case "image":
+        pushRaw(out, token.raw, "md_image", marks, state);
+        break;
+      case "html":
+        if (BR_TAG.test(token.raw)) out.push(withMarks({ type: "hardBreak", attrs: { mdRaw: token.raw } }, marks));
+        else pushRaw(out, token.raw, "md_html", marks, state);
+        break;
+      default:
+        pushRaw(out, token.raw, "md_raw", marks, state);
+    }
+  }
+  return out;
+}
+function inlineRaw(tokens) {
+  return (tokens ?? []).map((token) => token.raw).join("");
+}
+function blockJSON(token, state) {
+  const { body } = splitTrail(token.raw);
+  switch (token.type) {
+    case "paragraph":
+    case "text": {
+      const tokens = token.tokens;
+      const inline = tokens && tokens.length ? tokens : null;
+      if (inline && inlineRaw(inline) !== body) return null;
+      const content = inline ? inlineJSON(inline, [], 0, state) : (() => {
+        const out = [];
+        pushText(out, body, [], state);
+        return out;
+      })();
+      return {
+        type: "paragraph",
+        attrs: { mdId: state.nextId() },
+        ...content.length ? { content } : {}
+      };
+    }
+    case "heading": {
+      const heading = token;
+      const raw = inlineRaw(heading.tokens);
+      let open;
+      let close;
+      const atx = /^ {0,3}#{1,6}(?:[ \t]+|$)/.exec(body);
+      if (atx) {
+        open = atx[0];
+        const rest = body.slice(open.length);
+        if (!rest.startsWith(raw)) return null;
+        close = rest.slice(raw.length);
+      } else {
+        if (!body.startsWith(raw)) return null;
+        open = "";
+        close = body.slice(raw.length);
+        if (!close.startsWith("\n")) return null;
+      }
+      const content = inlineJSON(heading.tokens ?? [], [], 0, state);
+      return {
+        type: "heading",
+        attrs: {
+          level: heading.depth,
+          mdId: state.nextId(),
+          mdOpen: restorePlaceholders(open, state.islands),
+          mdClose: restorePlaceholders(close, state.islands)
+        },
+        ...content.length ? { content } : {}
+      };
+    }
+    case "list": {
+      const list = token;
+      const items = [];
+      const ids = [];
+      const trails = [];
+      for (const item of list.items) {
+        const json = listItemJSON(item, state);
+        if (!json) return null;
+        items.push(json);
+        ids.push(String(json.attrs?.mdId));
+        trails.push(splitTrail(item.raw).trail);
+      }
+      ids.forEach(
+        (id, index) => state.adjacency.set(id, { trail: trails[index] ?? "", next: ids[index + 1] ?? null })
+      );
+      const start = list.ordered ? Number(list.start === "" ? 1 : list.start) || 1 : null;
+      return {
+        type: list.ordered ? "orderedList" : "bulletList",
+        attrs: { mdId: state.nextId(), ...start !== null ? { start } : {} },
+        content: items
+      };
+    }
+    case "blockquote": {
+      const quote = token;
+      const prefix = /^ {0,3}> ?/.exec(quote.raw)?.[0] ?? "> ";
+      const children = childrenJSON(quote.tokens, state);
+      if (!children || children.length === 0) return null;
+      const alert = takeAlertMarker(children);
+      if (children.length === 0) return null;
+      return {
+        type: "blockquote",
+        attrs: { mdId: state.nextId(), mdPrefix: prefix, mdAlert: alert },
+        content: children
+      };
+    }
+    case "hr":
+      return { type: "horizontalRule", attrs: { mdId: state.nextId(), mdRaw: body } };
+    case "table": {
+      const table = token;
+      const lines = body.split("\n");
+      if (lines.length !== table.rows.length + 2) return null;
+      const header = lines[0] ?? "";
+      const headerSegs = splitRowSegments(header);
+      const leadPipe = headerSegs.length > 1 && (headerSegs[0] ?? "").trim() === "";
+      const trailPipe = headerSegs.length > 1 && (headerSegs[headerSegs.length - 1] ?? "").trim() === "";
+      const pipes = leadPipe && trailPipe ? "both" : leadPipe ? "lead" : trailPipe ? "trail" : "none";
+      const aligns = table.align.map((align) => align ?? null);
+      const cellJSON = (cell, isHeader, index) => {
+        const content = inlineJSON(cell.tokens, [], 0, state);
+        return {
+          type: isHeader ? "tableHeader" : "tableCell",
+          attrs: { align: aligns[index] ?? null },
+          content: [{ type: "paragraph", ...content.length ? { content } : {} }]
+        };
+      };
+      const rowJSON = (cells, isHeader, line) => ({
+        type: "tableRow",
+        attrs: {
+          mdRaw: restorePlaceholders(line, state.islands),
+          mdCells: JSON.stringify(
+            cells.map((cell) => restorePlaceholders(inlineRaw(cell.tokens), state.islands))
+          ),
+          // The row's exact bytes split on its unescaped pipes: an untouched cell
+          // is written back as ITS segment, never re-serialized (markdown-serialize.ts).
+          mdSegs: JSON.stringify(splitRowSegments(restorePlaceholders(line, state.islands)))
+        },
+        content: cells.map((cell, index) => cellJSON(cell, isHeader, index))
+      });
+      return {
+        type: "table",
+        attrs: {
+          mdId: state.nextId(),
+          mdDelim: restorePlaceholders(lines[1] ?? "", state.islands),
+          mdAligns: JSON.stringify(aligns),
+          mdPipes: pipes
+        },
+        content: [
+          rowJSON(table.header, true, header),
+          ...table.rows.map((row, index) => rowJSON(row, false, lines[index + 2] ?? ""))
+        ]
+      };
+    }
+    default:
+      return null;
+  }
+}
+const ALERT_MARKER = /^\[!(NOTE|TIP|IMPORTANT|WARNING|CAUTION)\](?=\n|$)/i;
+function takeAlertMarker(children) {
+  const first = children[0];
+  const text = first?.type === "paragraph" ? first.content?.[0] : void 0;
+  if (!first || !text || text.type !== "text" || text.marks?.length) return null;
+  const match = ALERT_MARKER.exec(text.text ?? "");
+  if (!match) return null;
+  const rest = (text.text ?? "").slice(match[0].length);
+  if (rest === "") {
+    first.content = first.content?.slice(1);
+    if (!first.content?.length) children.shift();
+  } else if (rest.startsWith("\n")) {
+    text.text = rest.slice(1);
+    if (!text.text) first.content = first.content?.slice(1);
+  } else {
+    return null;
+  }
+  return match[0];
+}
+function listItemJSON(item, state) {
+  const head = /^( {0,3})([-*+]|\d{1,9}[.)])([ \t]*)/.exec(item.raw);
+  if (!head) return null;
+  const tokens = item.tokens.filter((token) => token.type !== "checkbox");
+  const checkbox = item.tokens.find((token) => token.type === "checkbox");
+  const children = childrenJSON(tokens, state);
+  if (!children) return null;
+  if (children.length === 0) children.push({ type: "paragraph", attrs: { mdId: state.nextId() } });
+  if (children[0]?.type !== "paragraph") return null;
+  const lines = splitTrail(item.raw).body.split("\n");
+  const continuation = lines.slice(1).find((line) => line.trim() !== "");
+  return {
+    type: "listItem",
+    attrs: {
+      mdId: state.nextId(),
+      mdLead: head[1] ?? "",
+      mdMarker: head[2] ?? "-",
+      mdAfter: head[3] ?? "",
+      mdTask: checkbox ? checkbox.raw : null,
+      mdIndent: continuation ? continuation.length - continuation.trimStart().length : null
+    },
+    content: children
+  };
+}
+function childrenJSON(tokens, state) {
+  const out = [];
+  const ids = [];
+  const trails = [];
+  for (const token of tokens) {
+    if (token.type === "space") {
+      if (trails.length === 0) return null;
+      trails[trails.length - 1] += token.raw;
+      continue;
+    }
+    const json = blockJSON(token, state);
+    if (!json) return null;
+    out.push(json);
+    ids.push(String(json.attrs?.mdId));
+    trails.push(splitTrail(token.raw).trail);
+  }
+  ids.forEach(
+    (id, index) => state.adjacency.set(id, { trail: trails[index] ?? "", next: ids[index + 1] ?? null })
+  );
+  return out;
+}
+function lockedJSON(raw, reason, id) {
+  return { type: "sourceLocked", attrs: { raw, reason, mdId: id } };
+}
+function parseProseBlock(block, schema, adjacency, nextId, linkDefinitions) {
+  if (block.raw.includes("\r")) return { children: [], lockedReason: "Windows line endings" };
+  if (hasPrivateUseCharacter(block.raw)) {
+    return { children: [], lockedReason: "private-use characters" };
+  }
+  const { text, islands } = withPlaceholders(block);
+  const { segments, lead } = splitAtBlankLines(text);
+  if (lead) return { children: [], lockedReason: "leading blank lines" };
+  if (segments.some((segment) => segment.text === "")) {
+    return { children: [], lockedReason: "markdown the parser could not map" };
+  }
+  const state = { schema, islands, adjacency, nextId };
+  const children = [];
+  for (const segment of segments) {
+    let tokens;
+    try {
+      const lexer = new Lexer({ ...LEXER_OPTIONS, tokenizer: new RuleTableTokenizer(islands, segment.text) });
+      for (const [label, def] of linkDefinitions ?? []) lexer.tokens.links[label] = { href: def.url, title: def.title ?? void 0 };
+      tokens = lexer.lex(segment.text);
+    } catch {
+      return { children: [], lockedReason: "markdown the parser could not read" };
+    }
+    if (tokens.map((token) => token.raw).join("") !== segment.text) {
+      return { children: [], lockedReason: "markdown the parser could not map" };
+    }
+    const segmentStart = children.length;
+    for (const token of tokens) {
+      if (token.type === "space") {
+        const last2 = children[children.length - 1];
+        if (!last2 || children.length === segmentStart) {
+          return { children: [], lockedReason: "leading blank lines" };
+        }
+        last2.trail += token.raw;
+        continue;
+      }
+      children.push(parseTopToken(token, state));
+    }
+    const last = children[children.length - 1];
+    if (!last || children.length === segmentStart) {
+      return { children: [], lockedReason: "markdown the parser could not map" };
+    }
+    last.trail += segment.after;
+  }
+  children.forEach(
+    (child, index) => adjacency.set(child.id, { trail: child.trail, next: children[index + 1]?.id ?? null })
+  );
+  if (children.length === 0) return { children, lockedReason: "empty block" };
+  if (children.every((child) => child.lockedReason !== null)) {
+    return { children, lockedReason: children[0]?.lockedReason ?? "source" };
+  }
+  return { children, lockedReason: null };
+}
+function splitAtBlankLines(text) {
+  if (!/^[ \t]+$/m.test(text)) return { segments: [trimSegmentEnd(text, "")], lead: "" };
+  const ranges = [];
+  let current = null;
+  let position = 0;
+  for (const line of text.split("\n")) {
+    const lineEnd = position + line.length;
+    if (/^[ \t]*$/.test(line)) {
+      if (current) ranges.push(current);
+      current = null;
+    } else if (current) {
+      current.end = lineEnd;
+    } else {
+      current = { start: position, end: lineEnd };
+    }
+    position = lineEnd + 1;
+  }
+  if (current) ranges.push(current);
+  const segments = ranges.map(
+    (range, index) => trimSegmentEnd(
+      text.slice(range.start, range.end),
+      text.slice(range.end, ranges[index + 1]?.start ?? text.length)
+    )
+  );
+  return { segments, lead: text.slice(0, ranges[0]?.start ?? text.length) };
+}
+function trimSegmentEnd(text, after) {
+  const tail = /[ \t]+$/.exec(text)?.[0] ?? "";
+  return tail ? { text: text.slice(0, text.length - tail.length), after: tail + after } : { text, after };
+}
+function parseTopToken(token, state) {
+  const { schema, islands, adjacency, nextId } = state;
+  const { body, trail } = splitTrail(token.raw);
+  const raw = restorePlaceholders(body, islands);
+  const scratch = new Map(adjacency);
+  const scratchState = { ...state, adjacency: scratch };
+  const pageBreak = (token.type === "paragraph" || token.type === "html") && isPageBreakLine(raw);
+  let json = pageBreak ? null : blockJSON(token, scratchState);
+  let lockedReason = pageBreak ? PAGE_BREAK_REASON : null;
+  if (pageBreak) {
+  } else if (json) {
+    try {
+      const node = schema.nodeFromJSON(json);
+      node.check();
+      const written = serializeBlock(node, createSerializeContext(scratch));
+      if (written !== raw) {
+        json = null;
+        lockedReason = "formatting the visual editor cannot keep byte-for-byte";
+      }
+    } catch {
+      json = null;
+      lockedReason = "structure the visual editor cannot hold";
+    }
+  } else {
+    lockedReason = LOCK_REASONS[token.type] ?? "formatting the visual editor cannot keep byte-for-byte";
+  }
+  if (json) {
+    for (const [key, value] of scratch) adjacency.set(key, value);
+  } else {
+    json = lockedJSON(raw, lockedReason ?? "source", nextId());
+  }
+  return { json, id: String(json.attrs?.mdId), raw, trail, lockedReason };
+}
+export {
+  PAGE_BREAK_REASON,
+  parseProseBlock
+};

package/dist/core/markdown-serialize.d.ts

@@ -0,0 +1,27 @@
+import type { Node as PMNode } from "@tiptap/pm/model";
+export interface Adjacency {
+    /** The exact bytes that followed this node in the source (its line breaks). */
+    readonly trail: string;
+    /** The id of the sibling that followed it, if any. */
+    readonly next: string | null;
+}
+export interface SerializeContext {
+    readonly adjacency: ReadonlyMap<string, Adjacency>;
+    /** Ids already written in this pass — a duplicate (copy/paste) is new text. */
+    readonly claimed: Set<string>;
+    /**
+     * Exact stored bytes for an unchanged node, when the caller holds them.
+     * Returning null means "serialize it".
+     */
+    readonly reuse?: (id: string, node: PMNode) => string | null;
+}
+export declare function createSerializeContext(adjacency?: ReadonlyMap<string, Adjacency>, reuse?: SerializeContext["reuse"]): SerializeContext;
+type ContainerKind = "wrapper" | "blockquote" | "listItem" | "doc";
+/** Serialize a container's block children with original spacing. */
+export declare function serializeChildren(parent: PMNode, kind: ContainerKind, ctx: SerializeContext): string;
+/** Serialize one block node (fresh — the caller decides about reuse). */
+export declare function serializeBlock(node: PMNode, ctx: SerializeContext): string;
+export type ColumnAlign = "left" | "center" | "right" | null;
+/** Serialize a textblock's inline content — marks nested as the source nested them. */
+export declare function serializeInline(parent: PMNode): string;
+export {};