@crossworks/content-core 0.230.43

package/src/markdown-refs.test.ts

@@ -0,0 +1,143 @@
+/**
+ * The shared reference-scheme helpers. Small surface, but two converters, the
+ * turn finalizer and the Telegram sender all read it, so the edges (partial
+ * markers mid-stream, a link that is not an image) are pinned here rather than
+ * re-discovered in each caller.
+ */
+import { describe, expect, it } from 'vitest';
+import {
+  fileRawSrc,
+  inlineMediaImageIds,
+  markdownRefs,
+  mediaFileId,
+  stripInlineMediaImages,
+  drawNodeId,
+} from './markdown-refs';
+
+describe('mediaFileId', () => {
+  it('extracts the id from a media: href', () => {
+    expect(mediaFileId('media:f-1')).toBe('f-1');
+    expect(mediaFileId('media:0e5c1a4e-1111-2222-3333-444455556666')).toBe(
+      '0e5c1a4e-1111-2222-3333-444455556666',
+    );
+  });
+
+  it('returns null for anything else', () => {
+    for (const href of ['https://x/y.png', 'page:p-9', 'mention:node:n-1', 'media:', '', undefined])
+      expect(mediaFileId(href)).toBeNull();
+  });
+});
+
+describe('fileRawSrc', () => {
+  it('builds the same path PageImage.renderHTML derives from nodeId', () => {
+    expect(fileRawSrc('f-1')).toBe('/api/files/files/f-1?raw=1');
+  });
+});
+
+describe('inlineMediaImageIds', () => {
+  it('collects every id the reply placed itself', () => {
+    const ids = inlineMediaImageIds('Step 1\n\n![a](media:f-1)\n\nStep 2\n\n![b](media:f-2)');
+    expect([...ids].sort()).toEqual(['f-1', 'f-2']);
+  });
+
+  it('counts a marker written mid-prose', () => {
+    expect([...inlineMediaImageIds('see ![x](media:f-9) here')]).toEqual(['f-9']);
+  });
+
+  it('ignores a file-embed LINK, which does not render the picture', () => {
+    expect(inlineMediaImageIds('[spec.pdf](media:f-2)').size).toBe(0);
+  });
+
+  it('ignores partial markers and non-media images', () => {
+    expect(inlineMediaImageIds('![a](media:').size).toBe(0);
+    expect(inlineMediaImageIds('![a](https://x/y.png)').size).toBe(0);
+    expect(inlineMediaImageIds('').size).toBe(0);
+    expect(inlineMediaImageIds(null).size).toBe(0);
+  });
+});
+
+describe('stripInlineMediaImages', () => {
+  it('drops a marker that owns its line, and the blank run it leaves', () => {
+    const { text, stripped } = stripInlineMediaImages(
+      'Step one.\n\n![the form](media:f-1)\n\nStep two.',
+    );
+    expect(text).toBe('Step one.\n\nStep two.');
+    expect(stripped).toBe(1);
+  });
+
+  it('leaves the alt text behind for a marker sitting in prose', () => {
+    const { text, stripped } = stripInlineMediaImages('Open ![the form](media:f-1) now.');
+    expect(text).toBe('Open the form now.');
+    expect(stripped).toBe(1);
+  });
+
+  it('leaves ordinary images and file-embed links alone', () => {
+    const src = 'See ![arch](https://x/y.png) and [spec.pdf](media:f-2).';
+    expect(stripInlineMediaImages(src)).toEqual({ text: src, stripped: 0 });
+  });
+
+  it('is a no-op on text with no markers', () => {
+    expect(stripInlineMediaImages('Just prose.')).toEqual({ text: 'Just prose.', stripped: 0 });
+  });
+});
+
+describe('markdownRefs', () => {
+  it('collects media, page and explicit node mentions with their required type', () => {
+    const refs = markdownRefs(
+      'Intro\n\n![a](media:f-1)\n\n[Spec](page:p-2) and [Bob](mention:node:n-3).',
+    );
+    expect(refs).toEqual([
+      { scheme: 'media', id: 'f-1', nodeType: 'file' },
+      { scheme: 'page', id: 'p-2', nodeType: 'page' },
+      { scheme: 'mention', id: 'n-3' },
+    ]);
+  });
+
+  it('treats a media file-EMBED link the same as an image — both need a real id', () => {
+    expect(markdownRefs('[spec.pdf](media:f-2)')).toEqual([
+      { scheme: 'media', id: 'f-2', nodeType: 'file' },
+    ]);
+  });
+
+  it('skips entity mentions, which name no node', () => {
+    expect(markdownRefs('[Acme](mention:entity:e-1) and [Acme](mention:e-2)')).toEqual([]);
+  });
+
+  it('ignores ordinary links and images', () => {
+    expect(markdownRefs('[docs](https://x/y) ![pic](https://x/y.png) ![a](media:')).toEqual([]);
+  });
+
+  it('dedupes a ref used twice', () => {
+    expect(markdownRefs('![a](media:f-1)\n\n![again](media:f-1)')).toEqual([
+      { scheme: 'media', id: 'f-1', nodeType: 'file' },
+    ]);
+  });
+
+  it('is safe on empty input', () => {
+    expect(markdownRefs('')).toEqual([]);
+    expect(markdownRefs(null)).toEqual([]);
+  });
+});
+
+describe('draw: references', () => {
+  it('extracts an embedded drawing as a draw-typed ref', () => {
+    const refs = markdownRefs('Here is the plan:\n\n![Architecture](draw:abc-123)\n');
+    expect(refs).toEqual([{ scheme: 'draw', id: 'abc-123', nodeType: 'draw' }]);
+  });
+
+  it('reads the id back off a draw href, and only a draw href', () => {
+    expect(drawNodeId('draw:abc-123')).toBe('abc-123');
+    expect(drawNodeId('media:abc-123')).toBeNull();
+    expect(drawNodeId('https://example.com/x.png')).toBeNull();
+    expect(drawNodeId(undefined)).toBeNull();
+  });
+
+  it('does not collide with media or page refs in one source', () => {
+    const refs = markdownRefs('![a](media:f1) ![b](draw:d1) [c](page:p1)');
+    expect(refs).toEqual([
+      { scheme: 'media', id: 'f1', nodeType: 'file' },
+      { scheme: 'draw', id: 'd1', nodeType: 'draw' },
+      { scheme: 'page', id: 'p1', nodeType: 'page' },
+    ]);
+  });
+});

package/src/markdown-refs.ts

@@ -0,0 +1,172 @@
+/**
+ * The reference-link schemes of Saskia's rich-markdown dialect: the round-trip
+ * forms `docToMarkdown` emits for app-native nodes (see rich-writing.md §2):
+ *
+ *   ![alt](media:<file-id>)      an uploaded image, by node id
+ *   [label](media:<file-id>)     a file embed
+ *   [Title](page:<page-id>)      a child page
+ *   ![alt](draw:<draw-id>)       an embedded drawing
+ *   [Label](mention:<ref>:<id>)  a mention chip
+ *
+ * Kept here, alone, with **no dependencies**, because two converters read these
+ * schemes and they must not drift:
+ *
+ *   - `markdown-to-doc.ts`            → ProseMirror JSON (Pages, server-side)
+ *   - `client/web/lib/rich-markdown`  → HTML (the chat RichText, client-side)
+ *
+ * They diverged once already: Pages resolved `media:` into a real image while
+ * chat let it fall through to a broken `<img src="media:…">`, which is why a
+ * reply could not place a picture mid-sentence. Both now import the scheme from
+ * here, and a drift test pins them to the same answers.
+ *
+ * Browser-safe leaf, importable from client code without dragging the
+ * `@mantle/content` barrel (and its DB deps) into the bundle.
+ */
+
+/** `[Label](mention:node:<id>)` / `[Label](mention:entity:<id>)`. The ref
+ *  segment is optional and defaults to `entity` at the call site. */
+export const MENTION_HREF = /^mention:(?:(node|entity):)?([^\s]+)$/;
+/** `media:<file-id>`, an uploaded file by node id. */
+export const MEDIA_HREF = /^media:([^\s]+)$/;
+/** `page:<page-id>`, a child page by node id. */
+export const PAGE_HREF = /^page:([^\s]+)$/;
+/** `draw:<draw-id>`, a drawing embedded as a picture. Deliberately a LIVE
+ *  reference, not a copy: the page renders the drawing's current committed
+ *  snapshot, so editing the drawing updates every page that embeds it. */
+export const DRAW_HREF = /^draw:([^\s]+)$/;
+
+/** The draw node id behind a `draw:` href, or null for any other href. */
+export function drawNodeId(href: string | undefined | null): string | null {
+  return DRAW_HREF.exec(href ?? '')?.[1] ?? null;
+}
+
+/** The owner-gated serve path for a drawing's committed snapshot. The one
+ *  place this route is spelled out for markdown conversion, mirroring
+ *  `fileRawSrc` for uploaded files. */
+export function drawRawSrc(nodeId: string): string {
+  // Encoded: DRAW_HREF accepts any non-whitespace run, so a hand-written
+  // `![x](draw:../../foo)` or an id carrying `?`/`#` would otherwise reshape
+  // the path or the query it is spliced into.
+  return `/api/draws/${encodeURIComponent(nodeId)}/svg?raw=1`;
+}
+
+/** The file node id behind a `media:` href, or null for any other href. */
+export function mediaFileId(href: string | undefined | null): string | null {
+  return MEDIA_HREF.exec(href ?? '')?.[1] ?? null;
+}
+
+/**
+ * The owner-gated serve path for a file's bytes. This is the ONE place the
+ * route is spelled out for markdown conversion; `PageImage.renderHTML` builds
+ * the same string from `nodeId`, so an emitted `<img>` and a re-rendered one
+ * agree. Unencoded on purpose, so it matches what the node itself produces.
+ */
+export function fileRawSrc(nodeId: string): string {
+  return `/api/files/files/${nodeId}?raw=1`;
+}
+
+/**
+ * Every file id referenced as an INLINE IMAGE (`![alt](media:<id>)`) in a
+ * markdown source. Used to suppress the trailing attachment gallery for a
+ * picture the reply already placed itself. Otherwise a turn that writes the
+ * image inline *and* calls `show_image` for the same file shows it twice.
+ *
+ * Deliberately images only: a `[label](media:<id>)` file-embed link is a
+ * different affordance and does not render the picture, so it must not
+ * suppress the gallery copy.
+ */
+export function inlineMediaImageIds(source: string | undefined | null): Set<string> {
+  const ids = new Set<string>();
+  if (!source) return ids;
+  const re = /!\[[^\]]*\]\(\s*media:([^\s)]+)\s*\)/g;
+  for (const m of source.matchAll(re)) if (m[1]) ids.add(m[1]);
+  return ids;
+}
+
+/** A reference-link target found in a markdown source: which scheme it used,
+ *  the id it points at, and the node type that scheme requires (unset ⇒ any
+ *  node). `nodeType` is what makes a wrong-type reference reportable. */
+export type MarkdownRef = {
+  scheme: 'media' | 'page' | 'mention' | 'draw';
+  id: string;
+  nodeType?: 'file' | 'page' | 'draw';
+};
+
+/** Any markdown link or image, capturing the href. */
+const REF_LINK_RE = /!?\[[^\]]*\]\(\s*([^\s)]+)\s*\)/g;
+
+/**
+ * Every app-native reference a markdown source points at, deduped. The write
+ * path checks these against real nodes before storing the body — a dangling
+ * `media:` id renders as nothing at all, silently, and the model that wrote it
+ * gets no feedback (see packages/tools/src/preconditions.ts).
+ *
+ * `mention:entity:<id>` is deliberately NOT returned: entities live outside the
+ * node table, so there is no node id to check. Only the explicit
+ * `mention:node:<id>` form yields a ref.
+ */
+export function markdownRefs(source: string | undefined | null): MarkdownRef[] {
+  const out: MarkdownRef[] = [];
+  if (!source) return out;
+  const seen = new Set<string>();
+  for (const m of source.matchAll(REF_LINK_RE)) {
+    const href = m[1];
+    if (!href) continue;
+    let ref: MarkdownRef | null = null;
+    const media = MEDIA_HREF.exec(href);
+    if (media?.[1]) ref = { scheme: 'media', id: media[1], nodeType: 'file' };
+    const page = PAGE_HREF.exec(href);
+    if (page?.[1]) ref = { scheme: 'page', id: page[1], nodeType: 'page' };
+    const draw = DRAW_HREF.exec(href);
+    if (draw?.[1]) ref = { scheme: 'draw', id: draw[1], nodeType: 'draw' };
+    const mention = MENTION_HREF.exec(href);
+    // Bare `mention:<id>` defaults to an entity — skip; only `mention:node:` is
+    // a node reference.
+    if (mention?.[1] === 'node' && mention[2]) ref = { scheme: 'mention', id: mention[2] };
+    if (!ref) continue;
+    const key = `${ref.scheme}:${ref.id}`;
+    if (seen.has(key)) continue;
+    seen.add(key);
+    out.push(ref);
+  }
+  return out;
+}
+
+/**
+ * Drop inline `![alt](media:<id>)` image markers from a markdown source, for a
+ * surface that cannot render them (Telegram sends plain text, so the marker
+ * would arrive as literal `![...](media:...)` gibberish). Returns the cleaned text
+ * plus how many markers were removed, so the caller can trace it.
+ *
+ * A marker alone on its line takes the line with it; one sitting in prose
+ * leaves its alt text behind, which is the closest thing to the picture the
+ * surface can carry.
+ */
+export function stripInlineMediaImages(source: string): { text: string; stripped: number } {
+  let stripped = 0;
+  const lines = source.replace(/\r\n/g, '\n').split('\n');
+  const kept: string[] = [];
+  for (const line of lines) {
+    const soleMarker = /^\s*!\[([^\]]*)\]\(\s*media:[^\s)]+\s*\)\s*$/.exec(line);
+    if (soleMarker) {
+      stripped++;
+      continue; // drop the whole line, nothing else was on it
+    }
+    if (!/!\[[^\]]*\]\(\s*media:[^\s)]+\s*\)/.test(line)) {
+      kept.push(line);
+      continue;
+    }
+    kept.push(
+      line.replace(/!\[([^\]]*)\]\(\s*media:[^\s)]+\s*\)/g, (_all, alt: string) => {
+        stripped++;
+        return alt;
+      }),
+    );
+  }
+  // Collapse the blank-line runs a dropped marker line can leave behind.
+  const text = kept
+    .join('\n')
+    .replace(/\n{3,}/g, '\n\n')
+    .trim();
+  return { text, stripped };
+}

package/src/markdown-to-doc.test.ts

@@ -0,0 +1,179 @@
+import { describe, expect, it } from 'vitest';
+import { markdownToDoc } from './markdown-to-doc';
+
+type N = {
+  type: string;
+  attrs?: Record<string, unknown>;
+  content?: N[];
+  text?: string;
+  marks?: { type: string }[];
+};
+const top = (md: string) => (markdownToDoc(md) as { content: N[] }).content;
+const find = (md: string, type: string) => top(md).find((n) => n.type === type);
+
+describe('markdownToDoc', () => {
+  it('always returns a doc with at least an empty paragraph', () => {
+    const doc = markdownToDoc('') as N;
+    expect(doc.type).toBe('doc');
+    expect(doc.content?.[0]?.type).toBe('paragraph');
+  });
+
+  it('maps headings (clamped to 1–3) and inline marks', () => {
+    const h = find('## Title', 'heading');
+    expect(h?.attrs?.level).toBe(2);
+    const big = find('###### deep', 'heading');
+    expect(big?.attrs?.level).toBe(3);
+    const p = find('a **b** *c* `d` ==e==', 'paragraph')!;
+    const markTypes = (p.content ?? []).flatMap((t) => (t.marks ?? []).map((m) => m.type));
+    expect(markTypes).toEqual(expect.arrayContaining(['bold', 'italic', 'code', 'highlight']));
+  });
+
+  it('maps [text]{color=…}/{highlight=…} to themed textColor/highlight marks', () => {
+    const p = find(
+      'a [b]{color=chart-2} [c]{highlight=chart-4} [d]{color=chart-1 highlight=chart-3}',
+      'paragraph',
+    )!;
+    const marks = (p.content ?? []).flatMap(
+      (t) => (t.marks ?? []) as Array<{ type: string; attrs?: { color?: string } }>,
+    );
+    const has = (type: string, color: string) =>
+      marks.some((m) => m.type === type && m.attrs?.color === color);
+    expect(has('textColor', 'chart-2')).toBe(true);
+    expect(has('highlight', 'chart-4')).toBe(true);
+    expect(has('textColor', 'chart-1')).toBe(true); // both keys on one span
+    expect(has('highlight', 'chart-3')).toBe(true);
+    // Unknown token / non-colour attr leaves plain text (no colour mark).
+    const plain = find('x [y]{color=red} [z]{foo=bar}', 'paragraph')!;
+    const types = (plain.content ?? []).flatMap((t) => (t.marks ?? []).map((m) => m.type));
+    expect(types).not.toContain('textColor');
+  });
+
+  it('maps callouts with a variant, defaulting unknown kinds to info', () => {
+    const c = find(':::warning\nbe careful\n:::', 'callout');
+    expect(c?.attrs?.variant).toBe('warning');
+    const d = find(':::bogus\nx\n:::', 'callout');
+    expect(d?.attrs?.variant).toBe('info');
+  });
+
+  it('maps asides, reading an optional themed colour (default chart-1)', () => {
+    const a = find(':::aside\na side note\n:::', 'aside');
+    expect(a?.attrs?.color).toBe('chart-1');
+    expect(a?.attrs?.angle).toBe(135);
+    expect(a?.content?.[0]?.type).toBe('paragraph');
+    const c = find(':::aside chart-3\ntinted\n:::', 'aside');
+    expect(c?.attrs?.color).toBe('chart-3');
+    // An out-of-range colour falls back to chart-1.
+    expect(find(':::aside chart-9\nx\n:::', 'aside')?.attrs?.color).toBe('chart-1');
+  });
+
+  it("imports Notion's <aside> … </aside> callout export as an aside block", () => {
+    const a = find('<aside>\n💡 A Notion callout.\n</aside>', 'aside');
+    expect(a?.attrs?.color).toBe('chart-1');
+    expect(a?.attrs?.angle).toBe(135);
+    expect(a?.content?.[0]?.type).toBe('paragraph');
+    // the leading-emoji text rides along in the body (no literal <aside> text)
+    const text = JSON.stringify(a);
+    expect(text).toContain('A Notion callout.');
+    expect(text).not.toContain('aside>');
+  });
+
+  it('handles a single-line <aside>…</aside> and multi-block bodies', () => {
+    const one = find('<aside>just one line</aside>', 'aside');
+    expect(one?.content?.[0]?.type).toBe('paragraph');
+    const multi = find('<aside>\n# Heading\n\nA paragraph.\n</aside>', 'aside');
+    const kinds = (multi?.content ?? []).map((b) => b.type);
+    expect(kinds).toContain('heading');
+    expect(kinds).toContain('paragraph');
+  });
+
+  it('cycles colour/angle across multiple imported <aside> blocks', () => {
+    const doc = markdownToDoc(
+      '<aside>\none\n</aside>\n\n<aside>\ntwo\n</aside>\n\n<aside>\nthree\n</aside>',
+    ) as { content: N[] };
+    const asides = doc.content.filter((n) => n.type === 'aside');
+    expect(asides.map((a) => a.attrs?.color)).toEqual(['chart-1', 'chart-2', 'chart-3']);
+    expect(asides[0]?.attrs?.angle).toBe(135);
+    expect(asides[1]?.attrs?.angle).toBe(60);
+  });
+
+  it('maps a columns block into columnList with 2+ columns', () => {
+    const cols = find(':::columns\nleft\n+++\nright\n:::', 'columnList');
+    expect(cols?.content?.length).toBe(2);
+    expect(cols?.content?.[0]?.type).toBe('column');
+  });
+
+  it('degrades a single-column columns block to plain blocks', () => {
+    expect(find(':::columns\nonly one\n:::', 'columnList')).toBeUndefined();
+    expect(find(':::columns\nonly one\n:::', 'paragraph')).toBeTruthy();
+  });
+
+  it('maps GFM task lists to taskList/taskItem with checked state', () => {
+    const tl = find('- [x] done\n- [ ] todo', 'taskList');
+    expect(tl?.content?.map((i) => i.attrs?.checked)).toEqual([true, false]);
+  });
+
+  it('maps fenced code with its language', () => {
+    const code = find('```ts\nconst x = 1;\n```', 'codeBlock');
+    expect(code?.attrs?.language).toBe('ts');
+    expect(code?.content?.[0]?.text).toContain('const x = 1;');
+  });
+
+  it('maps a ```mermaid fence to a diagram node with the source verbatim', () => {
+    const source = 'flowchart LR\n  A[Start] --> B[End]';
+    const d = find('```mermaid\n' + source + '\n```', 'diagram');
+    expect(d?.attrs?.source).toBe(source);
+    expect(d?.content).toBeUndefined();
+    // Case-insensitive language tag; other languages stay code blocks.
+    expect(find('```Mermaid\ngraph TD\n```', 'diagram')).toBeTruthy();
+    expect(find('```mermaidjs\nx\n```', 'diagram')).toBeUndefined();
+    expect(find('```mermaidjs\nx\n```', 'codeBlock')).toBeTruthy();
+    // marked's `lang` is the FULL info string — match on its first word, so
+    // '```mermaid title=x' still lands as a diagram.
+    expect(find('```mermaid title=x\ngraph TD\n```', 'diagram')).toBeTruthy();
+  });
+
+  it('maps a mermaid fence inside a callout to a nested diagram node', () => {
+    const callout = find(':::info\n```mermaid\npie\n  "a": 1\n```\n:::', 'callout');
+    expect(callout?.content?.some((n) => n.type === 'diagram')).toBe(true);
+  });
+
+  it('maps a GFM table to table/tableRow/tableHeader/tableCell', () => {
+    const t = find('| A | B |\n|---|---|\n| 1 | 2 |', 'table')!;
+    expect(t.content?.[0]?.content?.[0]?.type).toBe('tableHeader');
+    expect(t.content?.[1]?.content?.[0]?.type).toBe('tableCell');
+  });
+
+  it('lifts a markdown image into a block image node', () => {
+    const img = find('![arch](https://x/y.png)', 'image');
+    expect(img?.attrs?.src).toBe('https://x/y.png');
+    expect(img?.attrs?.alt).toBe('arch');
+  });
+
+  it('maps reference-link schemes to their app nodes', () => {
+    const mention = top('hi [Sarah](mention:entity:n-1)')
+      .find((n) => n.type === 'paragraph')!
+      .content!.find((c) => c.type === 'mention');
+    expect(mention?.attrs).toMatchObject({ id: 'n-1', label: 'Sarah', ref: 'entity' });
+    expect(find('![gantry](media:f-1)', 'image')?.attrs?.nodeId).toBe('f-1');
+    expect(find('[spec.pdf](media:f-2)', 'fileEmbed')?.attrs).toMatchObject({
+      nodeId: 'f-2',
+      filename: 'spec.pdf',
+    });
+    expect(find('[Sub plan](page:p-9)', 'childPage')?.attrs).toMatchObject({
+      pageId: 'p-9',
+      title: 'Sub plan',
+    });
+    // Inline (mixed with prose) media:/page: links stay plain links.
+    expect(find('see [spec.pdf](media:f-2) here', 'fileEmbed')).toBeUndefined();
+    // Ordinary links are untouched.
+    expect(find('[site](https://x.io)', 'fileEmbed')).toBeUndefined();
+    expect(find('[site](https://x.io)', 'childPage')).toBeUndefined();
+  });
+
+  it('maps $…$ to inline math and $$…$$ to block math', () => {
+    const p = top('Inline $E=mc^2$ here').find((n) => n.type === 'paragraph')!;
+    expect((p.content ?? []).some((c) => c.type === 'inlineMath')).toBe(true);
+    expect(find('$$\nx^2\n$$', 'blockMath')?.attrs?.latex).toBe('x^2');
+    expect(find('$$a+b$$', 'blockMath')?.attrs?.latex).toBe('a+b');
+  });
+});