mixdog 0.8.0 → 0.9.0

package/src/tui/markdown/format-token.test.mjs

@@ -0,0 +1,216 @@
+import test from 'node:test';
+import assert from 'node:assert/strict';
+import { marked } from 'marked';
+import stripAnsi from 'strip-ansi';
+import stringWidth from 'string-width';
+import { formatToken } from './format-token.mjs';
+import { setThemeSetting, theme } from '../theme.mjs';
+
+setThemeSetting('mixdog', { persist: false });
+
+function rgbSgr(value) {
+  const m = /^rgb\((\d+),\s*(\d+),\s*(\d+)\)$/.exec(String(value || ''));
+  return m ? `38;2;${m[1]};${m[2]};${m[3]}` : null;
+}
+
+function lexFirst(md, type) {
+  const tokens = marked.lexer(md);
+  return tokens.find((t) => t.type === type) ?? tokens[0];
+}
+
+test('fenced code block renders lang label without ``` fences and indented body', () => {
+  const token = lexFirst('```js\nconst x = 1;\n```\n', 'code');
+  const out = formatToken(token);
+  const plain = stripAnsi(out);
+  assert.ok(plain.includes('js'), 'language label visible');
+  assert.ok(!plain.includes('```'), 'no literal fence markers');
+  assert.ok(plain.includes('  const x = 1;'), 'body is two-space indented');
+  assert.ok(out.includes(rgbSgr(theme.mdCodeBlockBorder)), 'lang label uses border color');
+});
+
+test('short code line has no terminal-width trailing padding band', () => {
+  const token = lexFirst('```js\nshort\n```\n', 'code');
+  const out = formatToken(token, 0, null, null, 60);
+  const bodyLine = stripAnsi(out).split('\n').find((l) => l.includes('short'));
+  assert.ok(bodyLine, 'body line present');
+  assert.equal(bodyLine, '  short', 'body line is only indent + text');
+});
+
+test('long code line wraps within requested body width', () => {
+  const long = 'a'.repeat(40);
+  const token = lexFirst(`\`\`\`js\n${long}\n\`\`\`\n`, 'code');
+  const width = 20;
+  const out = formatToken(token, 0, null, null, width);
+  const bodyLines = stripAnsi(out).split('\n').filter((l) => l.startsWith('  '));
+  assert.ok(bodyLines.length > 1, 'long line is wrapped into multiple rows');
+  for (const line of bodyLines) {
+    assert.ok(line.length <= width, `wrapped line "${line}" fits width ${width}`);
+  }
+  assert.equal(bodyLines.join('').replace(/\s/g, ''), long, 'wrapped segments preserve content');
+});
+
+test('list item fenced code block respects narrowed width and has no fences', () => {
+  const long = 'b'.repeat(36);
+  const codeTok = lexFirst(`\`\`\`js\n${long}\n\`\`\`\n`, 'code');
+  const listTok = {
+    type: 'list',
+    ordered: false,
+    items: [{ type: 'list_item', tokens: [codeTok] }],
+  };
+  const outerWidth = 22;
+  const out = formatToken(listTok, 0, null, null, outerWidth);
+  const plain = stripAnsi(out);
+  assert.ok(!plain.includes('```'), 'no literal fence markers');
+  const prefixed = plain.split('\n').filter((l) => l.includes('b'));
+  assert.ok(prefixed.length > 1, 'nested list code wraps under narrow width');
+  for (const line of prefixed) {
+    assert.ok(stringWidth(line) <= outerWidth, `line fits outer width: "${line}"`);
+  }
+});
+
+test('blockquote fenced code block respects narrowed width and has no fences', () => {
+  const long = 'c'.repeat(36);
+  const codeTok = lexFirst(`\`\`\`js\n${long}\n\`\`\`\n`, 'code');
+  const quoteTok = { type: 'blockquote', tokens: [codeTok] };
+  const outerWidth = 24;
+  const out = formatToken(quoteTok, 0, null, null, outerWidth);
+  const plain = stripAnsi(out);
+  assert.ok(!plain.includes('```'), 'no literal fence markers');
+  const lines = plain.split('\n').filter((l) => l.includes('c'));
+  assert.ok(lines.length > 1, 'blockquote code wraps under narrow width');
+  for (const line of lines) {
+    assert.ok(stringWidth(line) <= outerWidth, `line fits outer width: "${line}"`);
+  }
+});
+
+test('js highlighting colors keyword, string, number distinctly', () => {
+  const token = lexFirst('```js\nconst n = 42;\nlet s = "hi";\n```\n', 'code');
+  const out = formatToken(token);
+  assert.ok(out.includes(rgbSgr(theme.syntaxKeyword)), 'keyword colored');
+  assert.ok(out.includes(rgbSgr(theme.syntaxNumber)), 'number colored');
+  assert.ok(out.includes(rgbSgr(theme.syntaxString)), 'string colored');
+});
+
+test('diff fenced block colors add/remove/hunk/header separately', () => {
+  const diff = [
+    '```diff',
+    'diff --git a/x b/x',
+    '--- a/x',
+    '+++ b/x',
+    '@@ -1,2 +1,2 @@',
+    '-old line',
+    '+new line',
+    ' context line',
+    '```',
+    '',
+  ].join('\n');
+  const token = lexFirst(diff, 'code');
+  const out = formatToken(token);
+  const added = rgbSgr(theme.mdDiffAdded);
+  const removed = rgbSgr(theme.mdDiffRemoved);
+  const hunk = rgbSgr(theme.mdDiffHunk);
+  const header = rgbSgr(theme.mdDiffHeader);
+  assert.ok(out.includes(added), 'added line color present');
+  assert.ok(out.includes(removed), 'removed line color present');
+  assert.ok(out.includes(hunk), 'hunk line color present');
+  assert.ok(out.includes(header), 'header line color present');
+  assert.equal(new Set([added, removed, hunk, header]).size, 4);
+  assert.ok(out.indexOf('+++ b/x') !== -1);
+});
+
+test('plus/minus file headers are not treated as add/remove', () => {
+  const diff = '```patch\n--- a/f\n+++ b/f\n@@ -0,0 +1 @@\n+x\n```\n';
+  const token = lexFirst(diff, 'code');
+  const out = formatToken(token);
+  const headerSgr = rgbSgr(theme.mdDiffHeader);
+  const addedSgr = rgbSgr(theme.mdDiffAdded);
+  const idxHeader = out.indexOf('+++ b/f');
+  const before = out.slice(0, idxHeader);
+  const lastHeader = before.lastIndexOf(headerSgr);
+  const lastAdded = before.lastIndexOf(addedSgr);
+  assert.ok(lastHeader > lastAdded, 'plus-plus-plus header uses header color, not added');
+});
+
+test('bare unified diff (no lang) is detected and colored', () => {
+  const token = lexFirst('```\n@@ -1 +1 @@\n-a\n+b\n```\n', 'code');
+  const out = formatToken(token);
+  assert.ok(out.includes(rgbSgr(theme.mdDiffHunk)), 'hunk colored on bare diff');
+  assert.ok(out.includes(rgbSgr(theme.mdDiffAdded)), 'add colored on bare diff');
+});
+
+test('unknown language falls back to flat code block color', () => {
+  const token = lexFirst('```foobarlang\nsome text\n```\n', 'code');
+  const out = formatToken(token);
+  assert.ok(out.includes(rgbSgr(theme.mdCodeBlock)), 'flat code-block color used');
+  assert.ok(stripAnsi(out).includes('  some text'), 'body still indented');
+});
+
+test('inline link emits OSC 8 hyperlink with styled label (URL hidden)', () => {
+  const token = lexFirst('[label](https://example.com)', 'paragraph');
+  const out = formatToken(token);
+  // OSC 8 wraps the visible label; the raw URL lives in the escape, not as
+  // on-screen text, so stripAnsi keeps only the label.
+  assert.ok(out.includes('\x1b]8;;https://example.com\x07'), 'OSC 8 link target present');
+  assert.ok(out.includes('\x1b]8;;\x07'), 'OSC 8 close present');
+  assert.ok(stripAnsi(out).includes('label'), 'link label text visible');
+  assert.ok(out.includes(rgbSgr(theme.mdLinkText)), 'link label colored');
+});
+
+test('bare-url link emits OSC 8 with the URL as visible clickable text', () => {
+  const token = lexFirst('<https://example.com>', 'paragraph');
+  const out = formatToken(token);
+  assert.ok(out.includes('\x1b]8;;https://example.com\x07'), 'OSC 8 link target present');
+  assert.ok(stripAnsi(out).includes('https://example.com'), 'URL visible as text');
+  assert.ok(out.includes(rgbSgr(theme.mdLink)), 'link URL colored');
+});
+
+test('strong and em use distinct mdStrong / mdEmph colors', () => {
+  const strongTok = lexFirst('**bold**', 'paragraph');
+  const emTok = lexFirst('*ital*', 'paragraph');
+  const strongOut = formatToken(strongTok);
+  const emOut = formatToken(emTok);
+  assert.ok(strongOut.includes(rgbSgr(theme.mdStrong)), 'strong colored');
+  assert.ok(emOut.includes(rgbSgr(theme.mdEmph)), 'em colored');
+  assert.ok(strongOut.includes('\x1b[1m'), 'strong is bold');
+  assert.ok(emOut.includes('\x1b[3m'), 'em is italic');
+});
+
+test('h1 heading is bold, italic, and underlined', () => {
+  const token = lexFirst('# Title', 'heading');
+  const out = formatToken(token);
+  assert.ok(out.includes('\x1b[1m'), 'h1 is bold');
+  assert.ok(out.includes('\x1b[3m'), 'h1 is italic');
+  assert.ok(out.includes('\x1b[4m'), 'h1 is underlined');
+});
+
+test('ordered list markers follow nesting depth (1. / a. / i.)', () => {
+  const md = [
+    '1. one',
+    '   1. two',
+    '      1. three',
+    '         1. four',
+  ].join('\n');
+  const tokens = marked.lexer(md);
+  const plain = stripAnsi(tokens.map((t) => formatToken(t)).join(''));
+  assert.ok(plain.includes('1. one'), 'depth 0 arabic');
+  assert.ok(plain.includes('1. two'), 'depth 1 arabic');
+  assert.ok(plain.includes('a. three'), 'depth 2 lowercase letter');
+  assert.ok(plain.includes('i. four'), 'depth 3 lowercase roman');
+});
+
+test('every palette defines the full extended key set', () => {
+  const required = [
+    'mdCodeBlockBorder', 'mdCodeBlockBg', 'mdLink', 'mdLinkText', 'mdStrong', 'mdEmph',
+    'mdDiffAdded', 'mdDiffRemoved', 'mdDiffHunk', 'mdDiffHeader', 'mdDiffContext',
+    'mdDiffAddedBg', 'mdDiffRemovedBg',
+    'syntaxComment', 'syntaxKeyword', 'syntaxFunction', 'syntaxVariable',
+    'syntaxString', 'syntaxNumber', 'syntaxType', 'syntaxOperator', 'syntaxPunctuation',
+  ];
+  for (const id of ['mixdog', 'pi-dark', 'claude-dark', 'dracula', 'tokyonight', 'nord', 'gruvbox', 'catppuccin', 'everforest']) {
+    setThemeSetting(id, { persist: false });
+    for (const key of required) {
+      assert.ok(/^rgb\(\d+,\s*\d+,\s*\d+\)$/.test(String(theme[key])), `${id}.${key} is an rgb() string`);
+    }
+  }
+  setThemeSetting('mixdog', { persist: false });
+});

package/src/tui/markdown/render-ansi.mjs

@@ -0,0 +1,94 @@
+/**
+ * markdown/render-ansi.mjs — markdown → token + ANSI helpers (no JSX).
+ *
+ * The lexing/token-cache and the partial-fence trim decision live here, free of
+ * the ink/JSX render stack, so the exact streaming-vs-stable token path the
+ * <Markdown> component renders is unit-testable without an ink renderer.
+ *
+ *   - lexMarkdown(content, { trimPartialFences }) → marked tokens. Stable
+ *     (non-streaming) content is cached; the streaming suffix is lexed fresh and
+ *     trimmed (a partial closing fence is dropped) and never cached, so trimming
+ *     can never corrupt the shared cache used by stable text.
+ *   - renderTokenAnsiSegments(content, opts) → array of { type, ansi?, token }
+ *     describing what the component emits: 'table' segments carry the token for
+ *     MarkdownTable; 'ansi' segments carry the formatToken() string.
+ */
+import { marked } from 'marked';
+import { formatToken } from './format-token.mjs';
+import { trimPartialClosingFences } from './stream-fence.mjs';
+
+const TOKEN_CACHE_MAX = 500;
+const tokenCache = new Map();
+const MD_SYNTAX_RE = /[#*`|[>\-_~]|\n\n|^\d+\. |\n\d+\. /;
+
+let _configured = false;
+export function configureMarked() {
+  if (_configured) return;
+  _configured = true;
+  // Disable strikethrough: models use ~ for "approximate" (~100), not <del>.
+  marked.use({ tokenizer: { del() { return undefined; } } });
+}
+
+export function hasMarkdownSyntax(text) {
+  const value = String(text ?? '');
+  return MD_SYNTAX_RE.test(value);
+}
+
+export function lexMarkdown(content, { trimPartialFences = false } = {}) {
+  configureMarked();
+  const text = String(content ?? '');
+  if (!hasMarkdownSyntax(text)) {
+    return [{
+      type: 'paragraph',
+      raw: text,
+      text,
+      tokens: [{ type: 'text', raw: text, text }],
+    }];
+  }
+  // Streaming suffix: lex fresh and trim a partial closing fence so an open code
+  // block does not include a lone trailing backtick as a body line. These tokens
+  // are transient per-delta and trimming mutates them in place, so they are NOT
+  // cached — caching would corrupt the shared cache for stable text.
+  if (trimPartialFences) {
+    const tokens = marked.lexer(text);
+    trimPartialClosingFences(tokens);
+    return tokens;
+  }
+  const hit = tokenCache.get(text);
+  if (hit) {
+    tokenCache.delete(text);
+    tokenCache.set(text, hit);
+    return hit;
+  }
+  const tokens = marked.lexer(text);
+  if (tokenCache.size >= TOKEN_CACHE_MAX) {
+    const first = tokenCache.keys().next().value;
+    if (first !== undefined) tokenCache.delete(first);
+  }
+  tokenCache.set(text, tokens);
+  return tokens;
+}
+
+/**
+ * Render markdown to an ordered list of segments the component maps to ink:
+ *   { type: 'table', token }      → <MarkdownTable token={token} />
+ *   { type: 'ansi', ansi: string } → <AnsiText>{ansi}</AnsiText>
+ * Blank-edge-only ansi segments are dropped (mirrors the component's pushAnsi).
+ */
+export function renderTokenAnsiSegments(content, opts = {}) {
+  const tokens = lexMarkdown(content, opts);
+  const width = Number(opts.width) || 0;
+  const segments = [];
+  for (const token of tokens) {
+    if (token.type === 'table') {
+      segments.push({ type: 'table', token });
+    } else if (token.type === 'space') {
+      continue;
+    } else {
+      const ansi = String(formatToken(token, 0, null, null, width) ?? '').replace(/^\n+|\n+$/g, '');
+      if (!ansi) continue;
+      segments.push({ type: 'ansi', ansi, token });
+    }
+  }
+  return segments;
+}

package/src/tui/markdown/render-ansi.test.mjs

@@ -0,0 +1,108 @@
+import test from 'node:test';
+import assert from 'node:assert/strict';
+import stripAnsi from 'strip-ansi';
+import { assistantBodyWidth } from './table-layout.mjs';
+import { renderTokenAnsiSegments, lexMarkdown, hasMarkdownSyntax } from './render-ansi.mjs';
+import { setThemeSetting } from '../theme.mjs';
+
+setThemeSetting('mixdog', { persist: false });
+
+// Mirrors the streaming suffix the <Markdown trimPartialFences> path renders.
+const STREAMING = '```js\nconst x = 1;\n`';
+
+function codeBlockVisibleLines(segments) {
+  const codeSeg = segments.find((s) => s.type === 'ansi' && s.token?.type === 'code');
+  assert.ok(codeSeg, 'a fenced code segment is rendered');
+  return stripAnsi(codeSeg.ansi).split('\n');
+}
+
+test('streaming render path drops the partial trailing backtick body line', () => {
+  const segments = renderTokenAnsiSegments(STREAMING, { trimPartialFences: true });
+  const lines = codeBlockVisibleLines(segments);
+  const bodyLines = lines.filter((l) => !/^js$/.test(l.trim()));
+  for (const l of bodyLines) {
+    assert.notEqual(l.trim(), '`', 'no body line is a lone partial fence');
+  }
+  // The real code body is still present.
+  assert.ok(lines.some((l) => l.includes('const x = 1;')), 'code body preserved');
+});
+
+test('without trimPartialFences the partial backtick survives as a body line', () => {
+  // Proves the trim flag is what removes it on the render path (not formatToken).
+  const segments = renderTokenAnsiSegments(STREAMING, { trimPartialFences: false });
+  const lines = codeBlockVisibleLines(segments);
+  const bodyLines = lines.filter((l) => !/^js$/.test(l.trim()));
+  assert.ok(
+    bodyLines.some((l) => l.trim() === '`'),
+    'untrimmed path keeps the partial fence as a body line',
+  );
+});
+
+test('trimPartialFences tokens are not cached (stable text unaffected)', () => {
+  const stable = '```js\nconst y = 2;\n```\n';
+  const a = lexMarkdown(stable, { trimPartialFences: false });
+  const b = lexMarkdown(stable, { trimPartialFences: false });
+  assert.strictEqual(a, b, 'stable content returns the cached token array');
+  // Streaming lex must be a fresh (non-cached) array.
+  const c = lexMarkdown(STREAMING, { trimPartialFences: true });
+  const d = lexMarkdown(STREAMING, { trimPartialFences: true });
+  assert.notStrictEqual(c, d, 'streaming lex is never cached');
+});
+
+test('closed code block renders lang label and body without fence markers', () => {
+  const segments = renderTokenAnsiSegments('```js\nconst z = 3;\n```\n', { trimPartialFences: true });
+  const lines = codeBlockVisibleLines(segments);
+  assert.ok(lines.some((l) => l.includes('const z = 3;')), 'body present');
+  assert.ok(lines.some((l) => l.trim() === 'js'), 'language label present');
+  assert.ok(!lines.some((l) => l.includes('```')), 'no literal fence markers');
+});
+
+test('renderTokenAnsiSegments width matches assistant body width for code pre-wrap', () => {
+  const long = 'z'.repeat(60);
+  const md = `\`\`\`js\n${long}\n\`\`\``;
+  const bodyWidth = assistantBodyWidth(40);
+  const atBody = renderTokenAnsiSegments(md, { width: bodyWidth });
+  const atDefault = renderTokenAnsiSegments(md, { width: 80 });
+  const bodyLines = (seg) => stripAnsi(seg.find((s) => s.token?.type === 'code').ansi)
+    .split('\n')
+    .filter((l) => l.startsWith('  '));
+  assert.ok(bodyLines(atBody).length > bodyLines(atDefault).length,
+    'narrow body width produces more wrapped code rows than width 80');
+});
+
+const PLAIN_PREFIX = 'x'.repeat(501);
+
+test('hasMarkdownSyntax detects strong markers after a 500+ char plain prefix', () => {
+  assert.ok(hasMarkdownSyntax(`${PLAIN_PREFIX}**late bold**`));
+});
+test('hasMarkdownSyntax detects inline code after a 500+ char plain prefix', () => {
+  assert.ok(hasMarkdownSyntax(`${PLAIN_PREFIX}\`late code\``));
+});
+
+test('render path parses fenced code after a 500+ char plain prefix', () => {
+  const input = `${PLAIN_PREFIX}\n\`\`\`js\nconst late = true;\n\`\`\``;
+  const tokens = lexMarkdown(input);
+  assert.ok(
+    tokens.some((t) => t.type === 'code' && String(t.text).includes('const late = true;')),
+    'lexer emits a code token with the fenced body',
+  );
+  const segments = renderTokenAnsiSegments(input);
+  const lines = codeBlockVisibleLines(segments);
+  assert.ok(lines.some((l) => l.includes('const late = true;')), 'code block body is rendered');
+  const visible = segments.map((s) => stripAnsi(s.ansi ?? '')).join('\n');
+  assert.ok(!visible.includes(`${PLAIN_PREFIX}\`\`\`js`), 'prefix is not glued to raw fence markers');
+});
+
+test('render path parses strong after a 500+ char plain prefix (no raw **)', () => {
+  const segments = renderTokenAnsiSegments(`${PLAIN_PREFIX}**rendered**`);
+  const visible = segments.map((s) => stripAnsi(s.ansi ?? '')).join('');
+  assert.ok(visible.includes('rendered'), 'bold text is present');
+  assert.ok(!visible.includes('**'), 'markdown strong markers are not leaked');
+});
+
+test('render path parses inline code after a 500+ char plain prefix (no raw backticks)', () => {
+  const segments = renderTokenAnsiSegments(`${PLAIN_PREFIX}\`snippet\``);
+  const visible = segments.map((s) => stripAnsi(s.ansi ?? '')).join('');
+  assert.ok(visible.includes('snippet'), 'codespan text is present');
+  assert.ok(!visible.includes('`'), 'inline code backticks are not leaked');
+});

package/src/tui/markdown/stream-fence.mjs

@@ -0,0 +1,34 @@
+/**
+ * markdown/stream-fence.mjs — streamed partial closing-fence trimming.
+ *
+ * While a fenced code block is still streaming in, marked may briefly emit the
+ * partial closing fence (a lone ` or ``) as a final code-block line, which makes
+ * the rendered block grow then shrink as the remaining backticks arrive — a
+ * visible flicker. `trimPartialClosingFences` strips that trailing partial fence
+ * from the LAST code token (recursing through trailing list/blockquote nesting)
+ * so the block height stays stable across deltas.
+ *
+ * Kept in its own dependency-free module so it can be unit-tested without the
+ * ink/JSX render stack. Ref: earendil-works/pi packages/tui/src/components/markdown.ts.
+ */
+export function trimPartialClosingFences(tokens) {
+  const token = tokens?.[tokens.length - 1];
+  if (!token) return;
+  if (token.type === 'list') {
+    const items = token.items ?? [];
+    trimPartialClosingFences(items[items.length - 1]?.tokens ?? []);
+    return;
+  }
+  if (token.type === 'blockquote') {
+    trimPartialClosingFences(token.tokens ?? []);
+    return;
+  }
+  if (token.type !== 'code') return;
+  const marker = /^(`{3,}|~{3,})/.exec(token.raw ?? '')?.[1];
+  const lastLine = String(token.raw ?? '').split('\n').pop();
+  if (!marker || !lastLine) return;
+  // Only trim a partial fence: shorter than the opening marker and made up
+  // solely of the same fence char. A complete closing fence is left intact.
+  if (lastLine.length >= marker.length || lastLine !== marker[0].repeat(lastLine.length)) return;
+  token.text = String(token.text ?? '').slice(0, -lastLine.length).replace(/\n$/, '');
+}

package/src/tui/markdown/stream-fence.test.mjs

@@ -0,0 +1,26 @@
+import test from 'node:test';
+import assert from 'node:assert/strict';
+import { marked } from 'marked';
+import { trimPartialClosingFences } from './stream-fence.mjs';
+
+test('trims a partial closing fence (single backtick) from open code block', () => {
+  const tokens = marked.lexer('```js\nconst x = 1;\n`');
+  trimPartialClosingFences(tokens);
+  const code = tokens.find((t) => t.type === 'code');
+  assert.ok(code, 'code token exists');
+  assert.ok(!code.text.endsWith('`'), 'partial fence trimmed from text');
+  assert.ok(code.text.includes('const x = 1;'), 'real body preserved');
+});
+
+test('leaves a complete closing fence intact', () => {
+  const tokens = marked.lexer('```js\nconst x = 1;\n```\n');
+  const code = tokens.find((t) => t.type === 'code');
+  const before = code.text;
+  trimPartialClosingFences(tokens);
+  assert.equal(code.text, before, 'closed block unchanged');
+});
+
+test('no-op on non-code trailing token', () => {
+  const tokens = marked.lexer('hello world\n');
+  assert.doesNotThrow(() => trimPartialClosingFences(tokens));
+});