@gguf/coder 0.2.9 → 0.3.1

package/source/markdown-parser/index.spec.ts

@@ -0,0 +1,281 @@
+import test from 'ava';
+import type {Colors} from '../types/markdown-parser.js';
+import {parseMarkdown} from './index.js';
+
+console.log(`\nindex.spec.ts`);
+
+const mockColors: Colors = {
+	primary: '#3b82f6',
+	secondary: '#6b7280',
+	success: '#10b981',
+	error: '#ef4444',
+	warning: '#f59e0b',
+	info: '#3b82f6',
+	white: '#ffffff',
+	tool: '#8b5cf6',
+};
+
+// Inline code tests
+test('parseMarkdown handles inline code', t => {
+	const text = 'Use `npm install` to install packages';
+	const result = parseMarkdown(text, mockColors);
+	t.true(result.includes('npm install'));
+	t.false(result.includes('`npm install`'));
+});
+
+// Bold and italic tests
+test('parseMarkdown handles bold text', t => {
+	const text = 'This is **bold** text';
+	const result = parseMarkdown(text, mockColors);
+	t.true(result.includes('bold'));
+});
+
+test('parseMarkdown handles italic text', t => {
+	const text = 'This is *italic* text';
+	const result = parseMarkdown(text, mockColors);
+	t.true(result.includes('italic'));
+});
+
+// Underscore preservation tests
+test('parseMarkdown preserves underscores in identifiers', t => {
+	const text =
+		'Use `create_file`, `read_file`, or `search_file_contents` functions';
+	const result = parseMarkdown(text, mockColors);
+	t.true(result.includes('create_file'));
+	t.true(result.includes('read_file'));
+	t.true(result.includes('search_file_contents'));
+});
+
+test('parseMarkdown preserves underscores in regular text', t => {
+	const text = 'The variable_name and function_call should remain intact';
+	const result = parseMarkdown(text, mockColors);
+	t.true(result.includes('variable_name'));
+	t.true(result.includes('function_call'));
+});
+
+// Heading tests
+test('parseMarkdown handles headings', t => {
+	const text = '# Heading 1\n## Heading 2\n### Heading 3';
+	const result = parseMarkdown(text, mockColors);
+	t.true(result.includes('Heading 1'));
+	t.true(result.includes('Heading 2'));
+	t.true(result.includes('Heading 3'));
+});
+
+// Link tests
+test('parseMarkdown handles links', t => {
+	const text = 'Visit [Google](https://google.com) for search';
+	const result = parseMarkdown(text, mockColors);
+	t.true(result.includes('Google'));
+	t.true(result.includes('https://google.com'));
+});
+
+// Blockquote tests
+test('parseMarkdown handles blockquotes', t => {
+	const text = '> This is a quote\n> Another line';
+	const result = parseMarkdown(text, mockColors);
+	t.true(result.includes('This is a quote'));
+	t.true(result.includes('Another line'));
+});
+
+// List tests
+test('parseMarkdown handles unordered lists', t => {
+	const text = '- Item 1\n- Item 2\n- Item 3';
+	const result = parseMarkdown(text, mockColors);
+	t.true(result.includes('•'));
+	t.true(result.includes('Item 1'));
+	t.true(result.includes('Item 2'));
+});
+
+test('parseMarkdown handles ordered lists', t => {
+	const text = '1. First\n2. Second\n3. Third';
+	const result = parseMarkdown(text, mockColors);
+	t.true(result.includes('1.'));
+	t.true(result.includes('2.'));
+	t.true(result.includes('3.'));
+	t.true(result.includes('First'));
+	t.true(result.includes('Second'));
+});
+
+test('parseMarkdown handles nested/indented bullet lists', t => {
+	const text = `- Top level item
+  - Nested item 1
+  - Nested item 2
+    - Double nested
+- Another top level`;
+	const result = parseMarkdown(text, mockColors);
+	t.true(result.includes('  • Nested item 1'));
+	t.true(result.includes('    • Double nested'));
+});
+
+test('parseMarkdown handles nested numbered lists', t => {
+	const text = `1. First level
+  1. Nested first
+  2. Nested second
+2. Back to first level`;
+	const result = parseMarkdown(text, mockColors);
+	t.true(result.includes('  1. Nested first'));
+	t.true(result.includes('  2. Nested second'));
+});
+
+// HTML entity tests
+test('parseMarkdown decodes HTML entities', t => {
+	const text = '<div> & ©';
+	const result = parseMarkdown(text, mockColors);
+	t.true(result.includes('<div>'));
+	t.true(result.includes('&'));
+	t.true(result.includes('©'));
+});
+
+// Table tests
+test('parseMarkdown handles tables', t => {
+	const text = `| Name | Age |
+|------|-----|
+| John | 25  |`;
+	const result = parseMarkdown(text, mockColors);
+	t.true(result.includes('Name'));
+	t.true(result.includes('John'));
+});
+
+test('parseMarkdown renders tables with plain text (no markdown)', t => {
+	const text = `| Command | Description |
+|---------|-------------|
+| \`npm install\` | Install dependencies |
+| \`npm start\` | Start the application |`;
+	const result = parseMarkdown(text, mockColors);
+	t.true(result.includes('npm install'));
+	t.true(result.includes('npm start'));
+	t.false(result.includes('`npm install`'));
+});
+
+// Code block tests
+test('parseMarkdown handles code blocks with language', t => {
+	const text = '```javascript\nconst x = 5;\n```';
+	const result = parseMarkdown(text, mockColors);
+	t.true(result.includes('const x = 5'));
+});
+
+test('parseMarkdown handles code blocks without language', t => {
+	const text = '```\nPlain code\n```';
+	const result = parseMarkdown(text, mockColors);
+	t.true(result.includes('Plain code'));
+});
+
+// Edge case tests
+test('parseMarkdown does not create bullet list from hyphen in middle of line', t => {
+	const text = 'This is not - a list';
+	const result = parseMarkdown(text, mockColors);
+	t.false(result.includes('•'));
+	t.true(result.includes('This is not - a list'));
+});
+
+test('parseMarkdown does not format asterisks in math expressions', t => {
+	const text = 'Calculate 5 * 3 * 2 = 30';
+	const result = parseMarkdown(text, mockColors);
+	t.true(result.includes('5 * 3 * 2'));
+});
+
+test('parseMarkdown does not format asterisks in file globs', t => {
+	const text = 'Match files with *.js or *.tsx patterns';
+	const result = parseMarkdown(text, mockColors);
+	t.true(result.includes('*.js'));
+	t.true(result.includes('*.tsx'));
+});
+
+test('parseMarkdown does not create heading from hash in middle of line', t => {
+	const text = 'This # is not a heading';
+	const result = parseMarkdown(text, mockColors);
+	t.true(result.includes('This # is not a heading'));
+});
+
+test('parseMarkdown preserves asterisks inside code blocks', t => {
+	const text = '```javascript\nconst pattern = /\\*\\*/g;\n```';
+	const result = parseMarkdown(text, mockColors);
+	t.true(result.includes('/\\*\\*/g') || result.includes('**'));
+});
+
+test('parseMarkdown preserves markdown-like syntax inside inline code', t => {
+	const text = 'Use `**bold**` and `*italic*` syntax';
+	const result = parseMarkdown(text, mockColors);
+	// Code content should be preserved (though formatting may be applied by chalk)
+	t.false(result.includes('`**bold**`'));
+	t.false(result.includes('`*italic*`'));
+});
+
+test('parseMarkdown does not format single asterisk surrounded by word chars', t => {
+	const text = 'The pointer syntax is char*ptr or int*value';
+	const result = parseMarkdown(text, mockColors);
+	t.true(result.includes('char*ptr'));
+	t.true(result.includes('int*value'));
+});
+
+// BR tag test
+test('parseMarkdown converts <br> tags to newlines', t => {
+	const text = 'Line one<br>Line two<br/>Line three<BR>Line four';
+	const result = parseMarkdown(text, mockColors);
+	const lines = result.split('\n');
+	t.true(lines.length >= 4);
+	t.true(result.includes('Line one'));
+	t.true(result.includes('Line two'));
+	t.false(result.includes('<br'));
+});
+
+// Spacing tests
+test('parseMarkdown preserves spacing before bullet lists', t => {
+	const text = `I can assist with tasks such as:
+
+- First item
+- Second item
+
+Let me know what you'd like to work on.`;
+	const result = parseMarkdown(text, mockColors);
+	const lines = result.split('\n');
+	const suchAsIndex = lines.findIndex(l => l.includes('I can assist'));
+	const firstBulletIndex = lines.findIndex(l => l.includes('• First'));
+	t.true(
+		firstBulletIndex - suchAsIndex >= 2,
+		'Should have blank line before list',
+	);
+});
+
+// Mixed content tests
+test('parseMarkdown handles mixed markdown features', t => {
+	const text = `# Title
+
+This is **bold** and *italic* text with \`code\`.
+
+- List item 1
+- List item 2
+
+| Name | Value |
+|------|-------|
+| Test | 123   |`;
+	const result = parseMarkdown(text, mockColors);
+	t.true(result.includes('Title'));
+	t.true(result.includes('bold'));
+	t.true(result.includes('italic'));
+	t.true(result.includes('•'));
+	t.true(result.includes('Name'));
+});
+
+test('parseMarkdown handles plain text without markdown', t => {
+	const text = 'This is just plain text without any formatting';
+	const result = parseMarkdown(text, mockColors);
+	t.true(result.includes('This is just plain text'));
+});
+
+test('parseMarkdown handles empty string', t => {
+	const result = parseMarkdown('', mockColors);
+	t.is(result, '');
+});
+
+// Placeholder restoration test
+test('parseMarkdown restores inline code placeholders correctly', t => {
+	const text = 'Use `npm install` and `npm start` commands';
+	const result = parseMarkdown(text, mockColors);
+	t.true(result.includes('npm install'));
+	t.true(result.includes('npm start'));
+	t.false(result.includes('__INLINE_CODE'));
+	t.false(result.includes('_INLINE'));
+	t.false(result.includes('CODE_'));
+});

package/source/markdown-parser/index.ts

@@ -0,0 +1,126 @@
+import chalk from 'chalk';
+import {highlight} from 'cli-highlight';
+import type {Colors} from '../types/markdown-parser.js';
+import {decodeHtmlEntities} from './html-entities.js';
+import {parseMarkdownTable} from './table-parser.js';
+
+// Basic markdown parser for terminal
+export function parseMarkdown(text: string, themeColors: Colors): string {
+	// First decode HTML entities
+	let result = decodeHtmlEntities(text);
+
+	// Step 1: Parse tables FIRST (before <br> conversion and code extraction)
+	// Tables should have plain text only, no markdown formatting
+	// Note: We parse tables before converting <br> tags so that multi-line
+	// cells don't break the table regex
+	result = result.replace(
+		/(?:^|\n)((?:\|.+\|\n)+)/gm,
+		(_match, tableText: string) => {
+			return '\n' + parseMarkdownTable(tableText, themeColors) + '\n';
+		},
+	);
+
+	// Step 2: Convert <br> and <br/> tags to newlines (AFTER table parsing)
+	result = result.replace(/<br\s*\/?>/gi, '\n');
+
+	// Step 3: Extract and protect code blocks and inline code with placeholders
+	const codeBlocks: string[] = [];
+	const inlineCodes: string[] = [];
+
+	// Extract code blocks first (```language\ncode\n```)
+	result = result.replace(
+		/```([a-zA-Z0-9\-+#]+)?\n([\s\S]*?)```/g,
+		(_match, lang: string | undefined, code: string) => {
+			try {
+				const codeStr = String(code).trim();
+				// Apply syntax highlighting with detected language
+				const highlighted = highlight(codeStr, {
+					language: lang || 'plaintext',
+					theme: 'default',
+				});
+				const placeholder = `__CODE_BLOCK_${codeBlocks.length}__`;
+				codeBlocks.push(highlighted);
+				return placeholder;
+			} catch {
+				// Fallback to plain colored text if highlighting fails
+				const formatted = chalk.hex(themeColors.tool)(String(code).trim());
+				const placeholder = `__CODE_BLOCK_${codeBlocks.length}__`;
+				codeBlocks.push(formatted);
+				return placeholder;
+			}
+		},
+	);
+
+	// Extract inline code (`code`)
+	result = result.replace(/`([^`]+)`/g, (_match, code: string) => {
+		const formatted = chalk.hex(themeColors.tool)(String(code).trim());
+		const placeholder = `__INLINE_CODE_${inlineCodes.length}__`;
+		inlineCodes.push(formatted);
+		return placeholder;
+	});
+
+	// Step 4: Process markdown formatting (now safe from code interference)
+	// Process lists FIRST before italic, since * at start of line is a list, not italic
+	// List items (- item or * item or 1. item)
+	// Use [ \t]* instead of \s* to avoid consuming newlines before the list
+	// Preserve indentation for nested lists
+	result = result.replace(/^([ \t]*)[-*]\s+(.+)$/gm, (_match, indent, text) => {
+		return indent + chalk.hex(themeColors.white)(`• ${text}`);
+	});
+	result = result.replace(
+		/^([ \t]*)(\d+)\.\s+(.+)$/gm,
+		(_match, indent, num, text) => {
+			return indent + chalk.hex(themeColors.white)(`${num}. ${text}`);
+		},
+	);
+
+	// Bold (**text** only - avoid __ to prevent conflicts with snake_case)
+	result = result.replace(/\*\*([^*]+)\*\*/g, (_match, text) => {
+		return chalk.hex(themeColors.white).bold(text);
+	});
+
+	// Italic (*text* only - avoid _ to prevent conflicts with snake_case)
+	// Require whitespace or line boundaries around asterisks to avoid matching char*ptr
+	// Use [^*\n] to prevent matching across lines
+	// Only match if content contains at least one letter to avoid matching math like "5 * 3 * 2"
+	result = result.replace(
+		/(^|\s)\*([^*\n]*[a-zA-Z][^*\n]*)\*($|\s)/gm,
+		(_match, before, text, after) => {
+			return before + chalk.hex(themeColors.white).italic(text) + after;
+		},
+	);
+
+	// Headings (# Heading)
+	result = result.replace(/^(#{1,6})\s+(.+)$/gm, (_match, _hashes, text) => {
+		return chalk.hex(themeColors.primary).bold(text);
+	});
+
+	// Links [text](url)
+	result = result.replace(/\[([^\]]+)\]\(([^)]+)\)/g, (_match, text, url) => {
+		return (
+			chalk.hex(themeColors.info).underline(text) +
+			' ' +
+			chalk.hex(themeColors.secondary)(`(${url})`)
+		);
+	});
+
+	// Blockquotes (> text)
+	result = result.replace(/^>\s+(.+)$/gm, (_match, text) => {
+		return chalk.hex(themeColors.secondary).italic(`> ${text}`);
+	});
+
+	// Step 5: Restore code blocks and inline code from placeholders
+	result = result.replace(/__CODE_BLOCK_(\d+)__/g, (_match, index: string) => {
+		return codeBlocks[parseInt(index, 10)] || '';
+	});
+	result = result.replace(/__INLINE_CODE_(\d+)__/g, (_match, index: string) => {
+		return inlineCodes[parseInt(index, 10)] || '';
+	});
+
+	return result;
+}
+
+// Re-export utilities for convenience
+export {decodeHtmlEntities} from './html-entities.js';
+export {parseMarkdownTable} from './table-parser.js';
+export type {Colors} from '../types/markdown-parser.js';

package/source/markdown-parser/table-parser.spec.ts

@@ -0,0 +1,133 @@
+import test from 'ava';
+import type {Colors} from '../types/markdown-parser';
+import {parseMarkdownTable} from './table-parser';
+
+console.log(`\ntable-parser.spec.ts`);
+
+const mockColors: Colors = {
+	primary: '#3b82f6',
+	secondary: '#6b7280',
+	success: '#10b981',
+	error: '#ef4444',
+	warning: '#f59e0b',
+	info: '#3b82f6',
+	white: '#ffffff',
+	tool: '#8b5cf6',
+};
+
+test('parseMarkdownTable handles simple two-column table', t => {
+	const table = `| Name | Age |
+|------|-----|
+| John | 25  |
+| Jane | 30  |`;
+	const result = parseMarkdownTable(table, mockColors);
+	t.true(result.includes('Name'));
+	t.true(result.includes('Age'));
+	t.true(result.includes('John'));
+	t.true(result.includes('Jane'));
+	t.true(result.includes('25'));
+	t.true(result.includes('30'));
+});
+
+test('parseMarkdownTable handles table with varying cell lengths', t => {
+	const table = `| Short | Very Long Column Name |
+|-------|----------------------|
+| A     | This is a much longer text |
+| B     | Short |`;
+	const result = parseMarkdownTable(table, mockColors);
+	t.true(result.includes('Short'));
+	t.true(result.includes('Very Long Column Name'));
+	t.true(result.includes('This is a much longer text'));
+});
+
+test('parseMarkdownTable returns original text for invalid table', t => {
+	const table = `Not a table at all`;
+	const result = parseMarkdownTable(table, mockColors);
+	t.is(result, table);
+});
+
+test('parseMarkdownTable returns original text for table without data rows', t => {
+	const table = `| Name | Age |
+|------|-----|`;
+	const result = parseMarkdownTable(table, mockColors);
+	t.is(result, table);
+});
+
+test('parseMarkdownTable handles table with empty cells', t => {
+	const table = `| Name | Age |
+|------|-----|
+| John |     |
+|      | 30  |`;
+	const result = parseMarkdownTable(table, mockColors);
+	t.true(result.includes('John'));
+	t.true(result.includes('30'));
+});
+
+test('parseMarkdownTable normalizes column count', t => {
+	const table = `| A | B | C |
+|---|---|---|
+| 1 | 2 |
+| 3 | 4 | 5 | 6 |`;
+	const result = parseMarkdownTable(table, mockColors);
+	// Should handle mismatched columns gracefully
+	t.true(typeof result === 'string');
+	t.true(result.length > 0);
+});
+
+test('parseMarkdownTable strips markdown from cells', t => {
+	const table = `| Command | Description |
+|---------|-------------|
+| \`npm install\` | Install **dependencies** |
+| \`npm start\` | Start the *application* |`;
+	const result = parseMarkdownTable(table, mockColors);
+	// Should have plain text (backticks and formatting removed)
+	t.true(result.includes('npm install'));
+	t.true(result.includes('npm start'));
+	t.true(result.includes('dependencies'));
+	t.true(result.includes('application'));
+	// Should NOT have markdown syntax
+	t.false(result.includes('`npm install`'));
+	t.false(result.includes('**dependencies**'));
+	t.false(result.includes('*application*'));
+});
+
+test('parseMarkdownTable strips HTML from cells', t => {
+	const table = `| Name | Description |
+|------|-------------|
+| Test | Text with <br> tag |
+| Another | Text with <b>bold</b> |`;
+	const result = parseMarkdownTable(table, mockColors);
+	t.true(result.includes('Text with  tag')); // HTML tag removed
+	t.true(result.includes('Text with bold')); // HTML tags removed
+	t.false(result.includes('<br>'));
+	t.false(result.includes('<b>'));
+});
+
+test('parseMarkdownTable handles links in cells', t => {
+	const table = `| Name | Link |
+|------|------|
+| Google | [Visit](https://google.com) |`;
+	const result = parseMarkdownTable(table, mockColors);
+	t.true(result.includes('Visit'));
+	t.false(result.includes('[Visit]'));
+	t.false(result.includes('(https://google.com)'));
+});
+
+test('parseMarkdownTable handles alignment markers', t => {
+	const table = `| Left | Center | Right |
+|:-----|:------:|------:|
+| A    | B      | C     |`;
+	const result = parseMarkdownTable(table, mockColors);
+	t.true(result.includes('Left'));
+	t.true(result.includes('Center'));
+	t.true(result.includes('Right'));
+});
+
+test('parseMarkdownTable creates table with borders', t => {
+	const table = `| A | B |
+|---|---|
+| 1 | 2 |`;
+	const result = parseMarkdownTable(table, mockColors);
+	// Should contain table border characters
+	t.true(result.includes('─') || result.includes('│'));
+});

package/source/markdown-parser/table-parser.ts

@@ -0,0 +1,114 @@
+import {DEFAULT_TERMINAL_WIDTH, TABLE_COLUMN_MIN_WIDTH} from '@/constants';
+import chalk from 'chalk';
+import Table from 'cli-table3';
+import type {Colors} from '../types/markdown-parser';
+import {stripMarkdown} from './utils';
+
+// Parse markdown tables using cli-table3
+export function parseMarkdownTable(
+	tableText: string,
+	themeColors: Colors,
+): string {
+	const lines = tableText.trim().split('\n');
+	if (lines.length < 2) return tableText;
+
+	// Extract rows
+	const rows = lines.map(line =>
+		line
+			.split('|')
+			.map(cell => cell.trim())
+			.filter(cell => cell.length > 0),
+	);
+
+	// Check if second row is separator (e.g., |---|---|)
+	const separatorRow = rows[1];
+	const isSeparator = separatorRow?.every(cell => /^:?-+:?$/.test(cell));
+	if (!isSeparator || rows.length < 3) return tableText;
+
+	// Helper to clean cell content - remove all markdown and HTML
+	const cleanCell = (text: string): string => {
+		let result = text;
+		// Remove inline code backticks (extract the content)
+		result = result.replace(/`([^`]+)`/g, '$1');
+		// Remove HTML tags
+		result = result.replace(/<[^>]+>/g, '');
+		// Strip markdown formatting (bold, italic, links)
+		result = stripMarkdown(result);
+		return result.trim();
+	};
+
+	// Extract header and data rows - clean all content
+	const header = rows[0].map(cell => cleanCell(cell));
+	const dataRows = rows.slice(2).map(row => row.map(cell => cleanCell(cell)));
+
+	// Calculate widths from cleaned content
+	const headerWidths = header.map(cell => cell.length);
+	const dataWidths = dataRows.map(row => row.map(cell => cell.length));
+
+	// Calculate column widths properly
+	const terminalWidth = process.stdout.columns || DEFAULT_TERMINAL_WIDTH;
+	const numCols = header.length;
+
+	// Get max content width for each column
+	const contentWidths = headerWidths.map((headerWidth, colIdx) => {
+		let maxWidth = headerWidth;
+		for (const rowWidths of dataWidths) {
+			if (rowWidths[colIdx]) {
+				maxWidth = Math.max(maxWidth, rowWidths[colIdx]);
+			}
+		}
+		return maxWidth;
+	});
+
+	// Calculate available width (accounting for borders and padding)
+	const borderWidth = numCols + 1; // vertical bars
+	const paddingWidth = numCols * 2; // 1 space on each side of each column
+	const availableWidth = terminalWidth - borderWidth - paddingWidth;
+
+	// Distribute width proportionally
+	const totalContentWidth = contentWidths.reduce((a, b) => a + b, 0);
+	const colWidths = contentWidths.map(width =>
+		Math.max(
+			TABLE_COLUMN_MIN_WIDTH,
+			Math.floor((width / totalContentWidth) * availableWidth),
+		),
+	);
+
+	// Create table with cli-table3 - full borders, proper alignment
+	const table = new Table({
+		head: header.map(cell => chalk.hex(themeColors.primary).bold(cell)),
+		colWidths: colWidths,
+		style: {
+			head: [], // Don't apply default styles, we're using chalk
+			border: ['gray'], // Subtle border color
+			'padding-left': 1,
+			'padding-right': 1,
+		},
+		chars: {
+			top: '─',
+			'top-mid': '┬',
+			'top-left': '┌',
+			'top-right': '┐',
+			bottom: '─',
+			'bottom-mid': '┴',
+			'bottom-left': '└',
+			'bottom-right': '┘',
+			left: '│',
+			'left-mid': '├',
+			mid: '─',
+			'mid-mid': '┼',
+			right: '│',
+			'right-mid': '┤',
+			middle: '│',
+		},
+		wordWrap: true,
+		wrapOnWordBoundary: true,
+	});
+
+	// Add data rows - don't style them, let cli-table3 handle layout
+	for (const row of dataRows) {
+		table.push(row);
+	}
+
+	return table.toString();
+}