@bendyline/squisq 0.1.1

package/src/markdown/htmlParse.ts

@@ -0,0 +1,212 @@
+/**
+ * HTML ↔ HtmlNode Conversion
+ *
+ * Parses raw HTML strings into the squisq HtmlNode tree representation
+ * using hast-util-from-html (parse5-backed), and serializes HtmlNode trees
+ * back to HTML strings with a lightweight custom serializer.
+ */
+
+import { fromHtml } from 'hast-util-from-html';
+import type { HtmlNode } from './types.js';
+
+// ============================================
+// Hast → HtmlNode conversion
+// ============================================
+
+/**
+ * Mapping from hast DOM-style property names to HTML attribute names.
+ * Only includes names where the property and attribute differ.
+ */
+const HAST_PROP_TO_ATTR: Record<string, string> = {
+  className: 'class',
+  htmlFor: 'for',
+  httpEquiv: 'http-equiv',
+  acceptCharset: 'accept-charset',
+};
+
+/** HTML void elements that are self-closing. */
+const VOID_ELEMENTS = new Set([
+  'area',
+  'base',
+  'br',
+  'col',
+  'embed',
+  'hr',
+  'img',
+  'input',
+  'link',
+  'meta',
+  'param',
+  'source',
+  'track',
+  'wbr',
+]);
+
+/**
+ * Convert a hast property name to an HTML attribute name.
+ */
+function propertyToAttribute(prop: string): string {
+  if (HAST_PROP_TO_ATTR[prop]) return HAST_PROP_TO_ATTR[prop];
+
+  // data-* attributes: dataFooBar → data-foo-bar
+  if (prop.length > 4 && prop.startsWith('data') && prop[4] >= 'A' && prop[4] <= 'Z') {
+    return (
+      'data-' +
+      prop
+        .slice(4)
+        .replace(/[A-Z]/g, (m) => '-' + m.toLowerCase())
+        .replace(/^-/, '')
+    );
+  }
+
+  // aria-* attributes: ariaLabel → aria-label
+  if (prop.length > 4 && prop.startsWith('aria') && prop[4] >= 'A' && prop[4] <= 'Z') {
+    return (
+      'aria-' +
+      prop
+        .slice(4)
+        .replace(/[A-Z]/g, (m) => '-' + m.toLowerCase())
+        .replace(/^-/, '')
+    );
+  }
+
+  // Most HTML attributes are lowercase (tabIndex → tabindex, readOnly → readonly, etc.)
+  return prop.toLowerCase();
+}
+
+/**
+ * Convert hast element properties to a flat HTML attribute map.
+ */
+function convertHastProperties(properties: Record<string, unknown>): Record<string, string> {
+  const attrs: Record<string, string> = {};
+  for (const [key, value] of Object.entries(properties)) {
+    if (value === false || value === undefined || value === null) continue;
+
+    const attrName = propertyToAttribute(key);
+
+    if (value === true) {
+      attrs[attrName] = '';
+    } else if (Array.isArray(value)) {
+      attrs[attrName] = value.join(' ');
+    } else {
+      attrs[attrName] = String(value);
+    }
+  }
+  return attrs;
+}
+
+/**
+ * Recursively convert hast child nodes to HtmlNode[].
+ */
+function hastChildrenToNodes(children: unknown[]): HtmlNode[] {
+  const result: HtmlNode[] = [];
+  for (const child of children) {
+    const node = hastNodeToHtmlNode(child as Record<string, unknown>);
+    if (node) result.push(node);
+  }
+  return result;
+}
+
+/**
+ * Convert a single hast node to an HtmlNode.
+ */
+function hastNodeToHtmlNode(node: Record<string, unknown>): HtmlNode | null {
+  switch (node.type) {
+    case 'element': {
+      const tagName = node.tagName as string;
+      const properties = (node.properties ?? {}) as Record<string, unknown>;
+      const children = (node.children ?? []) as unknown[];
+      return {
+        type: 'htmlElement',
+        tagName,
+        attributes: convertHastProperties(properties),
+        children: hastChildrenToNodes(children),
+        selfClosing: VOID_ELEMENTS.has(tagName) && children.length === 0,
+      };
+    }
+    case 'text':
+      return {
+        type: 'htmlText',
+        value: node.value as string,
+      };
+    case 'comment':
+      return {
+        type: 'htmlComment',
+        value: node.value as string,
+      };
+    case 'doctype':
+      // Doctype nodes are rare in markdown HTML; skip them
+      return null;
+    default:
+      return null;
+  }
+}
+
+// ============================================
+// Public API
+// ============================================
+
+/**
+ * Parse a raw HTML string into an HtmlNode[] tree.
+ *
+ * Uses the HTML5 parsing algorithm (via parse5) for correct handling of
+ * self-closing tags, optional end tags, entity decoding, etc.
+ *
+ * @param html - Raw HTML string to parse
+ * @returns Array of top-level HtmlNode elements
+ */
+export function parseHtmlToNodes(html: string): HtmlNode[] {
+  if (!html || !html.trim()) return [];
+
+  try {
+    const tree = fromHtml(html, { fragment: true });
+    return hastChildrenToNodes(tree.children as unknown[]);
+  } catch {
+    // If parsing fails, return the raw text as a single text node
+    return [{ type: 'htmlText', value: html }];
+  }
+}
+
+/**
+ * Escape special characters in HTML attribute values.
+ */
+function escapeHtmlAttr(value: string): string {
+  return value
+    .replace(/&/g, '&amp;')
+    .replace(/"/g, '&quot;')
+    .replace(/</g, '&lt;')
+    .replace(/>/g, '&gt;');
+}
+
+/**
+ * Serialize an HtmlNode back to an HTML string.
+ */
+function stringifyNode(node: HtmlNode): string {
+  switch (node.type) {
+    case 'htmlElement': {
+      const attrs = Object.entries(node.attributes)
+        .map(([k, v]) => (v === '' ? k : `${k}="${escapeHtmlAttr(v)}"`))
+        .join(' ');
+      const open = attrs ? `<${node.tagName} ${attrs}>` : `<${node.tagName}>`;
+
+      if (node.selfClosing) return open;
+
+      const inner = node.children.map(stringifyNode).join('');
+      return `${open}${inner}</${node.tagName}>`;
+    }
+    case 'htmlText':
+      return node.value;
+    case 'htmlComment':
+      return `<!--${node.value}-->`;
+  }
+}
+
+/**
+ * Serialize an HtmlNode[] tree back to an HTML string.
+ *
+ * @param nodes - Array of HtmlNode elements to serialize
+ * @returns HTML string
+ */
+export function stringifyHtmlNodes(nodes: HtmlNode[]): string {
+  return nodes.map(stringifyNode).join('');
+}

package/src/markdown/index.ts

@@ -0,0 +1,110 @@
+/**
+ * @bendyline/squisq Markdown Module
+ *
+ * Provides a complete JSON representation of markdown documents with
+ * perfect-fidelity round-trip conversion (markdown ↔ JSON).
+ *
+ * Supports CommonMark + GFM (tables, strikethrough, task lists, footnotes) +
+ * math ($...$, $$...$$) + directives (:::container, ::leaf, :text).
+ * Raw HTML is parsed into a structured HtmlNode sub-DOM.
+ *
+ * @example
+ * ```ts
+ * import { parseMarkdown, stringifyMarkdown } from '@bendyline/squisq/markdown';
+ *
+ * // Parse markdown to JSON
+ * const doc = parseMarkdown('# Hello\n\nWorld **bold** and *italic*');
+ *
+ * // Inspect the tree
+ * console.log(doc.children[0].type); // 'heading'
+ *
+ * // Serialize back to markdown
+ * const md = stringifyMarkdown(doc);
+ * ```
+ */
+
+// Types (all interfaces + union types)
+export type {
+  // Position tracking
+  MarkdownPoint,
+  MarkdownSourcePosition,
+
+  // Heading template annotation
+  HeadingTemplateAnnotation,
+
+  // HTML sub-DOM
+  HtmlElement,
+  HtmlText,
+  HtmlComment,
+  HtmlNode,
+
+  // Document root
+  MarkdownDocument,
+
+  // Block content
+  MarkdownHeading,
+  MarkdownParagraph,
+  MarkdownBlockquote,
+  MarkdownList,
+  MarkdownListItem,
+  MarkdownCodeBlock,
+  MarkdownThematicBreak,
+  MarkdownTable,
+  MarkdownTableRow,
+  MarkdownTableCell,
+  MarkdownHtmlBlock,
+  MarkdownMathBlock,
+  MarkdownLinkDefinition,
+  MarkdownFootnoteDefinition,
+  MarkdownContainerDirective,
+  MarkdownLeafDirective,
+  MarkdownDefinitionList,
+  MarkdownDefinitionTerm,
+  MarkdownDefinitionDescription,
+
+  // Inline content
+  MarkdownText,
+  MarkdownEmphasis,
+  MarkdownStrong,
+  MarkdownStrikethrough,
+  MarkdownInlineCode,
+  MarkdownLink,
+  MarkdownImage,
+  MarkdownBreak,
+  MarkdownInlineHtml,
+  MarkdownInlineMath,
+  MarkdownFootnoteReference,
+  MarkdownLinkReference,
+  MarkdownImageReference,
+  MarkdownTextDirective,
+
+  // Union types
+  MarkdownBlockNode,
+  MarkdownInlineNode,
+  MarkdownNode,
+
+  // Options
+  ParseOptions,
+  StringifyOptions,
+} from './types.js';
+
+// Parser + serializer
+export { parseMarkdown } from './parse.js';
+export { stringifyMarkdown } from './stringify.js';
+
+// Conversion layer (for advanced use: working with remark plugins directly)
+export { fromMdast, toMdast } from './convert.js';
+
+// HTML sub-DOM utilities
+export { parseHtmlToNodes, stringifyHtmlNodes } from './htmlParse.js';
+
+// Tree utilities
+export {
+  getChildren,
+  walkMarkdownTree,
+  findNodesByType,
+  extractPlainText,
+  countNodes,
+  createDocument,
+  parseFrontmatter,
+} from './utils.js';

package/src/markdown/parse.ts

@@ -0,0 +1,103 @@
+/**
+ * Markdown → JSON Parser
+ *
+ * Parses a markdown string into a MarkdownDocument JSON structure
+ * using the unified/remark ecosystem with GFM, math, and directive extensions.
+ */
+
+import { unified } from 'unified';
+import remarkParse from 'remark-parse';
+import remarkGfm from 'remark-gfm';
+import remarkMath from 'remark-math';
+import remarkDirective from 'remark-directive';
+import remarkFrontmatter from 'remark-frontmatter';
+import type { MarkdownDocument, ParseOptions } from './types.js';
+import { fromMdast } from './convert.js';
+import { parseFrontmatter } from './utils.js';
+
+// Cache the default processor (all extensions enabled) to avoid rebuilding on every call.
+let defaultProcessor: any;
+
+/**
+ * Parse a markdown string into a MarkdownDocument.
+ *
+ * All extensions (GFM, math, directives) are enabled by default.
+ * Use the `options` parameter to disable specific extensions.
+ *
+ * @param markdown - The markdown source text
+ * @param options - Parser options (extensions, HTML parsing)
+ * @returns A MarkdownDocument representing the parsed markdown
+ *
+ * @example
+ * ```ts
+ * const doc = parseMarkdown('# Hello\n\nWorld **bold** and *italic*');
+ * // doc.type === 'document'
+ * // doc.children[0].type === 'heading'
+ * // doc.children[1].type === 'paragraph'
+ * ```
+ */
+export function parseMarkdown(markdown: string, options?: ParseOptions): MarkdownDocument {
+  // Use cached default processor when all extensions are enabled (the common case).
+  const useDefaults =
+    !options ||
+    (options.gfm !== false &&
+      options.math !== false &&
+      options.directive !== false &&
+      options.frontmatter !== false);
+
+  let processor: any;
+
+  if (useDefaults) {
+    if (!defaultProcessor) {
+      defaultProcessor = unified()
+        .use(remarkParse)
+        .use(remarkGfm)
+        .use(remarkMath)
+        .use(remarkDirective)
+        .use(remarkFrontmatter, ['yaml']);
+    }
+    processor = defaultProcessor;
+  } else {
+    // Build a custom processor with requested extensions.
+    // unified's .use() chaining changes the generic signature each time,
+    // making strict typing impractical — use a widened Processor type.
+    processor = unified().use(remarkParse);
+
+    if (options?.gfm !== false) {
+      processor = processor.use(remarkGfm);
+    }
+    if (options?.math !== false) {
+      processor = processor.use(remarkMath);
+    }
+    if (options?.directive !== false) {
+      processor = processor.use(remarkDirective);
+    }
+    if (options?.frontmatter !== false) {
+      processor = processor.use(remarkFrontmatter, ['yaml']);
+    }
+  }
+
+  // Parse markdown → mdast tree (result is an mdast Root node)
+  const mdastTree = processor.parse(markdown) as {
+    type: string;
+    children?: Array<{ type: string; value?: string }>;
+  };
+
+  // Convert mdast → MarkdownDocument
+  const doc = fromMdast(mdastTree as Parameters<typeof fromMdast>[0], {
+    parseHtml: options?.parseHtml,
+  });
+
+  // Extract YAML frontmatter if present
+  if (options?.frontmatter !== false) {
+    const yamlNode = mdastTree.children?.find((n) => n.type === 'yaml');
+    if (yamlNode?.value) {
+      const fm = parseFrontmatter(yamlNode.value);
+      if (fm && Object.keys(fm).length > 0) {
+        doc.frontmatter = fm;
+      }
+    }
+  }
+
+  return doc;
+}

package/src/markdown/stringify.ts

@@ -0,0 +1,122 @@
+/**
+ * JSON → Markdown Serializer
+ *
+ * Converts a MarkdownDocument JSON structure back to a markdown string
+ * using the unified/remark ecosystem with GFM, math, and directive extensions.
+ */
+
+import { unified } from 'unified';
+import remarkStringify from 'remark-stringify';
+import remarkGfm from 'remark-gfm';
+import remarkMath from 'remark-math';
+import remarkDirective from 'remark-directive';
+import type { MarkdownDocument, StringifyOptions } from './types.js';
+import { toMdast } from './convert.js';
+
+// Cache the default processor (all extensions, default formatting) to avoid rebuilding on every call.
+let defaultProcessor: any;
+
+/**
+ * Serialize a MarkdownDocument back to a markdown string.
+ *
+ * All extensions (GFM, math, directives) are enabled by default.
+ * Use the `options` parameter to control formatting and disable extensions.
+ *
+ * @param doc - The MarkdownDocument to serialize
+ * @param options - Serialization options (formatting, extensions)
+ * @returns A markdown string
+ *
+ * @example
+ * ```ts
+ * const doc: MarkdownDocument = {
+ *   type: 'document',
+ *   children: [
+ *     { type: 'heading', depth: 1, children: [{ type: 'text', value: 'Hello' }] },
+ *     { type: 'paragraph', children: [{ type: 'text', value: 'World' }] },
+ *   ],
+ * };
+ * const md = stringifyMarkdown(doc);
+ * // '# Hello\n\nWorld\n'
+ * ```
+ */
+export function stringifyMarkdown(doc: MarkdownDocument, options?: StringifyOptions): string {
+  // Convert MarkdownDocument → mdast tree
+  const mdastTree = toMdast(doc);
+
+  // Use cached default processor when all extensions and default formatting are used.
+  const useDefaults =
+    !options ||
+    (options.gfm !== false &&
+      options.math !== false &&
+      options.directive !== false &&
+      !options.bullet &&
+      !options.bulletOrdered &&
+      !options.emphasis &&
+      !options.strong &&
+      !options.rule &&
+      !options.fence &&
+      options.setext == null);
+
+  let processor: any;
+
+  if (useDefaults) {
+    if (!defaultProcessor) {
+      defaultProcessor = unified()
+        .use(remarkGfm)
+        .use(remarkMath)
+        .use(remarkDirective)
+        .use(remarkStringify, {
+          bullet: '-',
+          bulletOrdered: '.',
+          emphasis: '*',
+          strong: '*',
+          rule: '-',
+          fence: '`',
+          setext: false,
+        });
+    }
+    processor = defaultProcessor;
+  } else {
+    // Build a custom processor with requested options.
+    // unified's .use() chaining changes the generic signature each time,
+    // making strict typing impractical — use a widened Processor type.
+    processor = unified();
+
+    if (options?.gfm !== false) {
+      processor = processor.use(remarkGfm);
+    }
+    if (options?.math !== false) {
+      processor = processor.use(remarkMath);
+    }
+    if (options?.directive !== false) {
+      processor = processor.use(remarkDirective);
+    }
+
+    processor = processor.use(remarkStringify, {
+      bullet: options?.bullet ?? '-',
+      bulletOrdered: options?.bulletOrdered ?? '.',
+      emphasis: options?.emphasis ?? '*',
+      strong: options?.strong ?? '*',
+      rule: options?.rule ?? '-',
+      fence: options?.fence ?? '`',
+      setext: options?.setext ?? false,
+    });
+  }
+
+  // Stringify mdast → markdown string
+  const result = processor.stringify(mdastTree) as string;
+
+  // remark-stringify escapes `[` in text nodes (to prevent link syntax),
+  // which turns `{[template]}` into `{\[template]}`. Unescape our annotation syntax.
+  const cleaned = result.replace(/\{\\\[([^\]]+)\]\}/g, '{[$1]}');
+
+  // Prepend YAML frontmatter if present
+  if (doc.frontmatter && Object.keys(doc.frontmatter).length > 0) {
+    const yamlLines = Object.entries(doc.frontmatter).map(
+      ([k, v]) => `${k}: ${typeof v === 'string' ? v : JSON.stringify(v)}`,
+    );
+    return `---\n${yamlLines.join('\n')}\n---\n\n${cleaned}`;
+  }
+
+  return cleaned;
+}