@fileverse/content-processor 0.0.1

package/src/markdown-parser.ts

@@ -0,0 +1,148 @@
+import MarkdownIt from 'markdown-it';
+import markdownItFootnote from 'markdown-it-footnote';
+import { createDOMPurifyInstance, parseHTMLString } from './dom-setup.js';
+
+const markdownIt = new MarkdownIt().use(markdownItFootnote);
+
+export function stripFrontmatter(markdown: string): string {
+  const fmRegex = /^---\n[\s\S]*?\n---\n*/;
+  return markdown.replace(fmRegex, '').replace(/^\s*\n/, '');
+}
+
+export function parseMarkdownToHTML(markdown: string): string {
+  let cleanMarkdown = stripFrontmatter(markdown);
+
+  cleanMarkdown = cleanMarkdown.replace(/(\d)\*(\d)/g, '$1\\*$2');
+  cleanMarkdown = cleanMarkdown.replace(/(\])\*(\[)/g, '$1\\*$2');
+  cleanMarkdown = cleanMarkdown.replace(/(\))\*(\()/g, '$1\\*$2');
+
+  let convertedHtml = markdownIt.render(cleanMarkdown);
+
+  return convertedHtml;
+}
+
+export function processHTMLForEditor(html: string): string {
+  const doc = parseHTMLString(html);
+  const body = doc.body;
+
+  const topLevelPs = body.querySelectorAll(':scope > p');
+  topLevelPs.forEach((p) => {
+    if (p.childNodes.length === 0 && p.textContent === '') {
+      p.remove();
+    }
+  });
+
+  const calloutAsides = body.querySelectorAll('aside.callout');
+  calloutAsides.forEach((el) => {
+    el.setAttribute('data-type', 'callout');
+    el.removeAttribute('class');
+  });
+
+  const callouts = body.querySelectorAll('aside[data-type="callout"]');
+  callouts.forEach((aside) => {
+    const ps = aside.querySelectorAll('p');
+    ps.forEach((p) => {
+      const isEmpty = Array.from(p.childNodes).every((node) => {
+        if (node.nodeType === 1) {
+          return (node as Element).nodeName === 'BR';
+        }
+        if (node.nodeType === 3) {
+          return node.textContent?.trim() === '';
+        }
+        return false;
+      });
+      if (isEmpty && p.parentNode) {
+        p.parentNode.removeChild(p);
+      }
+    });
+  });
+
+  const lists = body.getElementsByTagName('ul');
+  for (let i = 0; i < lists.length; i++) {
+    const list = lists[i];
+    const items = list.getElementsByTagName('li');
+    let isTodoList = false;
+
+    for (let j = 0; j < items.length; j++) {
+      const item = items[j];
+      const text = item.textContent || '';
+      const todoMatch = text.match(/^\[([ x])\]\s*(.*)/i);
+
+      if (todoMatch) {
+        isTodoList = true;
+        const isChecked = todoMatch[1].toLowerCase() === 'x';
+        const content = todoMatch[2];
+
+        item.setAttribute('data-type', 'taskItem');
+        item.setAttribute('data-checked', isChecked.toString());
+        item.textContent = content;
+      }
+    }
+
+    if (isTodoList) {
+      list.setAttribute('data-type', 'taskList');
+    }
+  }
+
+  const paragraphs = body.getElementsByTagName('p');
+  for (let i = paragraphs.length - 1; i >= 0; i--) {
+    const p = paragraphs[i];
+    if (p.childNodes.length === 1 && p.firstChild?.nodeName === 'IMG') {
+      p.parentNode?.replaceChild(p.firstChild, p);
+    }
+    if (
+      p.childNodes.length === 1 &&
+      p.firstChild?.textContent?.trim() === '==='
+    ) {
+      const pageBreakDiv = doc.createElement('div');
+      pageBreakDiv.setAttribute('data-type', 'page-break');
+      pageBreakDiv.setAttribute('data-page-break', 'true');
+      p.parentNode?.replaceChild(pageBreakDiv, p);
+    }
+  }
+
+  let processedHtml = body.innerHTML;
+
+  const superscriptRegex = /\^([^\s^]+)\^/g;
+  const subscriptRegex = /~([^\s~](?:[^~]*[^\s~])?)~/g;
+  const pageBreakRegex = /===\s*$/gm;
+
+  processedHtml = processedHtml.replace(
+    superscriptRegex,
+    '<sup data-type="sup">$1</sup>'
+  );
+
+  processedHtml = processedHtml.replace(
+    subscriptRegex,
+    '<sub data-type="sub">$1</sub>'
+  );
+
+  processedHtml = processedHtml.replace(
+    pageBreakRegex,
+    '<div data-type="page-break" data-page-break="true"></div>'
+  );
+
+  const DOMPurify = createDOMPurifyInstance();
+  processedHtml = DOMPurify.sanitize(processedHtml, {
+    ADD_TAGS: ['div', 'aside'],
+    ADD_ATTR: [
+      'data-type',
+      'data-page-break',
+      'data-checked',
+      'url',
+      'src',
+      'media-type',
+      'alt',
+      'title',
+      'width',
+      'height',
+    ],
+  });
+
+  return processedHtml;
+}
+
+export function convertMarkdownToHTML(markdown: string): string {
+  const rawHtml = parseMarkdownToHTML(markdown);
+  return processHTMLForEditor(rawHtml);
+}

package/test/sample.md

@@ -0,0 +1,80 @@
+---
+title: Test Document
+date: 2024-01-01
+---
+
+# Heading 1
+
+This is a paragraph with **bold** and *italic* text.
+
+## Heading 2
+
+Here's a list:
+
+- Item 1
+- Item 2
+- Item 3
+
+### Ordered List
+
+1. First item
+2. Second item
+3. Third item
+
+### Task List
+
+- [ ] Unchecked task
+- [x] Checked task
+- [ ] Another task
+
+## Code
+
+Inline `code` example.
+
+```javascript
+function hello() {
+  console.log("Hello, World!");
+}
+```
+
+## Links and Images
+
+[Link text](https://example.com)
+
+![Alt text](https://example.com/image.png)
+
+## Table
+
+| Header 1 | Header 2 |
+|----------|----------|
+| Cell 1   | Cell 2   |
+| Cell 3   | Cell 4   |
+
+## Blockquote
+
+> This is a blockquote.
+> It can span multiple lines.
+
+## Page Break
+
+===
+
+## More Content
+
+Content after page break.
+
+<aside class="callout">
+This is a callout block.
+</aside>
+
+## Superscript and Subscript
+
+E = mc^2^
+
+H~2~O
+
+## Footnote
+
+This has a footnote[^1].
+
+[^1]: This is the footnote content.

package/test/test.ts

@@ -0,0 +1,30 @@
+import fs from 'fs';
+import path from 'path';
+import { fileURLToPath } from 'url';
+import { markdownToYjs, isYjsEncoded } from '../dist/index.js';
+
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = path.dirname(__filename);
+
+const sampleMarkdown = fs.readFileSync(
+  path.join(__dirname, 'sample.md'),
+  'utf-8'
+);
+
+console.log('Testing markdown-to-yjs converter...\n');
+console.log('Input markdown length:', sampleMarkdown.length, 'characters');
+console.log('---\n');
+
+try {
+  const yjsContent = markdownToYjs("Hello, World!, How are you?");
+
+  console.log('Output Yjs content length:', yjsContent.length, 'characters');
+  console.log('Is valid Yjs encoded:', isYjsEncoded(yjsContent));
+  console.log('\nFirst 100 characters of Yjs content:');
+  console.log(yjsContent);
+  console.log('\n---');
+  console.log('Conversion successful!');
+} catch (error) {
+  console.error('Conversion failed:', error);
+  process.exit(1);
+}

package/tsconfig.json

@@ -0,0 +1,20 @@
+{
+  "compilerOptions": {
+    "target": "ES2020",
+    "module": "ESNext",
+    "moduleResolution": "bundler",
+    "lib": ["ES2020", "DOM"],
+    "declaration": true,
+    "declarationMap": true,
+    "sourceMap": true,
+    "outDir": "./dist",
+    "rootDir": "./src",
+    "strict": true,
+    "esModuleInterop": true,
+    "skipLibCheck": true,
+    "forceConsistentCasingInFileNames": true,
+    "resolveJsonModule": true
+  },
+  "include": ["src/**/*"],
+  "exclude": ["node_modules", "dist"]
+}