@xyd-js/content 0.1.0-xyd.13 → 0.1.0-xyd.16

package/packages/md/plugins/utils/mdParameters.test.ts

@@ -0,0 +1,114 @@
+import { describe, it, expect } from 'vitest';
+import { mdParameters } from './mdParameters';
+
+describe('mdParameters', () => {
+    describe("attributes", () => {
+        it('1.basic', () => {
+            const result = mdParameters('Hello [class="test" id="main"] world');
+
+            expect(result.attributes).toEqual({
+                class: 'test',
+                id: 'main'
+            });
+            expect(result.props).toEqual({});
+            expect(result.sanitizedText).toBe('Hello world');
+        });
+
+        it('2.markdown+links', () => {
+            const result = mdParameters('[desc="This is [xyd](https://github.com/xyd-js) - a docs framework"]');
+
+            expect(result.attributes).toEqual({
+                desc: 'This is [xyd](https://github.com/xyd-js) - a docs framework',
+            });
+            expect(result.props).toEqual({});
+            expect(result.sanitizedText).toBe('');
+        });
+    })
+
+    describe("props", () => {
+        it('1.basic', () => {
+            const result = mdParameters('Hello {title="tooltip" disabled} world');
+
+            expect(result.attributes).toEqual({});
+            expect(result.props).toEqual({
+                title: 'tooltip',
+                disabled: 'true'
+            });
+            expect(result.sanitizedText).toBe('Hello world');
+        });
+    })
+
+    describe("attributes + props", () => {
+        it('1.basic', () => {
+            const result = mdParameters('Hello [class="test"] {title="tooltip"} world');
+
+            expect(result.attributes).toEqual({
+                class: 'test'
+            });
+            expect(result.props).toEqual({
+                title: 'tooltip'
+            });
+            expect(result.sanitizedText).toBe('Hello world');
+        });
+    })
+
+    describe("attributes + htmlMd option", () => {
+        it('1.basic', () => {
+            const result = mdParameters(
+                `[desc="Produces static files within <code>.xyd/build/client</code> which you an deploy easily."]`,
+                {
+                    htmlMd: true
+                }
+            );
+            expect(result.attributes).toEqual({
+                desc: "Produces static files within `.xyd/build/client` which you an deploy easily."
+            });
+
+            expect(result.sanitizedText).toBe('');
+        });
+
+        it('2.multiple HTML tags in attribute', () => {
+            const result = mdParameters(
+                `[desc="This is <strong>bold</strong> and <em>italic</em> text with <code>inline code</code>."]`,
+                {
+                    htmlMd: true
+                }
+            );
+            expect(result.attributes).toEqual({
+                desc: "This is **bold** and *italic* text with `inline code`."
+            });
+        });
+    })
+
+    describe("attributes + :::component", () => {
+        it('1.basic', () => {
+            const result = mdParameters(`[desc="Choose between light/dark mode. :::button \n **xyd 0.1.0-alpha is coming soon!** \n :::"]`);
+            const expectedText = `Choose between light/dark mode.\n :::button \n **xyd 0.1.0-alpha is coming soon!**\n :::`;
+
+            expect(result.attributes).toEqual({
+                desc: expectedText
+            });
+            expect(result.sanitizedText).toBe('');
+        })
+
+        it("2", () => {
+            const result = mdParameters(`[desc=":::button\\n **xyd 0.1.0-alpha is coming soon!**\\n"]`);
+            const expectedText = `:::button\n **xyd 0.1.0-alpha is coming soon!**\n:::`;
+
+            expect(result.attributes).toEqual({
+                desc: expectedText
+            });
+            expect(result.sanitizedText).toBe('');
+        })
+
+        it("3", () => {
+            const result = mdParameters(`[desc="Choose between :::color-scheme-button\\n::: mode"]`);
+            const expectedText = `Choose between\n :::color-scheme-button\n :::\n  mode`;
+
+            expect(result.attributes).toEqual({
+                desc: expectedText
+            });
+            expect(result.sanitizedText).toBe('');
+        })
+    })
+});

package/packages/md/plugins/utils/mdParameters.ts

@@ -0,0 +1,249 @@
+export function mdParameters(
+    text: string,
+    options = {
+        htmlMd: false,
+    }
+) {
+    const attributes = parseAttributes(text, options.htmlMd);
+    const props = parseProps(text, options.htmlMd);
+    let sanitizedText = removeParameters(text);
+    
+    // Apply HTML to markdown transformation if requested
+    if (options?.htmlMd) {
+        sanitizedText = transformHtmlToMarkdown(sanitizedText);
+    }
+    
+    return {
+        attributes,
+        props,
+        sanitizedText
+    }
+}
+
+function parseAttributes(text: string, htmlMd: boolean = false) {
+    return parseParameters(text, '[', ']', htmlMd);
+}
+
+function parseProps(text: string, htmlMd: boolean = false) {
+    return parseParameters(text, '{', '}', htmlMd);
+}
+
+function sanitizeParameters(value: string): string {
+    // Convert escaped newlines to actual newlines
+    value = value.replace(/\\n/g, '\n');
+
+    // Remove any HTML tags
+    value = value.replace(/<[^>]*>/g, '');
+
+    // Remove any script tags and their content
+    value = value.replace(/<script\b[^<]*(?:(?!<\/script>)<[^<]*)*<\/script>/gi, '');
+
+    // Remove any potentially dangerous characters, but preserve newlines, :::component syntax, and markdown
+    // Don't remove quotes, colons, or asterisks as they're needed for ::: components and markdown
+    value = value.replace(/[<>]/g, '');
+
+    // Remove any control characters except newlines (\n = \x0A)
+    value = value.replace(/[\x00-\x09\x0B-\x1F\x7F-\x9F]/g, '');
+
+    // Ensure proper spacing around ::: components
+    // Add newline before ::: if there isn't one already
+    value = value.replace(/([^\n])\s*:::/g, '$1\n :::');
+
+    // Auto-close ::: components if they're not already closed
+    // Check if there's a ::: component that's not properly closed
+    if (value.includes(':::')) {
+        // Count the number of ::: occurrences
+        const colonCount = (value.match(/:::/g) || []).length;
+        
+        // If we have an odd number of ::: components, we need to add a closing one
+        if (colonCount % 2 === 1) {
+            // Only add closing ::: if the string doesn't already end with it
+            if (!value.trim().endsWith(':::')) {
+                // Add newline before closing ::: if the string doesn't end with newline
+                if (!value.endsWith('\n')) {
+                    value = value + '\n';
+                }
+                value = value + ':::';
+            }
+        } else {
+            // Even number of ::: components, but we need to ensure proper formatting
+            // Look for patterns like "::: text" and convert to ":::\n text"
+            // Handle both cases: "::: text" and ":::\n text"
+            value = value.replace(/:::\s+([^\n]+)/g, ':::\n $1');
+            // Also handle the case where there's already a newline but we need to ensure proper spacing
+            value = value.replace(/:::\n\s*([^\n]+)/g, ':::\n $1');
+            // Specific case: if we have ":::\n text", ensure it becomes ":::\n text"
+            value = value.replace(/:::\n([^\n]+)/g, ':::\n $1');
+        }
+    }
+
+    // Trim whitespace
+    value = value.trim();
+
+    return value;
+}
+
+function removeParameters(text: string): string {
+    // Use a stack-based approach to handle nested brackets
+    let result = '';
+    let stack = 0;
+    let inQuotes = false;
+    let quoteChar = '';
+    
+    for (let i = 0; i < text.length; i++) {
+        const char = text[i];
+        
+        // Handle quotes
+        if ((char === '"' || char === "'") && !inQuotes) {
+            inQuotes = true;
+            quoteChar = char;
+        } else if (char === quoteChar && inQuotes) {
+            inQuotes = false;
+            quoteChar = '';
+        }
+        
+        // Only process brackets when not in quotes
+        if (!inQuotes) {
+            if (char === '[' || char === '{') {
+                stack++;
+            } else if (char === ']' || char === '}') {
+                stack--;
+            } else if (stack === 0) {
+                result += char;
+            }
+        }
+    }
+    
+    // Clean up multiple spaces and trim
+    return result.replace(/\s+/g, ' ').trim();
+}
+
+function parseParameters(
+    text: string,
+    delimiter: string,
+    closingDelimiter: string,
+    htmlMd: boolean = false
+) {
+    const attributes: Record<string, string> = {};
+    
+    // Find all parameter blocks using stack-based approach
+    const blocks = findNestedBlocks(text, delimiter, closingDelimiter);
+    
+    for (const blockContent of blocks) {
+        // Then parse individual parameters within the block
+        const paramRegex = /(!)?([^=\s]+)(?:=(?:"([^"]*)"|([^\s]*)))?/g;
+        let paramMatch;
+        
+        while ((paramMatch = paramRegex.exec(blockContent)) !== null) {
+            const [_, isNegated, prop, quotedValue, unquotedValue] = paramMatch;
+            const value = quotedValue !== undefined ? quotedValue : unquotedValue;
+
+            // Sanitize the property name
+            let sanitizedParam = sanitizeParameters(prop);
+            
+            // Handle the value - apply HTML to markdown transformation first if requested
+            let sanitizedValue: string;
+            if (value) {
+                if (htmlMd) {
+                    // Apply HTML to markdown transformation first, then sanitize
+                    sanitizedValue = sanitizeParameters(transformHtmlToMarkdown(value));
+                } else {
+                    sanitizedValue = sanitizeParameters(value);
+                }
+            } else {
+                sanitizedValue = isNegated ? 'false' : 'true';
+            }
+
+            if (sanitizedParam.startsWith("#") && sanitizedValue === "true") {
+                sanitizedValue = sanitizedParam.replace("#", "").trim()
+                sanitizedParam = "id"
+            }
+
+            attributes[sanitizedParam] = sanitizedValue;
+        }
+    }
+
+    return attributes;
+}
+
+function findNestedBlocks(text: string, openDelimiter: string, closeDelimiter: string): string[] {
+    const blocks: string[] = [];
+    let stack = 0;
+    let startIndex = -1;
+    let inQuotes = false;
+    let quoteChar = '';
+    
+    for (let i = 0; i < text.length; i++) {
+        const char = text[i];
+        
+        // Handle quotes
+        if ((char === '"' || char === "'") && !inQuotes) {
+            inQuotes = true;
+            quoteChar = char;
+        } else if (char === quoteChar && inQuotes) {
+            inQuotes = false;
+            quoteChar = '';
+        }
+        
+        // Only process delimiters when not in quotes
+        if (!inQuotes) {
+            if (char === openDelimiter) {
+                if (stack === 0) {
+                    startIndex = i + 1; // +1 to skip the opening delimiter
+                }
+                stack++;
+            } else if (char === closeDelimiter) {
+                stack--;
+                if (stack === 0 && startIndex !== -1) {
+                    blocks.push(text.substring(startIndex, i));
+                    startIndex = -1;
+                }
+            }
+        }
+    }
+    
+    return blocks;
+}
+
+function transformHtmlToMarkdown(text: string): string {
+    // Transform common HTML tags to markdown
+    return text
+        // <code> tags to backticks
+        .replace(/<code\b[^>]*>(.*?)<\/code>/gi, '`$1`')
+        // <strong> and <b> tags to bold
+        .replace(/<(strong|b)\b[^>]*>(.*?)<\/(strong|b)>/gi, '**$2**')
+        // <em> and <i> tags to italic
+        .replace(/<(em|i)\b[^>]*>(.*?)<\/(em|i)>/gi, '*$2*')
+        // <a> tags to markdown links
+        .replace(/<a\b[^>]*href=["']([^"']*)["'][^>]*>(.*?)<\/a>/gi, '[$2]($1)')
+        // <br> tags to line breaks
+        .replace(/<br\b[^>]*>/gi, '\n')
+        // <p> tags to paragraphs (with line breaks)
+        .replace(/<p\b[^>]*>(.*?)<\/p>/gi, '$1\n\n')
+        // <h1> to <h6> tags to markdown headers
+        .replace(/<h1\b[^>]*>(.*?)<\/h1>/gi, '# $1\n\n')
+        .replace(/<h2\b[^>]*>(.*?)<\/h2>/gi, '## $1\n\n')
+        .replace(/<h3\b[^>]*>(.*?)<\/h3>/gi, '### $1\n\n')
+        .replace(/<h4\b[^>]*>(.*?)<\/h4>/gi, '#### $1\n\n')
+        .replace(/<h5\b[^>]*>(.*?)<\/h5>/gi, '##### $1\n\n')
+        .replace(/<h6\b[^>]*>(.*?)<\/h6>/gi, '###### $1\n\n')
+        // <ul> and <ol> lists
+        .replace(/<ul\b[^>]*>(.*?)<\/ul>/gis, (match, content) => {
+            return content.replace(/<li\b[^>]*>(.*?)<\/li>/gi, '- $1\n') + '\n';
+        })
+        .replace(/<ol\b[^>]*>(.*?)<\/ol>/gis, (match, content) => {
+            let counter = 1;
+            return content.replace(/<li\b[^>]*>(.*?)<\/li>/gi, () => `${counter++}. $1\n`) + '\n';
+        })
+        // <blockquote> tags
+        .replace(/<blockquote\b[^>]*>(.*?)<\/blockquote>/gis, (match, content) => {
+            return content.split('\n').map(line => `> ${line}`).join('\n') + '\n\n';
+        })
+        // <pre> tags for code blocks
+        .replace(/<pre\b[^>]*>(.*?)<\/pre>/gis, '```\n$1\n```\n\n')
+        // Remove any remaining HTML tags
+        .replace(/<[^>]*>/g, '')
+        // Clean up multiple line breaks
+        .replace(/\n{3,}/g, '\n\n')
+        .trim();
+}

package/packages/md/plugins/utils/mdastTypes.ts

@@ -0,0 +1,42 @@
+/**
+ * List of standard MDAST node types
+ */
+export const standardMdastTypes = [
+  'root',
+  'paragraph',
+  'heading',
+  'text',
+  'emphasis',
+  'strong',
+  'delete',
+  'blockquote',
+  'code',
+  'link',
+  'image',
+  'list',
+  'listItem',
+  'table',
+  'tableRow',
+  'tableCell',
+  'html',
+  'break',
+  'thematicBreak',
+  'yaml',
+  'definition',
+  'footnoteDefinition',
+  'footnoteReference',
+  'inlineCode',
+  'linkReference',
+  'imageReference',
+  'footnote',
+  'tableCaption'
+];
+
+/**
+ * Checks if a given type is a standard MDAST type
+ * @param type The node type to check
+ * @returns True if the type is a standard MDAST type, false otherwise
+ */
+export function isStandardMdastType(type: string): boolean {
+  return standardMdastTypes.includes(type);
+} 

package/packages/md/search/index.ts

@@ -0,0 +1,251 @@
+import { readFileSync, statSync } from 'node:fs'
+import path from 'node:path'
+
+import { unified } from 'unified'
+import remarkParse from 'remark-parse'
+import remarkRehype from 'remark-rehype'
+import rehypeStringify from 'rehype-stringify'
+import { visit } from "unist-util-visit";
+
+import GitHubSlugger from 'github-slugger';
+
+import type { Settings, Sidebar, SidebarRoute } from '@xyd-js/core'
+
+import { DocSectionSchema } from './types';
+import { markdownPlugins } from "../"
+
+export async function mapSettingsToDocSections(xydSettings: Settings) {
+    const pages = flatPages(xydSettings.navigation?.sidebar || [], {})
+        .map(page => ({
+            name: page,
+            path: path.join(process.cwd(), page)
+        }))
+
+    return await processSections(pages, xydSettings)
+}
+
+export async function mapContentToDocSections(
+    xydSettings: Settings,
+    route: string,
+    content: string,
+) {
+    const sections: DocSectionSchema[] = []
+    let currentSection: DocSectionSchema | null = null
+
+    const rootSection: DocSectionSchema = {
+        // id: '',
+        pageId: '',
+        pageUrl: '',
+        pageTitle: '',
+        headingLevel: 0,
+        headingTitle: '',
+        summary: '',
+        content: '',
+    }
+
+    const { remarkPlugins, rehypePlugins } = markdownPlugins({}, xydSettings)
+
+    const processor = unified()
+        .use(remarkParse)
+        .use(remarkPlugins)
+        .use(() => {
+            return (tree: any) => {
+                let currentContent = ''
+
+                visit(tree, (node) => {
+                    // Skip yaml frontmatter
+                    if (node.type === 'yaml') {
+                        return
+                    }
+
+                    // Handle headings
+                    if (node.type === 'heading') {
+                        const heading = flattenNode(node)
+
+                        // Save previous section's content
+                        if (currentSection && currentContent) {
+                            currentSection.content = currentContent.trim()
+                            currentContent = ''
+                        }
+
+                        const slug = slugify(heading)
+
+                        if (node.depth === 1) {
+                            if (rootSection.pageId) {
+                                console.error("Multiple h1 found")
+                            }
+
+                            rootSection.pageId = slug
+                            rootSection.pageUrl = `/${route}`
+                            rootSection.pageTitle = heading
+                            rootSection.headingLevel = 1
+                            rootSection.headingTitle = heading
+                            sections.push(rootSection)
+                            currentSection = rootSection
+                        } else {
+                            const section: DocSectionSchema = {
+                                pageId: rootSection.pageId,
+                                pageUrl: `${rootSection.pageUrl}#${slug}`, // TODO: QUERY PARAMS
+                                pageTitle: rootSection.headingTitle,
+                                headingLevel: node.depth,
+                                headingTitle: heading,
+                                content: '',
+                                summary: '',
+                            }
+                            sections.push(section)
+                            currentSection = section
+                        }
+                        return
+                    }
+
+                    // Skip if we're not in a section yet
+                    if (!currentSection) {
+                        return
+                    }
+
+                    // For list items, add proper formatting
+                    if (node.type === 'listItem') {
+                        const itemContent = flattenNode(node)
+                        currentContent += `- ${itemContent}\n`
+                        return
+                    }
+
+                    // For links, preserve the markdown format
+                    if (node.type === 'link') {
+                        const text = flattenNode(node)
+                        currentContent += `[${text}](${node.url})\n`
+                        return
+                    }
+
+                    // For other nodes, just add their content
+                    const nodeContent = flattenNode(node)
+                    if (nodeContent) {
+                        currentContent += nodeContent + '\n'
+                    }
+                })
+
+                // Save the last section's content
+                if (currentSection && currentContent) {
+                    currentSection.content = currentContent.trim()
+                }
+            }
+        })
+        .use(remarkRehype)
+        .use(rehypePlugins)
+        .use(rehypeStringify)
+
+    await processor.process(content)
+
+    return sections
+}
+
+function flattenNode(node: any): string {
+    if (node.value) {
+        return node.value
+    }
+
+    if (node.children) {
+        return node.children.map((child: any) => flattenNode(child)).join('')
+    }
+
+    return ''
+}
+
+
+async function processSections(pages: { name: string, path: string }[], xydSettings: Settings) {
+    return (await Promise.all(
+        pages.flatMap(async (file, i) => {
+            let filePath = ""
+            let err
+
+            try {
+                const p = file.path + ".md"
+                statSync(p)
+                filePath = p
+            } catch (error) {
+                err = error
+            }
+
+            if (!filePath) {
+                try {
+                    const p = file.path + ".mdx"
+                    statSync(p)
+                    filePath = p
+                    err = null
+                } catch (error) {
+                    err = error
+                }
+            }
+
+            if (err) {
+                console.error(err)
+                return []
+            }
+
+            const content = readFileSync(filePath, 'utf-8')
+            const sections = await mapContentToDocSections(
+                xydSettings,
+                file.name,
+                content,
+            )
+
+            return sections.flat()
+        })
+    )).flat()
+}
+
+// TODO: !!!! DRY !!!
+function flatPages(
+    sidebar: (SidebarRoute | Sidebar | string)[],
+    groups: { [key: string]: string },
+    resp: string[] = [],
+) {
+    sidebar.map(async side => {
+        if (typeof side === "string") {
+          resp.push(side)
+          return
+        }
+
+        if ("route" in side) {
+            side.pages?.map(item => {
+                return flatPages([item], groups, resp)
+            })
+            return
+        }
+
+        // Type guard to check if it's a Sidebar
+        if ("group" in side) {
+            const groupKey = side.group || "";
+            if (groups[groupKey]) {
+                const link = groups[groupKey];
+                if (link) {
+                    resp.push(link);
+                }
+            }
+
+            side.pages?.map(async page => {
+                if (typeof page === "string") {
+                    resp.push(page);
+                    return;
+                }
+                
+                if ("virtual" in page && page.virtual) {
+                    resp.push(page.virtual);
+                    return; 
+                }
+
+                if ("pages" in page) {
+                    return flatPages([page as Sidebar], groups, resp);
+                }
+            });
+        }
+    });
+
+    return resp;
+}
+
+function slugify(text: string) {
+    const slugger = new GitHubSlugger()
+
+    return slugger.slug(text)
+}

package/packages/md/search/types.ts

@@ -0,0 +1,36 @@
+export interface DocSectionSchema { // TODO: move to @xyd-js/plugins/search ?
+    /**
+     * Id of the page e.g. "getting-started"
+     */
+    pageId: string
+
+    /**
+     * URL of the page e.g. "/getting-started"
+     */
+    pageUrl: string
+
+    /**
+     * Title of the page e.g. "Getting Started"
+     */
+    pageTitle: string
+
+    /**
+     * Level of the heading e.g. 1, 2
+     */
+    headingLevel: number
+
+    /**
+     * Title of the heading e.g. "Making a new project"
+     */
+    headingTitle: string
+
+    /**
+     * Summary of the page e.g. "This is the summary of the page"
+     */
+    summary: string
+
+    /**
+     * Content of the section
+     */
+    content: string
+}