@xyd-js/content 0.0.0-build

package/packages/md/plugins/functions/__tests__/parseFunctionCall.test.ts

@@ -0,0 +1,70 @@
+import {describe, it, expect} from 'vitest';
+
+import {parseFunctionCall} from '../utils';
+import {FunctionName} from "../types";
+
+describe('parseFunctionCall', () => {
+    const testCases = [
+        {
+            name: '1a. should parse uniform with file path and options',
+            function: FunctionName.Uniform,
+            input: `@uniform('~/path/to/resource.ts', {"mini": "Settings"})`,
+            expected: ['~/path/to/resource.ts', {mini: 'Settings'}]
+        },
+        {
+            name: '1b. should parse uniform with file path and options',
+            function: FunctionName.Uniform,
+            input: `@uniform('~/path/to/resource.ts', {mini: 'Settings'})`,
+            expected: ['~/path/to/resource.ts', {mini: 'Settings'}]
+        },
+        {
+            name: '2. should parse uniform with file path only',
+            function: FunctionName.Uniform,
+            input: '@uniform(~/path/to/resource.ts)',
+            expected: ['~/path/to/resource.ts']
+        },
+        {
+            name: '3a. should handle malformed uniform call',
+            function: FunctionName.Uniform,
+            input: '@uniform_invalid()',
+            expected: null
+        },
+        {
+            name: '3b. should handle malformed uniform call',
+            function: FunctionName.Uniform,
+            input: 'uniform()',
+            expected: null
+        },
+        {
+            name: '4. should parse function with md attributes',
+            function: FunctionName.ImportCode,
+            input: `@importCode[descHead="Tip" desc="It's a nice code"] ~/path/to/resource.ts`,
+            expected: ['~/path/to/resource.ts', {__mdAttrs: {descHead: 'Tip', desc: "It's a nice code"}}]
+        },
+        {
+            name: '4. should parse function with md attributes + quotes',
+            function: FunctionName.ImportCode,
+            input: `@importCode[descHead="Tip" desc="You can use md attributes!"] "~/path/to/resource.ts"`,
+            expected: ['~/path/to/resource.ts', {__mdAttrs: {descHead: 'Tip', desc: "You can use md attributes!"}}]
+        },
+        {
+            name: '4. should parse function with md attributes + shortand attrs',
+            function: FunctionName.ImportCode,
+            input: `@importCode[!scroll lines descHead="Tip" desc="You can use md attributes!"] "~/path/to/resource.ts"`,
+            expected: ['~/path/to/resource.ts', {__mdAttrs: {scroll: "false", lines: "true", descHead: 'Tip', desc: "You can use md attributes!"}}]
+        }
+    ];
+
+    testCases.forEach(({name, function: functionName, input, expected}) => {
+        it(name, () => {
+            const node = {
+                children: [{
+                    type: 'text',
+                    value: input
+                }]
+            };
+            const result = parseFunctionCall(node, functionName);
+            expect(result).toEqual(expected);
+        });
+    });
+});

package/packages/md/plugins/functions/__tests__/testHelpers.ts

@@ -0,0 +1,95 @@
+import path from 'node:path';
+import {VFile} from 'vfile';
+
+export function importCodeMockTree(importPath: string) {
+    return {
+        type: 'root',
+        children: [
+            {
+                type: 'paragraph',
+                children: [
+                    {
+                        type: 'text',
+                        value: `@importCode "${importPath}"`
+                    }
+                ]
+            }
+        ]
+    };
+}
+
+export function importCodeMdAttributesMockTree(
+    importPath: string,
+    attributes: Record<string, string>
+) {
+    // Convert attributes object to md attribute string
+    const attrsString = Object.entries(attributes)
+        .map(([key, value]) => `${key}="${value}"`)
+        .join(' ');
+
+    return {
+        type: 'root',
+        children: [
+            {
+                type: 'paragraph',
+                children: [
+                    {
+                        type: 'text',
+                        value: `@importCode[${attrsString}] "${importPath}"`
+                    }
+                ]
+            }
+        ]
+    };
+}
+
+export function importCodeMockTreeAlternative(importPath: string) {
+    return {
+        type: 'root',
+        children: [
+            {
+                type: 'paragraph',
+                children: [
+                    {
+                        type: 'text',
+                        value: `@importCode("${importPath}")`
+                    }
+                ]
+            }
+        ]
+    };
+}
+
+
+export function createMockFile(dirname?: string) {
+    return new VFile({
+        path: 'test.md',
+        value: '',
+        dirname: dirname || path.resolve(__dirname, '../__fixtures__')
+    });
+}
+
+
+export function createMockFetch(mockResponses: Record<string, string>) {
+    const originalFetch = global.fetch;
+
+    global.fetch = (async (input: RequestInfo | URL, init?: RequestInit): Promise<Response> => {
+        const url = input.toString();
+
+        if (mockResponses[url]) {
+            return {
+                ok: true,
+                text: async () => mockResponses[url]
+            } as Response;
+        }
+
+        throw new Error(`Unexpected URL: ${url}`);
+    }) as typeof fetch;
+
+    return originalFetch;
+}
+
+
+export function restoreFetch(originalFetch: typeof global.fetch) {
+    global.fetch = originalFetch;
+}

package/packages/md/plugins/functions/index.ts

@@ -0,0 +1,15 @@
+export {
+    mdFunctionImportCode
+} from "./mdFunctionImportCode"
+
+export {
+    mdFunctionInclude
+} from "./mdFunctionInclude"
+
+export {
+    mdFunctionUniform
+} from "./mdFunctionUniform"
+
+export {
+    mdFunctionChangelog
+} from "./mdFunctionChangelog"

package/packages/md/plugins/functions/mdFunctionChangelog.ts

@@ -0,0 +1,135 @@
+import { visit } from 'unist-util-visit';
+import { VFile } from 'vfile';
+import { unified } from 'unified';
+import remarkParse from 'remark-parse';
+import remarkMdx from 'remark-mdx';
+
+import { Settings } from '@xyd-js/core';
+
+import { FunctionName } from './types';
+import { FunctionOptions, parseFunctionCall, downloadContent, resolvePathAlias } from './utils';
+import path from 'node:path';
+
+interface ChangelogEntry {
+    version: string;
+    date?: string;
+    content: string;
+}
+
+export function mdFunctionChangelog(settings?: Settings) {
+    return function (options: FunctionOptions = {}) {
+        return async function transformer(tree: any, file: VFile) {
+            const promises: Promise<void>[] = [];
+
+            visit(tree, 'paragraph', (node: any) => {
+                if (!file?.dirname) {
+                    return;
+                }
+                const result = parseFunctionCall(node, FunctionName.Changelog);
+                if (!result || !result.length) return;
+
+                const importPath = result[0];
+                // let fullDirPath = path.join(process.cwd(), file.dirname || "")
+                // fullDirPath = process.cwd()
+                const resolvedPath = resolvePathAlias(importPath, settings, file) || importPath;
+
+                const promise = (async () => {
+                    try {
+                        const content = await downloadContent(resolvedPath, file, options.resolveFrom);
+                        const entries = parseChangelog(content);
+
+                        // Replace the node with the generated content
+                        node.type = 'mdxJsxFlowElement';
+                        node.attributes = [];
+                        node.children = entries.map(entry => {
+                            // Parse the markdown content
+                            const parsedContent = unified()
+                                .use(remarkParse)
+                                .use(remarkMdx)
+                                .parse(entry.content);
+
+                            return {
+                                type: 'mdxJsxFlowElement',
+                                name: 'Update',
+                                attributes: [
+                                    {
+                                        type: 'mdxJsxAttribute',
+                                        name: 'version',
+                                        value: entry.version
+                                    },
+                                    {
+                                        type: 'mdxJsxAttribute',
+                                        name: 'date',
+                                        value: entry.date || ''
+                                    }
+                                ],
+                                children: parsedContent.children
+                            };
+                        });
+                    } catch (error) {
+                        console.error(`Error processing changelog: ${resolvedPath}`, error);
+                    }
+                })();
+
+                promises.push(promise);
+            });
+
+            await Promise.all(promises);
+        };
+    };
+}
+
+function parseChangelog(content: string): ChangelogEntry[] {
+    const entries: ChangelogEntry[] = [];
+    const lines = content.split('\n');
+    let currentEntry: ChangelogEntry | null = null;
+    let currentContent: string[] = [];
+
+    for (let i = 0; i < lines.length; i++) {
+        const line = lines[i];
+
+        // Match different version header formats:
+        // 1. ## [X.Y.Z] - any date format
+        // 2. ## [X.Y.Z]
+        // 3. ## X.Y.Z
+        // 4. ## [unreleased] - any date format
+        // 5. ## [0.0.1-alpha.0] - any date format
+        const versionMatch = line.match(/^##\s+(?:\[?([^\]]+)\]?(?:\s*-\s*([^\n]+))?)/);
+        if (versionMatch) {
+            // Save previous entry if exists
+            if (currentEntry) {
+                currentEntry.content = currentContent.join('\n').trim();
+                entries.push(currentEntry);
+            }
+
+            // Start new entry
+            currentEntry = {
+                version: versionMatch[1],
+                date: versionMatch[2]?.trim() || '',
+                content: ''
+            };
+            currentContent = [];
+            continue;
+        }
+
+        // Skip the main title (# xyd-js)
+        if (line.startsWith('# ')) {
+            continue;
+        }
+
+        // Add content to current entry
+        if (currentEntry) {
+            currentContent.push(line);
+        }
+    }
+
+    // Add last entry
+    if (currentEntry) {
+        currentEntry.content = currentContent.join('\n').trim();
+        entries.push(currentEntry);
+    }
+
+    return entries;
+}
+
+

package/packages/md/plugins/functions/mdFunctionImportCode.ts

@@ -0,0 +1,92 @@
+import { visit } from 'unist-util-visit';
+import { VFile } from 'vfile';
+
+import { Settings } from '@xyd-js/core';
+
+import { FunctionName } from "./types";
+import {
+    FunctionOptions,
+    parseFunctionCall,
+    parseImportPath,
+    processContent,
+    detectLanguage,
+    readLocalFile,
+    fetchFileContent,
+    resolvePathAlias
+} from './utils';
+import { injectCodeMeta } from '../utils/injectCodeMeta';
+import path from 'node:path';
+
+export function mdFunctionImportCode(settings?: Settings) {
+    return function (options: FunctionOptions = {}) {
+        return async function transformer(tree: any, file: VFile) {
+            // Collect promises for async operations
+            const promises: Promise<void>[] = [];
+
+            console.time('plugin:mdFunctionImportCode');
+            
+            visit(tree, 'paragraph', (node: any) => {
+                // Try to parse the function call
+                const result = parseFunctionCall(node, FunctionName.ImportCode);
+                if (!result) return;
+
+                const importPath = result[0];
+                const args = result[1];
+                const mdAttrs = args?.__mdAttrs || {};
+
+                // let fullDirPath = path.join(process.cwd(), file.dirname || "")
+                // fullDirPath = process.cwd()
+                // Parse the import path to extract file path, regions, and line ranges
+                const { filePath, regions, lineRanges } = parseImportPath(
+                    resolvePathAlias(importPath, settings, file) || importPath
+                );
+
+                // Determine if this is a local or external file
+                const isExternal = filePath.startsWith('http://') || filePath.startsWith('https://');
+
+                // Create a promise for this node
+                const promise = (async () => {
+                    try {
+                        // Get the file content
+                        let content: string;
+
+                        if (isExternal) {
+                            // Fetch external content
+                            content = await fetchFileContent(filePath);
+                        } else {
+                            const baseDir = options.resolveFrom || (file.dirname || process.cwd());
+                            content = readLocalFile(filePath, baseDir);
+                        }
+
+                        // Process the content based on regions and line ranges
+                        let processedContent = processContent(content, regions, lineRanges);
+
+                        // Detect language from file extension
+                        const language = detectLanguage(filePath);
+
+                        // Replace the paragraph with a code block
+                        node.type = 'code';
+                        node.lang = language;
+                        node.value = processedContent;
+                        node.children = undefined;
+
+                        // Inject code meta and props
+                        const attrsString = args?.__mdAttrs ? Object.entries(args.__mdAttrs).map(([k,v]) => `${k}="${v}"`).join(' ') : undefined;
+                        const metaString = attrsString ? `[${attrsString}]` : undefined;
+                        injectCodeMeta(node, metaString);
+                    } catch (error) {
+                        console.error(`Error importing file: ${filePath}`, error);
+                        // Keep the node as is if there's an error
+                    }
+                })();
+
+                // Add the promise to our collection
+                promises.push(promise);
+            });
+
+            // Wait for all promises to resolve
+            await Promise.all(promises);
+            console.timeEnd('plugin:mdFunctionImportCode');
+        };
+    }
+}

package/packages/md/plugins/functions/mdFunctionInclude.ts

@@ -0,0 +1,119 @@
+import { visit } from 'unist-util-visit';
+import { VFile } from 'vfile';
+import { unified } from 'unified';
+import remarkParse from 'remark-parse';
+import { fromMarkdown } from 'mdast-util-from-markdown';
+import { mdxjs } from 'micromark-extension-mdxjs';
+import { mdxFromMarkdown } from 'mdast-util-mdx';
+
+import { Settings } from '@xyd-js/core';
+
+import { FunctionName } from './types';
+import {
+    FunctionOptions,
+    parseFunctionCall,
+    downloadContent,
+    resolvePathAlias,
+} from './utils';
+import { defaultRemarkPlugins } from '..';
+import remarkFrontmatter from 'remark-frontmatter';
+import remarkMdxFrontmatter from 'remark-mdx-frontmatter';
+import { remarkMdxToc } from '../mdToc';
+import path from 'path';
+
+export function mdFunctionInclude(settings?: Settings) {
+    return function (options: FunctionOptions = {}) {
+        return async function transformer(tree: any, file: VFile) {
+            console.time('plugin:mdFunctionInclude');
+
+            // Collect all replacements to avoid index shifting issues
+            const replacements: Array<{
+                parent: any;
+                index: number;
+                children: any[];
+            }> = [];
+
+            // First pass: collect all @include nodes and their replacements
+            const promises: Promise<void>[] = [];
+
+            visit(tree, 'paragraph', (node: any, index: number | undefined, parent: any) => {
+                const result = parseFunctionCall(node, FunctionName.Include);
+                if (!result || index === undefined) return;
+
+                const importPath = result[0];
+                const resolvedPath =
+                    resolvePathAlias(importPath, settings, file) || importPath;
+
+                const resolvedIncludeBasePath =
+                    resolvePathAlias(resolvedPath, settings, file) || importPath;
+
+                const promise = (async () => {
+                    try {
+                        const ext = resolvedPath.split('.').pop();
+                        const content = await downloadContent(
+                            resolvedPath,
+                            file,
+                            options.resolveFrom,
+                        );
+
+                        const includeFilePath = path.join(file.dirname || "", resolvedIncludeBasePath)
+                        const includedFile = new VFile({
+                            path: includeFilePath,
+                            dirname: path.dirname(includeFilePath),
+                            value: content
+                        });
+
+                        let parsedTree;
+
+                        const remarkPlugins = defaultRemarkPlugins({} as any, settings)
+                            .filter(plugin => plugin !== remarkFrontmatter)
+                            .filter(plugin => plugin !== remarkMdxFrontmatter)
+                            .filter(plugin => !(typeof plugin === 'function' && (plugin.name === 'remarkMdxToc' || plugin.name === 'remarkMdxToc2')))
+                            
+                        if (ext === 'mdx') {
+                            // Parse MDX into MDAST using micromark + mdast-util-mdx
+                            parsedTree = fromMarkdown(content, {
+                                extensions: [mdxjs()],
+                                mdastExtensions: [mdxFromMarkdown()],
+                            });
+
+                            // Apply remark plugins to the parsed MDX tree with the correct file context
+                            const mdxProcessor = unified().use(remarkPlugins);
+                            parsedTree = await mdxProcessor.run(parsedTree, includedFile);
+                        } else {
+                            // Regular Markdown file using remarkParse
+                            const processor = unified()
+                                .use(remarkParse)
+                                .use(remarkPlugins);
+
+                            parsedTree = await processor.run(processor.parse(content), includedFile);
+                        }
+
+                        // Store the replacement instead of applying it immediately
+                        replacements.push({
+                            parent,
+                            index,
+                            children: parsedTree.children,
+                        });
+                    } catch (error) {
+                        console.error(`Error including file: ${resolvedPath}`, error);
+                    }
+                })();
+
+                promises.push(promise);
+            });
+
+            // Wait for all content to be downloaded and parsed
+            await Promise.all(promises);
+
+            // Second pass: apply replacements in reverse order to avoid index shifting
+            replacements
+                .sort((a, b) => b.index - a.index) // Sort by index in descending order
+                .forEach(({ parent, index, children }) => {
+                    parent.children.splice(index, 1, ...children);
+                });
+
+            console.timeEnd('plugin:mdFunctionInclude');
+        };
+    };
+}

package/packages/md/plugins/functions/mdFunctionUniform.ts

@@ -0,0 +1,79 @@
+import { visit } from 'unist-util-visit';
+import { VFile } from 'vfile';
+
+import { Settings } from '@xyd-js/core'
+import uniform, { pluginJsonView, Reference, TypeDocReferenceContext } from "@xyd-js/uniform"
+
+import { FunctionName } from "./types";
+import {
+    FunctionOptions,
+    parseFunctionCall,
+} from './utils';
+
+// Import the processUniformFunctionCall function
+import { processUniformFunctionCall } from './uniformProcessor';
+import { uniformToMiniUniform } from '@xyd-js/sources/ts';
+
+export function mdFunctionUniform(settings?: Settings) {
+    return function (options: FunctionOptions = {}) {
+        return async function transformer(tree: any, file: VFile) {
+            // Collect promises for async operations
+            const promises: Promise<void>[] = [];
+
+            console.time('plugin:mdFunctionUniform');
+            
+            visit(tree, 'paragraph', (node: any) => {
+                // Try to parse the function call
+                const result = parseFunctionCall(node, FunctionName.Uniform);
+                if (!result) return;
+
+                const importPath = result[0];
+                const importArgs = result[1];
+
+                // Create a promise for this node
+                const promise = (async () => {
+                    try {
+                        // Process the uniform function call
+                        let references = await processUniformFunctionCall(
+                            importPath,
+                            file,
+                            options.resolveFrom,
+                            settings,
+                        );
+
+                        if (importArgs?.mini && references) { // TODO: move to `processUniformFunctionCall`
+                            references = uniformToMiniUniform(importArgs.mini, references as Reference<TypeDocReferenceContext>[]);
+                        }
+
+                        if (references) {
+                            node.type = 'code';
+                            node.lang = 'json';
+                            const jsonViewRefs = uniform(references, {
+                                plugins: [
+                                    pluginJsonView()
+                                ]
+                            })
+
+                            // TODO: support multiple json views
+                            node.value = jsonViewRefs.out.jsonViews[0]
+                            node.children = undefined;
+                        }
+                    } catch (error) {
+                        console.error(`Error processing uniform function call: ${importPath}`, error);
+                        // Keep the node as is if there's an error
+                    }
+                })();
+
+                // Add the promise to our collection
+                promises.push(promise);
+            });
+
+            // Wait for all promises to resolve
+            await Promise.all(promises);
+            console.timeEnd('plugin:mdFunctionUniform');
+        };
+    }
+
+}
+
+

package/packages/md/plugins/functions/types.ts

@@ -0,0 +1,9 @@
+export enum FunctionName {
+    ImportCode = '@importCode',
+
+    Include = '@include',
+
+    Uniform = '@uniform',
+
+    Changelog = '@changelog',
+}