@xyd-js/content 0.1.0-xyd.13 → 0.1.0-xyd.16

package/packages/md/plugins/functions/index.ts

@@ -0,0 +1,11 @@
+export {
+    mdFunctionImportCode
+} from "./mdFunctionImportCode"
+
+export {
+    mdFunctionUniform
+} from "./mdFunctionUniform"
+
+export {
+    mdFunctionChangelog
+} from "./mdFunctionChangelog"

package/packages/md/plugins/functions/mdFunctionChangelog.ts

@@ -0,0 +1,124 @@
+import { visit } from 'unist-util-visit';
+import { VFile } from 'vfile';
+import { unified } from 'unified';
+import remarkParse from 'remark-parse';
+import remarkMdx from 'remark-mdx';
+
+import { FunctionName } from './types';
+import { FunctionOptions, parseFunctionCall, downloadContent } from './utils';
+
+interface ChangelogEntry {
+    version: string;
+    date?: string;
+    content: string;
+}
+
+export function mdFunctionChangelog() {
+    return function (options: FunctionOptions = {}) {
+        return async function transformer(tree: any, file: VFile) {
+            const promises: Promise<void>[] = [];
+
+            visit(tree, 'paragraph', (node: any) => {
+                const result = parseFunctionCall(node, FunctionName.Changelog);
+                if (!result) return;
+
+                const importPath = result[0];
+
+                const promise = (async () => {
+                    try {
+                        const content = await downloadContent(importPath, file, options.resolveFrom);
+                        const entries = parseChangelog(content);
+
+                        // Replace the node with the generated content
+                        node.type = 'mdxJsxFlowElement';
+                        node.attributes = [];
+                        node.children = entries.map(entry => {
+                            // Parse the markdown content
+                            const parsedContent = unified()
+                                .use(remarkParse)
+                                .use(remarkMdx)
+                                .parse(entry.content);
+
+                            return {
+                                type: 'mdxJsxFlowElement',
+                                name: 'Update',
+                                attributes: [
+                                    {
+                                        type: 'mdxJsxAttribute',
+                                        name: 'version',
+                                        value: entry.version
+                                    },
+                                    {
+                                        type: 'mdxJsxAttribute',
+                                        name: 'date',
+                                        value: entry.date || ''
+                                    }
+                                ],
+                                children: parsedContent.children
+                            };
+                        });
+                    } catch (error) {
+                        console.error(`Error processing changelog: ${importPath}`, error);
+                    }
+                })();
+
+                promises.push(promise);
+            });
+
+            await Promise.all(promises);
+        };
+    };
+}
+
+function parseChangelog(content: string): ChangelogEntry[] {
+    const entries: ChangelogEntry[] = [];
+    const lines = content.split('\n');
+    let currentEntry: ChangelogEntry | null = null;
+    let currentContent: string[] = [];
+
+    for (let i = 0; i < lines.length; i++) {
+        const line = lines[i];
+
+        // Match different version header formats:
+        // 1. ## [X.Y.Z] - any date format
+        // 2. ## [X.Y.Z]
+        // 3. ## X.Y.Z
+        const versionMatch = line.match(/^##\s+(?:\[?([\d.]+)\]?(?:\s*-\s*([^\n]+))?)/);
+        if (versionMatch) {
+            // Save previous entry if exists
+            if (currentEntry) {
+                currentEntry.content = currentContent.join('\n').trim();
+                entries.push(currentEntry);
+            }
+
+            // Start new entry
+            currentEntry = {
+                version: versionMatch[1],
+                date: versionMatch[2]?.trim() || '',
+                content: ''
+            };
+            currentContent = [];
+            continue;
+        }
+
+        // Skip the main title (# xyd-js)
+        if (line.startsWith('# ')) {
+            continue;
+        }
+
+        // Add content to current entry
+        if (currentEntry) {
+            currentContent.push(line);
+        }
+    }
+
+    // Add last entry
+    if (currentEntry) {
+        currentEntry.content = currentContent.join('\n').trim();
+        entries.push(currentEntry);
+    }
+
+    return entries;
+}
+
+

package/packages/md/plugins/functions/mdFunctionImportCode.ts

@@ -0,0 +1,83 @@
+import { visit } from 'unist-util-visit';
+import { VFile } from 'vfile';
+
+import { Settings } from '@xyd-js/core';
+
+import { FunctionName } from "./types";
+import {
+    FunctionOptions,
+    parseFunctionCall,
+    parseImportPath,
+    processContent,
+    detectLanguage,
+    readLocalFile,
+    fetchFileContent,
+    resolvePathAlias
+} from './utils';
+
+export function mdFunctionImportCode(settings?: Settings) {
+    return function (options: FunctionOptions = {}) {
+        return async function transformer(tree: any, file: VFile) {
+            // Collect promises for async operations
+            const promises: Promise<void>[] = [];
+
+            console.time('plugin:mdFunctionImportCode');
+            
+            visit(tree, 'paragraph', (node: any) => {
+                // Try to parse the function call
+                const result = parseFunctionCall(node, FunctionName.ImportCode);
+                if (!result) return;
+
+                const importPath = result[0];
+
+                // Parse the import path to extract file path, regions, and line ranges
+                const { filePath, regions, lineRanges } = parseImportPath(
+                    resolvePathAlias(importPath, settings, process.cwd()) || importPath
+                );
+
+                // Determine if this is a local or external file
+                const isExternal = filePath.startsWith('http://') || filePath.startsWith('https://');
+
+                // Create a promise for this node
+                const promise = (async () => {
+                    try {
+                        // Get the file content
+                        let content: string;
+
+                        if (isExternal) {
+                            // Fetch external content
+                            content = await fetchFileContent(filePath);
+                        } else {
+                            const baseDir = options.resolveFrom || (file.dirname || process.cwd());
+                            content = readLocalFile(filePath, baseDir);
+                        }
+
+                        // Process the content based on regions and line ranges
+                        let processedContent = processContent(content, regions, lineRanges);
+
+                        // Detect language from file extension
+                        const language = detectLanguage(filePath);
+
+                        // Replace the paragraph with a code block
+                        node.type = 'code';
+                        node.lang = language;
+                        node.value = processedContent;
+                        node.children = undefined;
+
+                    } catch (error) {
+                        console.error(`Error importing file: ${filePath}`, error);
+                        // Keep the node as is if there's an error
+                    }
+                })();
+
+                // Add the promise to our collection
+                promises.push(promise);
+            });
+
+            // Wait for all promises to resolve
+            await Promise.all(promises);
+            console.timeEnd('plugin:mdFunctionImportCode');
+        };
+    }
+}
+

package/packages/md/plugins/functions/mdFunctionUniform.ts

@@ -0,0 +1,79 @@
+import { visit } from 'unist-util-visit';
+import { VFile } from 'vfile';
+
+import { Settings } from '@xyd-js/core'
+import uniform, { pluginJsonView, Reference, TypeDocReferenceContext } from "@xyd-js/uniform"
+
+import { FunctionName } from "./types";
+import {
+    FunctionOptions,
+    parseFunctionCall,
+} from './utils';
+
+// Import the processUniformFunctionCall function
+import { processUniformFunctionCall } from './uniformProcessor';
+import { uniformToMiniUniform } from '@xyd-js/sources/ts';
+
+export function mdFunctionUniform(settings?: Settings) {
+    return function (options: FunctionOptions = {}) {
+        return async function transformer(tree: any, file: VFile) {
+            // Collect promises for async operations
+            const promises: Promise<void>[] = [];
+
+            console.time('plugin:mdFunctionUniform');
+            
+            visit(tree, 'paragraph', (node: any) => {
+                // Try to parse the function call
+                const result = parseFunctionCall(node, FunctionName.Uniform);
+                if (!result) return;
+
+                const importPath = result[0];
+                const importArgs = result[1];
+
+                // Create a promise for this node
+                const promise = (async () => {
+                    try {
+                        // Process the uniform function call
+                        let references = await processUniformFunctionCall(
+                            importPath,
+                            file,
+                            options.resolveFrom,
+                            settings,
+                        );
+
+                        if (importArgs?.mini && references) { // TODO: move to `processUniformFunctionCall`
+                            references = uniformToMiniUniform(importArgs.mini, references as Reference<TypeDocReferenceContext>[]);
+                        }
+
+                        if (references) {
+                            node.type = 'code';
+                            node.lang = 'json';
+                            const jsonViewRefs = uniform(references, {
+                                plugins: [
+                                    pluginJsonView()
+                                ]
+                            })
+
+                            // TODO: support multiple json views
+                            node.value = jsonViewRefs.out.jsonViews[0]
+                            node.children = undefined;
+                        }
+                    } catch (error) {
+                        console.error(`Error processing uniform function call: ${importPath}`, error);
+                        // Keep the node as is if there's an error
+                    }
+                })();
+
+                // Add the promise to our collection
+                promises.push(promise);
+            });
+
+            // Wait for all promises to resolve
+            await Promise.all(promises);
+            console.timeEnd('plugin:mdFunctionUniform');
+        };
+    }
+
+}
+
+

package/packages/md/plugins/functions/types.ts

@@ -0,0 +1,6 @@
+export enum FunctionName {
+    ImportCode = '@importCode',
+    Import = '@import',
+    Uniform = '@uniform',
+    Changelog = '@changelog',
+}

package/packages/md/plugins/functions/uniformProcessor.ts

@@ -0,0 +1,349 @@
+import * as path from 'node:path';
+import * as fs from 'node:fs';
+import * as os from 'node:os';
+import { VFile } from 'vfile';
+
+import { Metadata, Settings } from '@xyd-js/core';
+import { sourcesToUniform, sourcesToUniformV2, type TypeDocReferenceContext } from '@xyd-js/sources/ts';
+import { reactDocgenToUniform, uniformToReactUniform } from '@xyd-js/sources/react';
+import { gqlSchemaToReferences } from "@xyd-js/gql"
+import {oapSchemaToReferences, deferencedOpenAPI, uniformPluginXDocsSidebar} from "@xyd-js/openapi"
+
+import { downloadContent, LineRange, parseImportPath, Region, resolvePathAlias } from './utils';
+import uniform, { Reference, ReferenceContext } from '@xyd-js/uniform';
+// TODO: rewrite to async
+
+/**
+ * Process a uniform function call and return the references
+ * 
+ * @param settings The settings object
+ * @param value The value containing the uniform function call
+ * @param file The VFile object
+ * @param resolveFrom Optional base directory to resolve relative paths from
+ * @returns A promise that resolves to the references or null if processing failed
+ */
+export async function processUniformFunctionCall(
+    value: string,
+    file: VFile,
+    resolveFrom?: string,
+    settings?: Settings,
+): Promise<Reference[] | null> {
+    // Parse the import path to extract file path
+    const { filePath, regions, lineRanges } = parseImportPath(value);
+
+    // Resolve path aliases and get the base directory
+    let resolvedFilePath = resolvePathAlias(filePath, settings, process.cwd());
+
+    if (resolvedFilePath.startsWith("~/")) {
+        resolvedFilePath = path.join(process.cwd(), resolvedFilePath.slice(2));
+    }
+
+    // Process the uniform file
+    const references = await processUniformFile(resolvedFilePath, regions, lineRanges, file, resolveFrom);
+
+    if (!references) {
+        return null
+    }
+
+    const plugins = globalThis.__xydUserUniformVitePlugins || []
+    const matter = file.data?.matter as Metadata
+    if (matter?.openapi) {
+        plugins.push(uniformPluginXDocsSidebar)
+    }
+
+    const uniformRefs = uniform(references, {
+        plugins: [
+            ...plugins,
+        ]
+    })
+
+    return uniformRefs.references
+}
+
+async function processUniformFile(
+    filePath: string,
+    regions: Region[],
+    lineRanges: LineRange[],
+    file: VFile,
+    resolveFrom?: string
+): Promise<any[] | null> {
+    try {
+        if (!isSupportedProgrammingSource(filePath)) {
+            // TODO: openapi + graphql
+            throw new Error(`Unsupported file type: ${filePath}`);
+        }
+
+        let ext = extension(filePath);
+
+        const matter = file.data?.matter as Metadata
+        if (matter?.openapi) {
+            ext = "openapi"
+        }
+
+        if (isLocalPath(filePath)) {
+            const baseDir = resolveFrom || (file.dirname || process.cwd());
+            const resolvedFilePath = path.resolve(baseDir, filePath);
+
+            switch (ext) {
+                case 'ts':
+                case 'tsx': {
+                    const packageDir = findClosestPackageJsonDir(
+                        baseDir,
+                        filePath,
+                    );
+
+                    if (packageDir) {
+                        // Extract the relative file path from the package directory
+                        const relativeFilePath = path.relative(packageDir, resolvedFilePath);
+
+                        try {
+                            let references: Reference[] = []
+
+                            switch (ext) {
+                                case 'ts': {
+                                    const typedocRefs = await sourcesToUniformV2(
+                                        packageDir,
+                                        [relativeFilePath]
+                                    )
+
+                                    if (!typedocRefs || !typedocRefs.references) {
+                                        console.error("Failed to process uniform file", filePath)
+                                        break
+                                    }
+
+                                    references = typedocRefs.references.filter(ref => {
+                                        const ctx = ref?.context as TypeDocReferenceContext
+        
+                                        const pathMatch = ctx?.fileFullPath === relativeFilePath
+                                        
+
+                                        if (regions.length > 0) {
+                                            const regionMatch = regions.some(region => {
+                                                return region.name === ctx?.symbolName // TODO: BETTER REGION API FOR TYPEDOC
+                                            })
+
+                                            return pathMatch && regionMatch
+                                        }
+
+                                        return pathMatch
+                                    })
+
+                                    break
+                                }
+                                
+                                case 'tsx': {
+                                    const resp = await sourcesToUniformV2(
+                                        packageDir,
+                                        [relativeFilePath]
+                                    )
+
+                                    if (!resp || !resp.references || !resp.projectJson) {
+                                        console.error("Failed to process uniform file", filePath)
+                                        return null
+                                    }
+                                    const typedocRefs = resp.references as Reference<TypeDocReferenceContext>[]
+
+                                    references = uniformToReactUniform(typedocRefs, resp.projectJson)
+
+                                    break
+                                }
+                            }
+
+                            return references
+                        } finally {
+                            // Clean up the temporary directory when done
+                            // cleanupTempFolder(tempDir);
+                        }
+                    } else {
+                        console.error("package.json not found", filePath)
+                    }
+                }
+
+                case 'graphql': {
+                    const references = await gqlSchemaToReferences(resolvedFilePath, {
+                        regions: regions.map(region => region.name),
+                    });
+
+                    return references;
+                }
+
+                case 'openapi': {
+                    const schema = await deferencedOpenAPI(resolvedFilePath);
+                    const references = oapSchemaToReferences(schema, {
+                        regions: regions.map(region => region.name)
+                    });
+
+                    return references;
+                }
+
+                default: {
+                    throw new Error(`Unsupported file extension: ${ext}`);
+                }
+            }
+
+        }
+
+        throw new Error("current implementation does not support remote files")
+
+        // For remote files, download the content
+        const content = await downloadContent(
+            filePath,
+            file,
+            resolveFrom,
+        );
+
+        // Fallback to creating temporary folder structure if no package.json found
+        const tempDir = await createTempFolderStructure(content);
+
+        try {
+            // Get the path to the package directory
+            const tempPackageDir = path.join(tempDir, 'packages', 'package');
+
+            // Process the content using sourcesToUniform
+            const references = await sourcesToUniformV2(
+                tempDir,
+                [tempPackageDir]
+            );
+
+            return references || null;
+        } finally {
+            // Clean up the temporary directory when done
+            cleanupTempFolder(tempDir);
+        }
+
+    } catch (error) {
+        console.error(`Error processing uniform file: ${filePath}`, error);
+        return null;
+    }
+}
+
+
+/**
+ * Creates a temporary folder structure with package.json, tsconfig.json, and src/index.ts
+ * @param content The content to be placed in src/index.ts
+ * @returns The path to the temporary directory
+ */
+async function createTempFolderStructure(content: string): Promise<string> {
+    // Create a temporary directory
+    const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), 'xyd-uniform-'));
+
+    // Create the package directory
+    const packageDir = path.join(tempDir, 'packages', 'package');
+    fs.mkdirSync(packageDir, { recursive: true });
+
+    // Create the src directory
+    const srcDir = path.join(packageDir, 'src');
+    fs.mkdirSync(srcDir, { recursive: true });
+
+    // Create package.json
+    const packageJson = {
+        "name": "@xyd-sources-examples/package-a",
+        "main": "dist/index.js"
+    };
+    fs.writeFileSync(
+        path.join(packageDir, 'package.json'),
+        JSON.stringify(packageJson, null, 2)
+    );
+
+    // Create tsconfig.json
+    const tsconfigJson = {
+        "compilerOptions": {
+            "outDir": "./dist"
+        }
+    };
+    fs.writeFileSync(
+        path.join(packageDir, 'tsconfig.json'),
+        JSON.stringify(tsconfigJson, null, 2)
+    );
+
+    // Create src/index.ts with the provided content
+    fs.writeFileSync(path.join(srcDir, 'index.ts'), content);
+
+    return tempDir;
+}
+
+/**
+ * Cleans up the temporary folder structure
+ * @param tempDir The path to the temporary directory
+ */
+function cleanupTempFolder(tempDir: string): void {
+    try {
+        // Recursively delete the temporary directory
+        fs.rmSync(tempDir, { recursive: true, force: true });
+    } catch (error) {
+        console.error(`Error cleaning up temporary directory: ${tempDir}`, error);
+    }
+}
+
+const supportedProgrammingExtensions: Record<string, boolean> = {
+    'ts': true,
+    'tsx': true,
+    'graphql': true,
+    'yaml': true,
+    'yml': true,
+    'json': true,
+
+    // TODO
+    // 'py': true,
+    // 'go': true,
+    // TODO: AND OTHER PROGRAMMING LANGUAGES IN THE FUTEURE
+}
+
+/**
+ * Check if a file is a programming source file based on its extension
+ * @param filePath The path to the file
+ * @returns True if the file is a programming source file
+ */
+function isSupportedProgrammingSource(filePath: string) {
+    const ext = extension(filePath);
+
+    if (supportedProgrammingExtensions[ext]) {
+        return true;
+    }
+
+    return false;
+}
+
+function extension(filePath: string) {
+    return path.extname(filePath).toLowerCase().replace('.', '');
+}
+
+/**
+ * Find the closest package.json directory for a given file path
+ * @param baseDir The base directory to start searching from (used as cwd for resolving relative paths)
+ * @param filePath The path to the file (can be relative)
+ * @returns The path to the directory containing package.json, or null if not found
+ */
+function findClosestPackageJsonDir(
+    baseDir: string,
+    filePath: string
+): string | null {
+    // Resolve the filePath relative to baseDir
+    const resolvedFilePath = path.resolve(baseDir, filePath);
+    let currentDir = path.dirname(resolvedFilePath);
+    const root = path.parse(currentDir).root;
+
+    while (currentDir !== root) {
+        try {
+            const files = fs.readdirSync(currentDir);
+            if (files.includes('package.json')) {
+                return currentDir;
+            }
+        } catch (error) {
+            // If we can't read the directory, move up
+            console.warn(`Cannot read directory ${currentDir}:`, error);
+        }
+        currentDir = path.dirname(currentDir);
+    }
+
+    return null;
+}
+
+/**
+ * Check if a path is a local file path (not a URL)
+ * @param filePath The path to check
+ * @returns True if the path is local
+ */
+function isLocalPath(filePath: string): boolean {
+    return !filePath.startsWith('http://') && !filePath.startsWith('https://');
+}