@xyd-js/content 0.1.0-build.171

package/packages/md/plugins/functions/utils.ts

@@ -0,0 +1,457 @@
+import * as fs from 'node:fs';
+import * as path from 'node:path';
+import {VFile} from 'vfile';
+
+import {Settings} from '@xyd-js/core';
+import {mdParameters} from "../utils/mdParameters";
+
+/**
+ * Common options for all function plugins
+ */
+export interface FunctionOptions {
+    resolveFrom?: string;
+}
+
+/**
+ * Parse a function call with arguments
+ * @param node The AST node to parse
+ * @param functionName The name of the function to look for
+ */
+export function parseFunctionCall<args = any>(node: any, functionName: string): [string, args] | null {
+    function resp(args: any) {
+        const response: any[] = []
+
+        response.push(args[0])
+        let jsonArgs = undefined;
+        let argsString = args[1]
+        if (argsString) {
+            try {
+                argsString = argsString
+                    .replace(/'/g, '"')
+                    .replace(/(\w+):/g, '"$1":');
+
+                jsonArgs = JSON.parse(argsString);
+                response.push(jsonArgs);
+            } catch (e) {
+            }
+        }
+
+        return response as [string, any];
+    }
+
+    // Check for the simple case with a single text node
+    if (node.children && node.children.length === 1 && node.children[0].type === 'text') {
+        const textNode = node.children[0];
+
+        // Check for Markdown attribute syntax for the given function name
+        const mdAttrMatch = textNode.value.match(new RegExp(`^@?${functionName}\\[(.*)\\]\\s+(.*)$`));
+        if (mdAttrMatch) {
+            const attrsString = mdAttrMatch[1];
+            let pathString = mdAttrMatch[2].trim();
+            if ((pathString.startsWith('"') && pathString.endsWith('"')) || (pathString.startsWith("'") && pathString.endsWith("'"))) {
+                pathString = pathString.slice(1, -1);
+            }
+
+            const {attributes} = mdParameters(`[${attrsString}]`)
+            return [pathString, { __mdAttrs: attributes } as any];
+        }
+
+        // Accept @ prefix for function name
+        const fnRegex = new RegExp(`^@?${functionName}`);
+        if (!fnRegex.test(textNode.value)) {
+            return null;
+        }
+
+        // Check for parentheses syntax with multiple arguments
+        const parenthesesMatch = textNode.value.match(new RegExp(`^@?${functionName}\\((.*)\\)$`));
+        if (parenthesesMatch) {
+            const argsText = parenthesesMatch[1];
+            const args = argsText.split(',').map((arg: string) => arg.trim().replace(/^['\"]|['\"]$/g, ''));
+            return resp(args)
+        }
+
+        // Check for the original syntax
+        const originalMatch = textNode.value.match(new RegExp(`^@?${functionName}\\s+['\"](.*)['\"]$`));
+        if (originalMatch) {
+            return resp([originalMatch[1], originalMatch[2]]);
+        }
+    }
+
+    // Check for the complex case with multiple nodes
+    if (!node || !node.children || node.children.length < 3) {
+        return null;
+    }
+
+    // Check if the first node contains the function part
+    const firstNode = node.children[0];
+    const middleNode = node.children[1];
+    const lastNode = node.children[2];
+
+    if (firstNode.type === 'text' &&
+        firstNode.value.startsWith(`${functionName} "`) &&
+        middleNode.type === 'link' &&
+        lastNode.type === 'text' &&
+        lastNode.value === '"') {
+
+        // We found a split command, extract the URL from the link node
+        const url = middleNode.url;
+
+        return resp([url]) // Create a match array with the URL in position 1
+    }
+
+    // Check for parentheses syntax with multiple arguments
+    if (firstNode.type === 'text' &&
+        firstNode.value.startsWith(`${functionName}(`) &&
+        lastNode.type === 'text' &&
+        lastNode.value === ')') {
+
+        // Extract the arguments from the middle node
+        if (middleNode.type === 'text') {
+            // Simple case: all arguments in a single text node
+            const argsText = middleNode.value;
+            // Split by comma and trim each argument
+            const args = argsText.split(',').map((arg: string) => arg.trim().replace(/^["']|["']$/g, ''));
+            // Return the first argument as the path
+            return resp(args);
+        } else if (middleNode.type === 'link') {
+            // Case with a link node (for URLs)
+            const url = middleNode.url;
+            return resp([url]);
+        }
+    }
+
+    return null;
+}
+
+/**
+ * Parse an import path to extract file path, regions, and line ranges
+ */
+export function parseImportPath(importPath: string): {
+    filePath: string;
+    regions: Region[];
+    lineRanges: LineRange[]
+} {
+    // Initialize result
+    const result = {
+        filePath: importPath,
+        regions: [] as Region[],
+        lineRanges: [] as LineRange[]
+    };
+
+    // First, handle line ranges in the main path (not in regions)
+    // Only match line ranges that contain numbers
+    const mainLineRangeMatch = result.filePath.match(/\{([0-9,\s:-]+)\}/);
+    if (mainLineRangeMatch) {
+        const lineRangeStr = mainLineRangeMatch[1];
+        result.filePath = result.filePath.replace(/\{[0-9,\s:-]+\}/, '');
+
+        // Parse line ranges like "1,2-4, 8:, :10"
+        const rangeParts = lineRangeStr.split(',').map(part => part.trim());
+
+        for (const part of rangeParts) {
+            if (part.includes('-')) {
+                // Range like "2-4"
+                const [start, end] = part.split('-').map(num => parseInt(num, 10));
+                result.lineRanges.push({start, end});
+            } else if (part.endsWith(':')) {
+                // Range like "8:"
+                const start = parseInt(part.replace(':', ''), 10);
+                result.lineRanges.push({start});
+            } else if (part.startsWith(':')) {
+                // Range like ":10"
+                const end = parseInt(part.replace(':', ''), 10);
+                result.lineRanges.push({end});
+            } else {
+                // Single line like "1"
+                const line = parseInt(part, 10);
+                result.lineRanges.push({start: line, end: line});
+            }
+        }
+    }
+
+    // Then, handle regions
+    const hashIndex = result.filePath.indexOf('#');
+    if (hashIndex !== -1) {
+        // Split the path at the hash
+        const basePath = result.filePath.substring(0, hashIndex);
+        const regionPart = result.filePath.substring(hashIndex + 1);
+
+        // Update the file path
+        result.filePath = basePath;
+
+        // Check if the region part contains a numeric line range
+        const regionLineRangeMatch = regionPart.match(/\{([0-9,\s:-]+)\}/);
+
+        if (regionLineRangeMatch) {
+            // If there are numeric line ranges in the region part, extract them
+            const regionLineRangeStr = regionLineRangeMatch[1];
+            const regionName = regionPart.replace(/\{[0-9,\s:-]+\}/, '').trim();
+
+            // Parse the line ranges
+            const rangeParts = regionLineRangeStr.split(',').map(part => part.trim());
+            const regionLineRanges: LineRange[] = [];
+
+            for (const part of rangeParts) {
+                if (part.includes('-')) {
+                    // Range like "2-4"
+                    const [start, end] = part.split('-').map(num => parseInt(num, 10));
+                    regionLineRanges.push({start, end});
+                } else if (part.endsWith(':')) {
+                    // Range like "8:"
+                    const start = parseInt(part.replace(':', ''), 10);
+                    regionLineRanges.push({start});
+                } else if (part.startsWith(':')) {
+                    // Range like ":10"
+                    const end = parseInt(part.replace(':', ''), 10);
+                    regionLineRanges.push({end});
+                } else {
+                    // Single line like "1"
+                    const line = parseInt(part, 10);
+                    regionLineRanges.push({start: line, end: line});
+                }
+            }
+
+            // Add the region with its line ranges
+            result.regions.push({name: regionName, lineRanges: regionLineRanges});
+        } else {
+            // If there are no numeric line ranges in the region part, check if it contains commas
+            if (regionPart.includes(',')) {
+                // Split by comma for multiple regions
+                const regionNames = regionPart.split(',');
+                for (const name of regionNames) {
+                    result.regions.push({name: name.trim()});
+                }
+            } else {
+                // Single region
+                result.regions.push({name: regionPart.trim()});
+            }
+        }
+    }
+
+    return result;
+}
+
+/**
+ * Process content based on regions and line ranges
+ */
+export function processContent(content: string, regions: Region[], lineRanges: LineRange[]): string {
+    const lines = content.split('\n');
+
+    // If no regions or line ranges specified, return the original content
+    if (regions.length === 0 && lineRanges.length === 0) {
+        return content;
+    }
+
+    // Process regions if present
+    if (regions.length > 0) {
+        const regionLines: string[] = [];
+
+        for (const region of regions) {
+            const regionStart = lines.findIndex(line => line.includes(`#region ${region.name}`));
+            const regionEnd = lines.findIndex(line => line.includes(`#endregion ${region.name}`));
+
+            if (regionStart !== -1 && regionEnd !== -1) {
+                // Only include the content between region markers, not the markers themselves
+                for (let i = regionStart + 1; i < regionEnd; i++) {
+                    regionLines.push(lines[i]);
+                }
+            }
+        }
+
+        // If we found regions, return only the region content
+        if (regionLines.length > 0) {
+            return regionLines.join('\n');
+        }
+    }
+
+    // Process line ranges if present
+    if (lineRanges.length > 0) {
+        // Create a set of line numbers to include
+        const lineSet = new Set<number>();
+
+        for (const range of lineRanges) {
+            const start = range.start || 1;
+            const end = range.end || lines.length;
+
+            // Adjust for 0-based indexing
+            const startIndex = Math.max(0, start - 1);
+            const endIndex = Math.min(lines.length, end);
+
+            for (let i = startIndex; i < endIndex; i++) {
+                lineSet.add(i);
+            }
+        }
+
+        // Filter lines based on the set
+        const selectedLines = lines.filter((_, index) => lineSet.has(index));
+        return selectedLines.join('\n');
+    }
+
+    return content;
+}
+
+// Map common extensions to languages
+const languageMap: Record<string, string> = {
+    'js': 'javascript',
+    'jsx': 'jsx',
+    'ts': 'typescript',
+    'tsx': 'tsx',
+    'py': 'python',
+    'rb': 'ruby',
+    'java': 'java',
+    'c': 'c',
+    'cpp': 'cpp',
+    'cs': 'csharp',
+    'go': 'go',
+    'rs': 'rust',
+    'php': 'php',
+    'swift': 'swift',
+    'kt': 'kotlin',
+    'scala': 'scala',
+    'html': 'html',
+    'css': 'css',
+    'scss': 'scss',
+    'less': 'less',
+    'json': 'json',
+    'xml': 'xml',
+    'yaml': 'yaml',
+    'yml': 'yaml',
+    'md': 'markdown',
+    'mdx': 'mdx',
+    'sh': 'bash',
+    'bash': 'bash',
+    'sql': 'sql',
+    'graphql': 'graphql',
+    'vue': 'vue',
+    'svelte': 'svelte',
+};
+
+/**
+ * Detect language from file extension
+ */
+export function detectLanguage(filePath: string): string {
+    const extension = path.extname(filePath).toLowerCase().replace('.', '');
+
+    return languageMap[extension] || path.extname(filePath).split('.').pop() || '';
+}
+
+/**
+ * Read a local file
+ */
+export function readLocalFile(filePath: string, baseDir: string): string {
+    filePath = parseIfLocalHomePath(filePath);
+
+    const fullPath = path.resolve(baseDir, filePath);
+    return fs.readFileSync(fullPath, 'utf8');
+}
+
+function parseIfLocalHomePath(filePath: string): string {
+    // Handle "~/" prefix by replacing it with the current working directory
+
+    if (filePath.startsWith('~/')) {
+        filePath = filePath.replace('~/', process.cwd() + '/');
+    }
+
+    return filePath;
+}
+
+/**
+ * Fetch file content from a URL
+ */
+export async function fetchFileContent(url: string): Promise<string> {
+    const response = await fetch(url);
+    if (!response.ok) {
+        throw new Error(`Failed to fetch ${url}: ${response.status} ${response.statusText}`);
+    }
+    return response.text();
+}
+
+/**
+ * Interface for line range
+ */
+export interface LineRange {
+    start?: number;
+    end?: number;
+}
+
+/**
+ * Interface for region
+ */
+export interface Region {
+    name: string;
+    lineRanges?: LineRange[];
+}
+
+
+export async function downloadContent(
+    filePath: string,
+    file: VFile,
+    resolveFrom?: string,
+) {
+    const isExternal = filePath.startsWith('http://') || filePath.startsWith('https://');
+
+    let content: string;
+
+    if (isExternal) {
+        // Fetch external content
+        content = await fetchFileContent(filePath);
+    } else {
+        const baseDir = resolveFrom || (file.dirname || process.cwd());
+        content = readLocalFile(filePath, baseDir);
+    }
+
+    return content;
+}
+
+export function functionMatch(value: string, functionName: string): boolean {
+    return value.startsWith(functionName); // TODO: better function matching like args etc
+}
+
+/**
+ * Resolves a path alias if the given path is an alias defined in the settings.
+ * @param inputPath The path to resolve
+ * @param settings The settings object containing path aliases
+ * @param baseDir Optional base directory to resolve the final path from
+ * @returns The resolved path or the original path if no alias is found
+ */
+export function resolvePathAlias(inputPath: string, settings?: Settings, currentFile?: VFile, cwd: string = process.cwd()): string {
+    const baseDir = path.join(cwd, currentFile?.dirname || "");
+
+    if (!settings?.engine?.paths) {
+        return parseIfLocalHomePath(inputPath)
+    }
+
+    // Find the longest matching alias
+    let resolvedPath = inputPath;
+    let longestMatch = '';
+
+    for (const [alias, aliasPaths] of Object.entries(settings.engine.paths)) {
+        // Convert alias pattern to regex, replacing * with .*
+        const aliasPattern = alias.replace(/\*/g, '.*');
+        const aliasRegex = new RegExp(`^${aliasPattern}`);
+
+        if (aliasRegex.test(inputPath) && alias.length > longestMatch.length) {
+            longestMatch = alias;
+            // Replace the alias with the path (handle both string and array)
+            const aliasPath = Array.isArray(aliasPaths) ? aliasPaths[0] : aliasPaths;
+            // Extract the part after the alias pattern
+            const aliasPrefix = alias.replace('*', '');
+            const matchedPart = inputPath.substring(aliasPrefix.length);
+            // Remove leading slash if present
+            const cleanMatchedPart = matchedPart.startsWith('/') ? matchedPart.slice(1) : matchedPart;
+            // If aliasPath contains *, replace it with the matched part, otherwise append the matched part
+            if (aliasPath.includes('*')) {
+                resolvedPath = aliasPath.replace(/\*/g, cleanMatchedPart);
+            } else {
+                resolvedPath = path.join(aliasPath, cleanMatchedPart);
+            }
+        }
+    }
+
+    // If we found a match and have a base directory, resolve the path relative to it
+    if (longestMatch) {
+        resolvedPath = path.resolve(cwd, resolvedPath);
+    }
+
+    return resolvedPath;
+}

package/packages/md/plugins/index.ts

@@ -0,0 +1,125 @@
+import remarkFrontmatter from "remark-frontmatter";
+import remarkMdxFrontmatter from "remark-mdx-frontmatter";
+import remarkGfm from "remark-gfm";
+import remarkDirective from 'remark-directive'
+import rehypeRaw from 'rehype-raw';
+import remarkMath from 'remark-math'
+import rehypeKatex from 'rehype-katex'
+
+import { Settings } from "@xyd-js/core";
+
+import { remarkMdxToc, RemarkMdxTocOptions } from "./mdToc";
+import { remarkInjectCodeMeta } from "./mdCode";
+import { extractThemeSettings } from "./mdThemeSettings";
+import { extractPage } from "./mdPage";
+import { mdHeadingId } from "./mdHeadingId";
+import { rehypeHeading } from "./rehypeHeading";
+import { mdComponentDirective } from "./component-directives";
+import { mdFunctionChangelog, mdFunctionImportCode, mdFunctionUniform, mdFunctionInclude } from "./functions"
+import { mdCodeRehype } from "./developer-writing";
+import { mdMeta } from "./meta";
+import { mdComposer } from "./composer/mdComposer";
+import { outputVars } from "./output-variables";
+import { mdImage } from "./mdImage";
+import { mdImageRehype } from "./mdImageRehype";
+
+import { recmaOverrideComponents } from "./recmaOverrideComponents";
+
+export function defaultRemarkPlugins(
+  toc: RemarkMdxTocOptions,
+  settings?: Settings
+) {
+  return [
+    ...thirdPartyRemarkPlugins(),
+    ...remarkPlugins(toc, settings),
+  ]
+}
+
+export function thirdPartyRemarkPlugins() {
+  return [
+    remarkFrontmatter,
+    remarkMdxFrontmatter,
+    remarkGfm,
+    remarkDirective,
+    remarkMath,
+  ]
+}
+
+function remarkPlugins(
+  toc: RemarkMdxTocOptions,
+  settings?: Settings
+) {
+  return [
+    mdHeadingId,
+    remarkInjectCodeMeta,
+    mdImage,
+    remarkMdxToc(toc),
+    extractThemeSettings, // TODO: to delet ?
+    extractPage, // TODO: to delete ?
+    outputVars,
+    mdComponentDirective(settings),
+    mdComposer(settings),
+    ...remarkFunctionPlugins(settings),
+    mdMeta(settings),
+  ]
+}
+
+export function includeRemarkPlugins(settings?: Settings) {
+  return [
+    remarkGfm,
+    remarkDirective,
+
+    mdHeadingId,
+    remarkInjectCodeMeta,
+    outputVars,
+    mdComponentDirective(settings),
+    ...remarkFunctionPlugins(settings),
+  ]
+}
+
+export function remarkFunctionPlugins(settings?: Settings) {
+  return [
+    mdFunctionImportCode(settings),
+    mdFunctionUniform(settings),
+    mdFunctionInclude(settings),
+    mdFunctionChangelog(settings),
+  ]
+}
+
+let rehypeMermaid
+async function getMermaidPlugin() {
+  if (!rehypeMermaid) {
+    rehypeMermaid = (await import('rehype-mermaid')).default
+  }
+  return rehypeMermaid
+}
+
+export async function thirdPartyRehypePlugins(settings?: Settings) {
+  const plugins = [
+    [rehypeRaw, {
+      passThrough: ['mdxjsEsm', 'mdxJsxFlowElement', 'mdxJsxTextElement'],
+    }] as any,
+    rehypeKatex,
+  ]
+
+  if (settings?.integrations?.diagrams) {
+    plugins.push(await getMermaidPlugin())
+  }
+
+  return plugins
+}
+
+export async function defaultRehypePlugins(settings?: Settings) {
+  return [
+    ...(await thirdPartyRehypePlugins(settings)),
+    rehypeHeading,
+    mdImageRehype,
+    mdCodeRehype(settings),
+  ]
+}
+
+export function defaultRecmaPlugins(settings?: Settings) {
+  return [
+    recmaOverrideComponents,
+  ]
+}

package/packages/md/plugins/mdCode.ts

@@ -0,0 +1,16 @@
+import { visit } from "unist-util-visit";
+import { injectCodeMeta } from "./utils/injectCodeMeta";
+
+/**
+ * This plugin injects the code meta into the code node's data
+ * so that it can be used in the code block component
+ */
+export function remarkInjectCodeMeta() {
+    return (tree: any) => {
+        console.time('plugin:remarkInjectCodeMeta');
+        visit(tree, 'code', (node) => {
+            injectCodeMeta(node, node.meta);
+        });
+        console.timeEnd('plugin:remarkInjectCodeMeta');
+    };
+}

package/packages/md/plugins/mdHeadingId.ts

@@ -0,0 +1,47 @@
+import { visit } from 'unist-util-visit';
+import GithubSlugger from 'github-slugger';
+import type { Plugin } from 'unified';
+import type { Root, Heading } from 'mdast';
+
+import {mdParameters} from "./utils/mdParameters"
+
+interface HeadingData {
+  hProperties?: {
+    id?: string;
+    [key: string]: unknown;
+  };
+}
+
+export const mdHeadingId: Plugin<[], Root> = () => {
+  let slugger = new GithubSlugger();
+
+  return (tree) => {
+    visit(tree, 'heading', (node: Heading & { data?: HeadingData }) => {
+      if (!node.data) {
+        node.data = {
+          hProperties: {}
+        };
+      }
+
+      if (node.depth === 1) {
+        slugger.reset();
+      }
+
+      // Create a slug from the heading text
+      const text = node.children
+        .map((child) => ('value' in child ? child.value : ''))
+        .join('');
+
+      
+      const {props, sanitizedText} = mdParameters(text)
+
+      const id = props.id || slugger.slug(sanitizedText);
+
+      // Add the id to the heading's data
+      node.data.hProperties = {
+        ...node.data.hProperties,
+        id,
+      };
+    });
+  };
+}; 

package/packages/md/plugins/mdImage.test.ts

@@ -0,0 +1,59 @@
+import {unified} from 'unified';
+import remarkParse from 'remark-parse';
+import {visit} from 'unist-util-visit';
+import {describe, it, expect} from 'vitest';
+
+import {mdImage} from './mdImage';
+
+describe('mdImage', () => {
+    it('should parse caption attribute from image and attach to hProperties', async () => {
+        const md = '![Alt text](/img.png){caption="Hello world"}';
+        const processor = unified()
+            .use(remarkParse)
+            .use(mdImage);
+        const tree = processor.parse(md);
+        await processor.run(tree);
+        let found = false;
+        visit(tree, 'image', (node: any) => {
+            found = true;
+            expect(node.url).toBe('/img.png');
+            expect(node.alt).toBe('Alt text');
+            expect(node.data).toBeDefined();
+            expect(node.data.hProperties).toBeDefined();
+            expect(node.data.hProperties.caption).toBe('Hello world');
+        });
+        expect(found).toBe(true);
+    });
+
+    it('should work with multiple attributes', async () => {
+        const md = '![Alt text](/img.png){caption="Hello world" style="border:1px solid red"}';
+        const processor = unified()
+            .use(remarkParse)
+            .use(mdImage);
+        const tree = processor.parse(md);
+        await processor.run(tree);
+        let found = false;
+        visit(tree, 'image', (node: any) => {
+            found = true;
+            expect(node.data.hProperties.caption).toBe('Hello world');
+            expect(node.data.hProperties.style).toBe('border:1px solid red');
+        });
+        expect(found).toBe(true);
+    });
+
+    it('should not fail if no attributes are present', async () => {
+        const md = '![Alt text](/img.png)';
+        const processor = unified()
+            .use(remarkParse)
+            .use(mdImage);
+        const tree = processor.parse(md);
+        await processor.run(tree);
+        let found = false;
+        visit(tree, 'image', (node: any) => {
+            found = true;
+            expect(node.alt).toBe('Alt text');
+            expect(node.data?.hProperties?.caption).toBeUndefined();
+        });
+        expect(found).toBe(true);
+    });
+});