@xyd-js/content 0.1.0-xyd.13 → 0.1.0-xyd.16

package/packages/md/plugins/composer/mdComposer.ts

@@ -0,0 +1,146 @@
+import { Plugin } from "unified";
+import { visit } from "unist-util-visit";
+import { Node as UnistNode } from "unist";
+import { highlight } from "codehike/code";
+import type { HighlightedCode, Token } from "codehike/code";
+
+import { Settings } from "@xyd-js/core";
+
+import { SymbolxVfile } from "../types";
+
+type Whitespace = string;
+
+// Define types for our examples structure
+export interface ExampleCode {
+    language: string;
+    title?: string;
+    tokens: (Token | Whitespace)[];
+}
+
+// TODO: ISSUES IF MULTIPLE OUTPUT VARS - PROBLEMS WITH ASYNC + currentQueue/currentVarName
+// Main plugin function
+export function mdComposer(settings?: Settings): Plugin {
+    return () => async (tree: UnistNode, file: SymbolxVfile<any>) => {
+        console.time('plugin:mdComposer');
+        const varQueue: Record<string, any> = {}
+
+        // Create a context object to store the current state for each variable
+        const contexts: Record<string, {
+            queue: (HighlightedCode | any[])[];
+            promises: Promise<any>[];
+        }> = {}
+
+        visit(tree, "outputVars", (node: any) => {
+            const varName = node.name
+            varQueue[varName] = []
+            contexts[varName] = {
+                queue: varQueue[varName],
+                promises: []
+            }
+
+            throughNodes(node, contexts[varName])
+        })
+
+        visit(tree, 'containerDirective', (node: any) => {
+            if (node.name !== "code-group") {
+                return
+            }
+
+            // Find the most recently created context
+            const lastVarName = Object.keys(contexts).pop()
+            if (!lastVarName) return
+
+            const context = contexts[lastVarName]
+            const group = [node.attributes.title]
+            context.queue.push(group)
+
+            if (node.data?.hName === "DirectiveCodeGroup") {
+                const codeblocksJSON = node.data.hProperties.codeblocks
+
+                if (codeblocksJSON) {
+                    const codeblocks = JSON.parse(codeblocksJSON)
+
+                    if (!codeblocks || !codeblocks.length) {
+                        return
+                    }
+
+                    for (const codeblock of codeblocks) {
+                        group.push(codeblock)
+                    }
+                }
+
+                return
+            }
+
+            for (const child of node.children) {
+                if (child.type !== 'code') {
+                    continue
+                }
+
+                highlightCode(child, group, context)
+            }
+        });
+
+        async function highlightCode(node: any, group?: any[], context?: typeof contexts[string]) {
+            const highlighted = await highlight({
+                value: node.value,
+                lang: node.lang,
+                meta: node.meta,
+            }, settings?.theme?.coder?.syntaxHighlight || "github-dark")
+
+            if (group && context) {
+                group.push(highlighted)
+                return
+            }
+
+            if (context) {
+                context.queue.push(highlighted)
+            }
+        }
+
+        function nodeCode(node: any, context: typeof contexts[string]) {
+            context.promises.push(highlightCode(node, undefined, context))
+        }
+
+        function nodeList(node: any, context: typeof contexts[string]) {
+            for (const item of node.children) {
+                if (item.type != "listItem") {
+                    continue
+                }
+
+                throughNodes(item, context)
+            }
+        }
+
+        function throughNodes(node: any, context: typeof contexts[string]) {
+            for (const child of node.children) {
+                switch (child.type) {
+                    case "code": {
+                        nodeCode(child, context)
+                        break
+                    }
+                    case "list": {
+                        nodeList(child, context)
+                        break
+                    }
+                }
+            }
+        }
+
+        // Wait for all promises from all contexts to resolve
+        await Promise.all(Object.values(contexts).flatMap(ctx => ctx.promises))
+        console.timeEnd('plugin:mdComposer');
+
+        // Process the results to create the final output structure
+        const outputVars = {}
+
+        for (const [varName, value] of Object.entries(varQueue)) {
+            outputVars[varName] = value
+        }
+
+        file.data.outputVars = {
+            ...(file.data.outputVars || {}),
+            ...outputVars
+        }
+    };
+}

package/packages/md/plugins/developer-writing/index.ts

@@ -0,0 +1,3 @@
+export {
+    mdCodeRehype
+} from "./mdCodeRehype"

package/packages/md/plugins/developer-writing/mdCodeRehype.ts

@@ -0,0 +1,78 @@
+import { visit } from "unist-util-visit";
+import { highlight } from "codehike/code"
+
+import { Settings } from "@xyd-js/core";
+import { ContentFS } from "../../../../src";
+import { defaultRemarkPlugins } from "..";
+
+export function mdCodeRehype(settings?: Settings) {
+    const maxTocDepth = settings?.theme?.maxTocDepth || 2
+    const remarkPlugins = [...defaultRemarkPlugins({
+        maxDepth: maxTocDepth,
+    }, settings)]
+
+    const contentFs = new ContentFS(settings || {}, remarkPlugins, [])
+
+    return function () {
+        return async (tree: any) => {
+            console.time('plugin:mdServerHighlight');
+            const promises: Promise<any>[] = []
+
+            visit(tree, 'element', (node) => {
+                if (node.tagName === 'pre') {
+                    const code = node.children[0].children[0].value
+                    const lang = node.children?.[0]?.properties?.className?.[0]?.replace("language-", "")
+
+                    // const regions = node.children?.[0]?.properties?.regions // TODO: in the future
+                    const lineRanges = (node.children?.[0]?.properties?.lineRanges
+                        ? JSON.parse(node.children?.[0]?.properties?.lineRanges)
+                        : []) as { start?: number, end?: number }[]
+
+                    const lineNumbers = node.children?.[0]?.properties?.lineNumbers
+                    const size = node.children?.[0]?.properties?.size
+                    const descriptionHead = node.children?.[0]?.properties?.descriptionHead
+                    const descriptionContent = node.children?.[0]?.properties?.descriptionContent
+                    const descriptionIcon = node.children?.[0]?.properties?.descriptionIcon
+
+                    const promise = (async () => {
+                        let descriptionContentCode = ""
+
+                        if (descriptionContent) {
+                            descriptionContentCode = await contentFs.compileContent(descriptionContent)
+                        }
+
+                        const highlighted = await highlight({
+                            value: code,
+                            lang: lang,
+                            meta: lang || "",
+                        }, settings?.theme?.coder?.syntaxHighlight || "github-dark")
+
+                        if (lineRanges && lineRanges.length) {
+                            highlighted.annotations = lineRanges.map((range) => ({
+                                name: "mark",
+                                query: "",
+                                fromLineNumber: range.start || 0,
+                                toLineNumber: range.end || 0,
+                            }))
+                        }
+
+                        node.properties = {
+                            ...node.properties,
+                            highlighted: JSON.stringify(highlighted),
+                            lineNumbers,
+                            size,
+                            descriptionHead,
+                            descriptionContent: descriptionContentCode,
+                            descriptionIcon
+                        };
+                    })()
+
+                    promises.push(promise)
+                }
+            });
+
+            await Promise.all(promises)
+            console.timeEnd('plugin:mdServerHighlight');
+        };
+    }
+}

package/packages/md/plugins/functions/__fixtures__/external.ts

@@ -0,0 +1,4 @@
+// This is a mock external file for testing
+export function externalFunction() { 
+  return "Hello from external"; 
+} 

package/packages/md/plugins/functions/__fixtures__/test.js

@@ -0,0 +1,11 @@
+// This is a JavaScript test file
+export function testFunction() {
+  return 'Hello, world!';
+}
+
+// This is a region for testing
+// #region testRegion
+export function regionFunction() {
+  return 'This is from a region';
+}
+// #endregion testRegion 

package/packages/md/plugins/functions/__fixtures__/test.py

@@ -0,0 +1,9 @@
+# This is a Python test file
+def test_function():
+    return "Hello, world!"
+
+# This is a region for testing
+# #region testRegion
+def region_function():
+    return "This is from a region"
+# #endregion testRegion 

package/packages/md/plugins/functions/__fixtures__/test.ts

@@ -0,0 +1,18 @@
+// This is a test file for the mdFunctionImport plugin
+export function testFunction(): string {
+  return 'Hello, world!';
+}
+
+// This is a region for testing
+// #region testRegion
+export function regionFunction(): string {
+  return 'This is from a region';
+}
+// #endregion testRegion
+
+// This is another region for testing
+// #region anotherRegion
+export function anotherRegionFunction(): string {
+  return 'This is from another region';
+}
+// #endregion anotherRegion 

package/packages/md/plugins/functions/__tests__/mdFunctionImportCode.test.ts

@@ -0,0 +1,295 @@
+import path from 'node:path';
+import fs from 'node:fs';
+
+import { describe, it, expect } from 'vitest';
+
+import { mdFunctionImportCode } from '../mdFunctionImportCode';
+import { parseImportPath, processContent, detectLanguage } from '../utils';
+
+import { createMockTree, createMockFile, createMockFetch, restoreFetch, createMockTreeAlternativeSyntax } from './testHelpers';
+
+// Get the absolute path to the fixtures directory
+const fixturesDir = path.resolve(__dirname, '../__fixtures__');
+
+describe('mdFunctionImportCode', () => {
+    describe('parseImportPath', () => {
+        it('should parse a simple file path', () => {
+            const result = parseImportPath('path/to/file.ts');
+            expect(result).toEqual({
+                filePath: 'path/to/file.ts',
+                regions: [],
+                lineRanges: []
+            });
+        });
+
+        it('should parse a file path with regions', () => {
+            const result = parseImportPath('path/to/file.ts#region1,region2');
+            expect(result).toEqual({
+                filePath: 'path/to/file.ts',
+                regions: [
+                    { name: 'region1' },
+                    { name: 'region2' }
+                ],
+                lineRanges: []
+            });
+        });
+
+
+        it('relative parent folder', () => {
+            const result = parseImportPath('../relative/parents#Theme');
+            expect(result).toEqual({
+                filePath: '../relative/parents',
+                regions: [
+                    { name: 'Theme' }
+                ],
+                lineRanges: []
+            });
+        });
+
+        it('should handle curly braces in region names for openapi like', () => {
+            const result = parseImportPath('~/api/rest/openapi.yaml#PUT /alerts/{id}');
+            expect(result).toEqual({
+                filePath: '~/api/rest/openapi.yaml',
+                regions: [
+                    { name: 'PUT /alerts/{id}' }
+                ],
+                lineRanges: []
+            });
+        });
+
+        it('should parse a file path with line ranges', () => {
+            const result = parseImportPath('path/to/file.ts{1,2-4,8:,:10}');
+            expect(result).toEqual({
+                filePath: 'path/to/file.ts',
+                regions: [],
+                lineRanges: [
+                    { start: 1, end: 1 },
+                    { start: 2, end: 4 },
+                    { start: 8 },
+                    { end: 10 }
+                ]
+            });
+        });
+
+        it('should parse a file path with both regions and line ranges', () => {
+            const result = parseImportPath('path/to/file.ts#region1{1-5}');
+            expect(result).toEqual({
+                filePath: 'path/to/file.ts',
+                regions: [
+                    { name: 'region1' }
+                ],
+                lineRanges: [
+                    { start: 1, end: 5 }
+                ]
+            });
+        });
+    });
+
+    describe('processContent', () => {
+        const sampleContent = `line1
+// #region test
+line2
+line3
+// #endregion test
+line4
+line5
+// #region another
+line6
+// #endregion another
+line7
+line8
+line9
+`;
+
+        it('should return the original content when no regions or line ranges are specified', () => {
+            const result = processContent(sampleContent, [], []);
+            expect(result).toBe(sampleContent);
+        });
+
+        it('should extract content from specified regions', () => {
+            const result = processContent(sampleContent, [{ name: 'test' }], []);
+            expect(result).toBe('line2\nline3');
+        });
+
+        it('should extract content from multiple regions', () => {
+            const result = processContent(sampleContent, [{ name: 'test' }, { name: 'another' }], []);
+            expect(result).toBe('line2\nline3\nline6');
+        });
+
+        it('should extract content based on line ranges', () => {
+            const result = processContent(sampleContent, [], [{ start: 3, end: 4 }]);
+            expect(result).toBe('line2\nline3');
+        });
+
+        it('should handle multiple line ranges', () => {
+            const result = processContent(sampleContent, [], [
+                { start: 1, end: 1 },
+                { start: 7, end: 7 }
+            ]);
+            expect(result).toBe('line1\nline5');
+        });
+
+        it('should handle open-ended ranges', () => {
+            const result = processContent(sampleContent, [], [{ start: 11 }]);
+            expect(result).toBe('line7\nline8\nline9\n');
+        });
+
+        it('should handle ranges from start', () => {
+            const result = processContent(sampleContent, [], [{ end: 1 }]);
+            expect(result).toBe('line1');
+        });
+    });
+
+    describe('detectLanguage', () => {
+        it('should detect JavaScript language', () => {
+            expect(detectLanguage('file.js')).toBe('javascript');
+        });
+
+        it('should detect TypeScript language', () => {
+            expect(detectLanguage('file.ts')).toBe('typescript');
+        });
+
+        it('should detect Python language', () => {
+            expect(detectLanguage('file.py')).toBe('python');
+        });
+
+        it('should return empty string for unknown extension', () => {
+            expect(detectLanguage('file.xyz')).toBe('xyz');
+        });
+
+        it('should handle uppercase extensions', () => {
+            expect(detectLanguage('file.TS')).toBe('typescript');
+        });
+    });
+
+    describe('transformer', () => {
+        it('should transform @importCode statements to code blocks', async () => {
+            const transformer = mdFunctionImportCode()();
+            const tree = createMockTree('test.ts');
+            const file = createMockFile();
+
+            await transformer(tree, file);
+
+            expect(tree.children[0].type).toBe('code');
+            expect((tree.children[0] as any).lang).toBe('typescript');
+            expect((tree.children[0] as any).value).toContain('export function testFunction');
+        });
+
+        it('should handle non-existent files gracefully', async () => {
+            const transformer = mdFunctionImportCode()();
+            const tree = createMockTree('nonexistent.ts');
+            const file = createMockFile();
+
+            // Mock console.error to prevent test output pollution
+            const originalConsoleError = console.error;
+            console.error = () => {
+            };
+
+            await transformer(tree, file);
+
+            // Restore console.error
+            console.error = originalConsoleError;
+
+            // The node should remain unchanged
+            expect(tree.children[0].type).toBe('paragraph');
+        });
+
+        it('should handle custom resolveFrom option', async () => {
+            const customFixturesDir = path.resolve(__dirname, '../__fixtures__');
+            const transformer = mdFunctionImportCode()({ resolveFrom: customFixturesDir });
+            const tree = createMockTree('test.ts');
+            const file = createMockFile('/some/other/directory');
+
+            await transformer(tree, file);
+
+            expect(tree.children[0].type).toBe('code');
+            expect((tree.children[0] as any).lang).toBe('typescript');
+            expect((tree.children[0] as any).value).toContain('export function testFunction');
+        });
+
+        it('should handle complex import paths with regions and line ranges', async () => {
+            const transformer = mdFunctionImportCode()()
+            const tree = createMockTree('test.ts#testRegion{1-3}');
+            const file = createMockFile();
+
+            await transformer(tree, file);
+
+            expect(tree.children[0].type).toBe('code');
+            expect((tree.children[0] as any).lang).toBe('typescript');
+            expect((tree.children[0] as any).value).toContain('export function regionFunction');
+        });
+
+        it('should handle import paths with only regions', async () => {
+            const transformer = mdFunctionImportCode()()
+            const tree = createMockTree('test.ts#testRegion');
+            const file = createMockFile();
+
+            await transformer(tree, file);
+
+            expect(tree.children[0].type).toBe('code');
+            expect((tree.children[0] as any).lang).toBe('typescript');
+            expect((tree.children[0] as any).value).toContain('export function regionFunction');
+        });
+
+        it('should handle import paths with only line ranges', async () => {
+            const transformer = mdFunctionImportCode()()
+            const tree = createMockTree('test.ts{1-3}');
+            const file = createMockFile();
+
+            await transformer(tree, file);
+
+            expect(tree.children[0].type).toBe('code');
+            expect((tree.children[0] as any).lang).toBe('typescript');
+            expect((tree.children[0] as any).value).toContain('export function testFunction');
+        });
+
+        it('should handle external URLs', async () => {
+            // Create mock fetch with our fixture content
+            const externalUrl = 'https://example.com/test.ts';
+            const fixtureContent = fs.readFileSync(path.resolve(fixturesDir, 'external.ts'), 'utf8');
+            const originalFetch = createMockFetch({
+                [externalUrl]: fixtureContent
+            });
+
+            const transformer = mdFunctionImportCode()()
+            const tree = createMockTree(externalUrl);
+            const file = createMockFile();
+
+            await transformer(tree, file);
+
+            // Restore original fetch
+            restoreFetch(originalFetch);
+
+            expect(tree.children[0].type).toBe('code');
+            expect((tree.children[0] as any).lang).toBe('typescript');
+            expect((tree.children[0] as any).value).toContain('export function externalFunction');
+        });
+
+        it('should debug visit function', async () => {
+            const transformer = mdFunctionImportCode()()
+            const tree = createMockTree('test.ts');
+
+            const file = createMockFile();
+
+            // Create a promise that resolves when the transformer is done
+            const transformPromise = transformer(tree, file);
+
+            // Wait for the promise to resolve
+            await transformPromise;
+
+            expect(tree.children[0].type).toBe('code');
+        });
+
+        it('should handle alternative syntax like @importCode("test.ts")', async () => {
+            const transformer = mdFunctionImportCode()()
+            const tree = createMockTreeAlternativeSyntax('test.ts');
+            const file = createMockFile();
+
+            await transformer(tree, file);
+
+            expect(tree.children[0].type).toBe('code');
+            expect((tree.children[0] as any).lang).toBe('typescript');
+            expect((tree.children[0] as any).value).toContain('export function testFunction');
+        });
+    });
+}); 

package/packages/md/plugins/functions/__tests__/parseFunctionCall.test.ts

@@ -0,0 +1,47 @@
+import {describe, it, expect} from 'vitest';
+
+import {parseFunctionCall} from '../utils';
+import {FunctionName} from "../types";
+
+describe('parseFunctionCall', () => {
+    const testCases = [
+        {
+            name: '1a. should parse uniform with file path and options',
+            input: `@uniform('~/path/to/resource.ts', {"mini": "Settings"})`,
+            expected: ['~/path/to/resource.ts', {mini: 'Settings'}]
+        },
+        {
+            name: '1b. should parse uniform with file path and options',
+            input: `@uniform('~/path/to/resource.ts', {mini: 'Settings'})`,
+            expected: ['~/path/to/resource.ts', {mini: 'Settings'}]
+        },
+        {
+            name: '2. should parse uniform with file path only',
+            input: '@uniform(~/path/to/resource.ts)',
+            expected: ['~/path/to/resource.ts']
+        },
+        {
+            name: '3a. should handle malformed uniform call',
+            input: '@uniform_invalid()',
+            expected: null
+        },
+        {
+            name: '3b. should handle malformed uniform call',
+            input: 'uniform()',
+            expected: null
+        }
+    ];
+
+    testCases.forEach(({name, input, expected}) => {
+        it(name, () => {
+            const node = {
+                children: [{
+                    type: 'text',
+                    value: input
+                }]
+            };
+            const result = parseFunctionCall(node, FunctionName.Uniform);
+            expect(result).toEqual(expected);
+        });
+    });
+});

package/packages/md/plugins/functions/__tests__/testHelpers.ts

@@ -0,0 +1,71 @@
+import path from 'node:path';
+import { VFile } from 'vfile';
+
+
+export function createMockTree(importPath: string) {
+    return {
+        type: 'root',
+        children: [
+            {
+                type: 'paragraph',
+                children: [
+                    {
+                        type: 'text',
+                        value: `@importCode "${importPath}"`
+                    }
+                ]
+            }
+        ]
+    };
+}
+
+export function createMockTreeAlternativeSyntax(importPath: string) {
+    return {
+        type: 'root',
+        children: [
+            {
+                type: 'paragraph',
+                children: [
+                    {
+                        type: 'text',
+                        value: `@importCode("${importPath}")`
+                    }
+                ]
+            }
+        ]
+    };
+}
+
+
+export function createMockFile(dirname?: string) {
+    return new VFile({
+        path: 'test.md',
+        value: '',
+        dirname: dirname || path.resolve(__dirname, '../__fixtures__')
+    });
+}
+
+
+export function createMockFetch(mockResponses: Record<string, string>) {
+    const originalFetch = global.fetch;
+
+    global.fetch = (async (input: RequestInfo | URL, init?: RequestInit): Promise<Response> => {
+        const url = input.toString();
+
+        if (mockResponses[url]) {
+            return {
+                ok: true,
+                text: async () => mockResponses[url]
+            } as Response;
+        }
+
+        throw new Error(`Unexpected URL: ${url}`);
+    }) as typeof fetch;
+
+    return originalFetch;
+}
+
+
+export function restoreFetch(originalFetch: typeof global.fetch) {
+    global.fetch = originalFetch;
+}