@xyd-js/content 0.0.0-build

package/packages/md/plugins/composer/mdComposer.ts

@@ -0,0 +1,146 @@
+import { Plugin } from "unified";
+import { visit } from "unist-util-visit";
+import { Node as UnistNode } from "unist";
+import { highlight } from "codehike/code";
+import type { HighlightedCode, Token } from "codehike/code";
+
+import { Settings } from "@xyd-js/core";
+
+import { SymbolxVfile } from "../types";
+
+type Whitespace = string;
+
+// Define types for our examples structure
+export interface ExampleCode {
+    language: string;
+    title?: string;
+    tokens: (Token | Whitespace)[];
+}
+
+// TODO: ISSUES IF MULTIPLE OUTPUT VARS - PROBLEMS WITH ASYNC + currentQueue/currentVarName
+// Main plugin function
+export function mdComposer(settings?: Settings): Plugin {
+    return () => async (tree: UnistNode, file: SymbolxVfile<any>) => {
+        console.time('plugin:mdComposer');
+        const varQueue: Record<string, any> = {}
+
+        // Create a context object to store the current state for each variable
+        const contexts: Record<string, {
+            queue: (HighlightedCode | any[])[];
+            promises: Promise<any>[];
+        }> = {}
+
+        visit(tree, "outputVars", (node: any) => {
+            const varName = node.name
+            varQueue[varName] = []
+            contexts[varName] = {
+                queue: varQueue[varName],
+                promises: []
+            }
+
+            throughNodes(node, contexts[varName])
+        })
+
+        visit(tree, 'containerDirective', (node: any) => {
+            if (node.name !== "code-group") {
+                return
+            }
+
+            // Find the most recently created context
+            const lastVarName = Object.keys(contexts).pop()
+            if (!lastVarName) return
+
+            const context = contexts[lastVarName]
+            const group = [node.attributes.title]
+            context.queue.push(group)
+
+            if (node.data?.hName === "DirectiveCodeGroupFINISH") {
+                const codeblocksJSON = node.data.hProperties.codeblocks
+
+                if (codeblocksJSON) {
+                    const codeblocks = JSON.parse(codeblocksJSON)
+
+                    if (!codeblocks || !codeblocks.length) {
+                        return
+                    }
+
+                    for (const codeblock of codeblocks) {
+                        group.push(codeblock)
+                    }
+                }
+
+                return
+            }
+
+            for (const child of node.children) {
+                if (child.type !== 'code') {
+                    continue
+                }
+
+                highlightCode(child, group, context)
+            }
+        });
+
+        async function highlightCode(node: any, group?: any[], context?: typeof contexts[string]) {
+            const highlighted = await highlight({
+                value: node.value,
+                lang: node.lang,
+                meta: node.meta,
+            }, settings?.theme?.coder?.syntaxHighlight || "github-dark")
+
+            if (group && context) {
+                group.push(highlighted)
+                return
+            }
+
+            if (context) {
+                context.queue.push(highlighted)
+            }
+        }
+
+        function nodeCode(node: any, context: typeof contexts[string]) {
+            context.promises.push(highlightCode(node, undefined, context))
+        }
+
+        function nodeList(node: any, context: typeof contexts[string]) {
+            for (const item of node.children) {
+                if (item.type != "listItem") {
+                    continue
+                }
+
+                throughNodes(item, context)
+            }
+        }
+
+        function throughNodes(node: any, context: typeof contexts[string]) {
+            for (const child of node.children) {
+                switch (child.type) {
+                    case "code": {
+                        nodeCode(child, context)
+                        break
+                    }
+                    case "list": {
+                        nodeList(child, context)
+                        break
+                    }
+                }
+            }
+        }
+
+        // Wait for all promises from all contexts to resolve
+        await Promise.all(Object.values(contexts).flatMap(ctx => ctx.promises))
+        console.timeEnd('plugin:mdComposer');
+
+        // Process the results to create the final output structure
+        const outputVars = {}
+
+        for (const [varName, value] of Object.entries(varQueue)) {
+            outputVars[varName] = value
+        }
+
+        file.data.outputVars = {
+            ...(file.data.outputVars || {}),
+            ...outputVars
+        }
+    };
+}

package/packages/md/plugins/developer-writing/index.ts

@@ -0,0 +1,3 @@
+export {
+    mdCodeRehype
+} from "./mdCodeRehype"

package/packages/md/plugins/developer-writing/mdCodeRehype.ts

@@ -0,0 +1,81 @@
+import { visit } from "unist-util-visit";
+import { highlight } from "codehike/code"
+
+import { Settings } from "@xyd-js/core";
+import { ContentFS } from "../../../../src";
+import { defaultRemarkPlugins } from "..";
+
+export function mdCodeRehype(settings?: Settings) {
+    const maxTocDepth = settings?.theme?.writer?.maxTocDepth || 2
+    const remarkPlugins = [...defaultRemarkPlugins({
+        maxDepth: maxTocDepth,
+    }, settings)]
+
+    const contentFs = new ContentFS(settings || {}, remarkPlugins, [])
+
+    return function () {
+        return async (tree: any) => {
+            console.time('plugin:mdServerHighlight');
+            const promises: Promise<any>[] = []
+
+            visit(tree, 'element', (node) => {
+                if (node.tagName === 'pre') {
+                    const code = node.children[0].children[0].value
+                    const lang = node.children?.[0]?.properties?.className?.[0]?.replace("language-", "")
+
+                    // const regions = node.children?.[0]?.properties?.regions // TODO: in the future
+                    const lineRanges = (node.children?.[0]?.properties?.lineRanges
+                        ? JSON.parse(node.children?.[0]?.properties?.lineRanges)
+                        : []) as { start?: number, end?: number }[]
+
+                    const lineNumbers = node.children?.[0]?.properties?.lineNumbers
+                    const size = node.children?.[0]?.properties?.size
+                    const descriptionHead = node.children?.[0]?.properties?.descriptionHead
+                    const descriptionContent = node.children?.[0]?.properties?.descriptionContent
+                    const descriptionIcon = node.children?.[0]?.properties?.descriptionIcon
+                    const meta = node.children?.[0]?.properties?.meta
+                    const title = node.children?.[0]?.properties?.title
+
+                    const promise = (async () => {
+                        let descriptionContentCode = ""
+
+                        if (descriptionContent) {
+                            descriptionContentCode = await contentFs.compileContent(descriptionContent)
+                        }
+
+                        const highlighted = await highlight({
+                            value: code,
+                            lang: lang,
+                            meta: meta || lang || "",
+                        }, settings?.theme?.coder?.syntaxHighlight || "github-dark")
+
+                        if (lineRanges && lineRanges.length) {
+                            highlighted.annotations = lineRanges.map((range) => ({
+                                name: "mark",
+                                query: "",
+                                fromLineNumber: range.start || 0,
+                                toLineNumber: range.end || 0,
+                            }))
+                        }
+
+                        node.properties = {
+                            ...node.properties,
+                            title,
+                            highlighted: JSON.stringify(highlighted),
+                            lineNumbers,
+                            size,
+                            descriptionHead,
+                            descriptionContent: descriptionContentCode,
+                            descriptionIcon
+                        };
+                    })()
+
+                    promises.push(promise)
+                }
+            });
+
+            await Promise.all(promises)
+            console.timeEnd('plugin:mdServerHighlight');
+        };
+    }
+}

package/packages/md/plugins/functions/__fixtures__/external.ts

@@ -0,0 +1,4 @@
+// This is a mock external file for testing
+export function externalFunction() { 
+  return "Hello from external"; 
+} 

package/packages/md/plugins/functions/__fixtures__/test-include.md

@@ -0,0 +1,31 @@
+# Included Content
+
+This is some content that will be included by the `@include` function.
+
+## Features
+
+- **Bold text** for emphasis
+- *Italic text* for subtle emphasis
+- `Code snippets` for technical content
+- [Links](https://example.com) for references
+
+## Code Example
+
+```typescript
+function includedFunction() {
+    return "This function is included from another file";
+}
+```
+
+## Lists
+
+1. First item
+2. Second item
+3. Third item
+
+- Unordered item
+- Another unordered item
+
+:::callout{kind="warning"}
+Hello World
+:::

package/packages/md/plugins/functions/__fixtures__/test.js

@@ -0,0 +1,11 @@
+// This is a JavaScript test file
+export function testFunction() {
+  return 'Hello, world!';
+}
+
+// This is a region for testing
+// #region testRegion
+export function regionFunction() {
+  return 'This is from a region';
+}
+// #endregion testRegion 

package/packages/md/plugins/functions/__fixtures__/test.py

@@ -0,0 +1,9 @@
+# This is a Python test file
+def test_function():
+    return "Hello, world!"
+
+# This is a region for testing
+# #region testRegion
+def region_function():
+    return "This is from a region"
+# #endregion testRegion 

package/packages/md/plugins/functions/__fixtures__/test.ts

@@ -0,0 +1,18 @@
+// This is a test file for the mdFunctionImport plugin
+export function testFunction(): string {
+  return 'Hello, world!';
+}
+
+// This is a region for testing
+// #region testRegion
+export function regionFunction(): string {
+  return 'This is from a region';
+}
+// #endregion testRegion
+
+// This is another region for testing
+// #region anotherRegion
+export function anotherRegionFunction(): string {
+  return 'This is from another region';
+}
+// #endregion anotherRegion 

package/packages/md/plugins/functions/__tests__/mdFunctionImportCode.test.ts

@@ -0,0 +1,314 @@
+import path from 'node:path';
+import fs from 'node:fs';
+
+import { describe, it, expect } from 'vitest';
+
+import { mdFunctionImportCode } from '../mdFunctionImportCode';
+import { parseImportPath, processContent, detectLanguage } from '../utils';
+
+import {
+    importCodeMockTree,
+    createMockFile,
+    createMockFetch,
+    restoreFetch,
+    importCodeMockTreeAlternative,
+    importCodeMdAttributesMockTree
+} from './testHelpers';
+
+// Get the absolute path to the fixtures directory
+const fixturesDir = path.resolve(__dirname, '../__fixtures__');
+
+describe('mdFunctionImportCode', () => {
+    describe('parseImportPath', () => {
+        it('should parse a simple file path', () => {
+            const result = parseImportPath('path/to/file.ts');
+            expect(result).toEqual({
+                filePath: 'path/to/file.ts',
+                regions: [],
+                lineRanges: []
+            });
+        });
+
+        it('should parse a file path with regions', () => {
+            const result = parseImportPath('path/to/file.ts#region1,region2');
+            expect(result).toEqual({
+                filePath: 'path/to/file.ts',
+                regions: [
+                    { name: 'region1' },
+                    { name: 'region2' }
+                ],
+                lineRanges: []
+            });
+        });
+
+
+        it('relative parent folder', () => {
+            const result = parseImportPath('../relative/parents#Theme');
+            expect(result).toEqual({
+                filePath: '../relative/parents',
+                regions: [
+                    { name: 'Theme' }
+                ],
+                lineRanges: []
+            });
+        });
+
+        it('should handle curly braces in region names for openapi like', () => {
+            const result = parseImportPath('~/api/rest/openapi.yaml#PUT /alerts/{id}');
+            expect(result).toEqual({
+                filePath: '~/api/rest/openapi.yaml',
+                regions: [
+                    { name: 'PUT /alerts/{id}' }
+                ],
+                lineRanges: []
+            });
+        });
+
+        it('should parse a file path with line ranges', () => {
+            const result = parseImportPath('path/to/file.ts{1,2-4,8:,:10}');
+            expect(result).toEqual({
+                filePath: 'path/to/file.ts',
+                regions: [],
+                lineRanges: [
+                    { start: 1, end: 1 },
+                    { start: 2, end: 4 },
+                    { start: 8 },
+                    { end: 10 }
+                ]
+            });
+        });
+
+        it('should parse a file path with both regions and line ranges', () => {
+            const result = parseImportPath('path/to/file.ts#region1{1-5}');
+            expect(result).toEqual({
+                filePath: 'path/to/file.ts',
+                regions: [
+                    { name: 'region1' }
+                ],
+                lineRanges: [
+                    { start: 1, end: 5 }
+                ]
+            });
+        });
+    });
+
+    describe('processContent', () => {
+        const sampleContent = `line1
+// #region test
+line2
+line3
+// #endregion test
+line4
+line5
+// #region another
+line6
+// #endregion another
+line7
+line8
+line9
+`;
+
+        it('should return the original content when no regions or line ranges are specified', () => {
+            const result = processContent(sampleContent, [], []);
+            expect(result).toBe(sampleContent);
+        });
+
+        it('should extract content from specified regions', () => {
+            const result = processContent(sampleContent, [{ name: 'test' }], []);
+            expect(result).toBe('line2\nline3');
+        });
+
+        it('should extract content from multiple regions', () => {
+            const result = processContent(sampleContent, [{ name: 'test' }, { name: 'another' }], []);
+            expect(result).toBe('line2\nline3\nline6');
+        });
+
+        it('should extract content based on line ranges', () => {
+            const result = processContent(sampleContent, [], [{ start: 3, end: 4 }]);
+            expect(result).toBe('line2\nline3');
+        });
+
+        it('should handle multiple line ranges', () => {
+            const result = processContent(sampleContent, [], [
+                { start: 1, end: 1 },
+                { start: 7, end: 7 }
+            ]);
+            expect(result).toBe('line1\nline5');
+        });
+
+        it('should handle open-ended ranges', () => {
+            const result = processContent(sampleContent, [], [{ start: 11 }]);
+            expect(result).toBe('line7\nline8\nline9\n');
+        });
+
+        it('should handle ranges from start', () => {
+            const result = processContent(sampleContent, [], [{ end: 1 }]);
+            expect(result).toBe('line1');
+        });
+    });
+
+    describe('detectLanguage', () => {
+        it('should detect JavaScript language', () => {
+            expect(detectLanguage('file.js')).toBe('javascript');
+        });
+
+        it('should detect TypeScript language', () => {
+            expect(detectLanguage('file.ts')).toBe('typescript');
+        });
+
+        it('should detect Python language', () => {
+            expect(detectLanguage('file.py')).toBe('python');
+        });
+
+        it('should return empty string for unknown extension', () => {
+            expect(detectLanguage('file.xyz')).toBe('xyz');
+        });
+
+        it('should handle uppercase extensions', () => {
+            expect(detectLanguage('file.TS')).toBe('typescript');
+        });
+    });
+
+    describe('function call', () => {
+        it('should call @importCode statements to code blocks', async () => {
+            const fnCall = mdFunctionImportCode()();
+            const tree = importCodeMockTree('test.ts');
+            const file = createMockFile();
+
+            await fnCall(tree, file);
+
+            expect(tree.children[0].type).toBe('code');
+            expect((tree.children[0] as any).lang).toBe('typescript');
+            expect((tree.children[0] as any).value).toContain('export function testFunction');
+        });
+
+        it('should handle non-existent files gracefully', async () => {
+            const fnCall = mdFunctionImportCode()();
+            const tree = importCodeMockTree('nonexistent.ts');
+            const file = createMockFile();
+
+            // Mock console.error to prevent test output pollution
+            const originalConsoleError = console.error;
+            console.error = () => {
+            };
+
+            await fnCall(tree, file);
+
+            // Restore console.error
+            console.error = originalConsoleError;
+
+            // The node should remain unchanged
+            expect(tree.children[0].type).toBe('paragraph');
+        });
+
+        it('should handle custom resolveFrom option', async () => {
+            const customFixturesDir = path.resolve(__dirname, '../__fixtures__');
+            const fnCall = mdFunctionImportCode()({ resolveFrom: customFixturesDir });
+            const tree = importCodeMockTree('test.ts');
+            const file = createMockFile('/some/other/directory');
+
+            await fnCall(tree, file);
+
+            expect(tree.children[0].type).toBe('code');
+            expect((tree.children[0] as any).lang).toBe('typescript');
+            expect((tree.children[0] as any).value).toContain('export function testFunction');
+        });
+
+        it('should handle complex import paths with regions and line ranges', async () => {
+            const fnCall = mdFunctionImportCode()()
+            const tree = importCodeMockTree('test.ts#testRegion{1-3}');
+            const file = createMockFile();
+
+            await fnCall(tree, file);
+
+            expect(tree.children[0].type).toBe('code');
+            expect((tree.children[0] as any).lang).toBe('typescript');
+            expect((tree.children[0] as any).value).toContain('export function regionFunction');
+        });
+
+        it('should handle import paths with only regions', async () => {
+            const fnCall = mdFunctionImportCode()()
+            const tree = importCodeMockTree('test.ts#testRegion');
+            const file = createMockFile();
+
+            await fnCall(tree, file);
+
+            expect(tree.children[0].type).toBe('code');
+            expect((tree.children[0] as any).lang).toBe('typescript');
+            expect((tree.children[0] as any).value).toContain('export function regionFunction');
+        });
+
+        it('should handle import paths with only line ranges', async () => {
+            const fnCall = mdFunctionImportCode()()
+            const tree = importCodeMockTree('test.ts{1-3}');
+            const file = createMockFile();
+
+            await fnCall(tree, file);
+
+            expect(tree.children[0].type).toBe('code');
+            expect((tree.children[0] as any).lang).toBe('typescript');
+            expect((tree.children[0] as any).value).toContain('export function testFunction');
+        });
+
+        it('should handle external URLs', async () => {
+            // Create mock fetch with our fixture content
+            const externalUrl = 'https://example.com/test.ts';
+            const fixtureContent = fs.readFileSync(path.resolve(fixturesDir, 'external.ts'), 'utf8');
+            const originalFetch = createMockFetch({
+                [externalUrl]: fixtureContent
+            });
+
+            const fnCall = mdFunctionImportCode()()
+            const tree = importCodeMockTree(externalUrl);
+            const file = createMockFile();
+
+            await fnCall(tree, file);
+
+            // Restore original fetch
+            restoreFetch(originalFetch);
+
+            expect(tree.children[0].type).toBe('code');
+            expect((tree.children[0] as any).lang).toBe('typescript');
+            expect((tree.children[0] as any).value).toContain('export function externalFunction');
+        });
+
+        it('should debug visit function', async () => {
+            const fnCall = mdFunctionImportCode()()
+            const tree = importCodeMockTree('test.ts');
+
+            const file = createMockFile();
+
+            // Create a promise that resolves when the transformer is done
+            await fnCall(tree, file);
+
+            expect(tree.children[0].type).toBe('code');
+        });
+
+        it('should handle alternative syntax like @importCode("test.ts")', async () => {
+            const fnCall = mdFunctionImportCode()()
+            const tree = importCodeMockTreeAlternative('test.ts');
+            const file = createMockFile();
+
+            await fnCall(tree, file);
+
+            expect(tree.children[0].type).toBe('code');
+            expect((tree.children[0] as any).lang).toBe('typescript');
+            expect((tree.children[0] as any).value).toContain('export function testFunction');
+        });
+
+        it('should handle md attributes', async () => {
+            const fnCall = mdFunctionImportCode()()
+            const tree = importCodeMdAttributesMockTree('test.ts', {
+                descHead: 'Tip',
+                desc: 'You can use md attributes!'
+            });
+            const file = createMockFile();
+
+            await fnCall(tree, file);
+
+            expect(tree.children[0].type).toBe('code');
+            expect((tree.children[0] as any).lang).toBe('typescript');
+            expect((tree.children[0] as any).value).toContain('export function testFunction');
+        });
+    });
+}); 

package/packages/md/plugins/functions/__tests__/mdFunctionInclude.test.ts

@@ -0,0 +1,44 @@
+import path from 'node:path';
+
+import {describe, it, expect} from 'vitest';
+
+import {mdFunctionInclude} from '../mdFunctionInclude';
+
+import {createMockFile} from './testHelpers';
+
+describe('mdFunctionInclude', () => {
+    it('should transform @include statements to include markdown content', async () => {
+        const transformer = mdFunctionInclude()();
+        const tree = {
+            type: 'root',
+            children: [
+                {
+                    type: 'paragraph',
+                    children: [
+                        {
+                            type: 'text',
+                            value: '@include "test-include.md"'
+                        }
+                    ]
+                }
+            ]
+        };
+        const file = createMockFile();
+
+        await transformer(tree, file);
+
+        expect(tree.children.map(c => c.type)).toEqual([
+                "heading",
+                "paragraph",
+                "heading",
+                "list",
+                "heading",
+                "code",
+                "heading",
+                "list",
+                "list",
+                "mdxJsxFlowElement"
+            ]
+        );
+    });
+});