@xyd-js/content 0.1.0-build.171

package/packages/md/search/index.ts

@@ -0,0 +1,257 @@
+import { readFileSync, statSync } from 'node:fs'
+import path from 'node:path'
+
+import { unified } from 'unified'
+import remarkParse from 'remark-parse'
+import remarkRehype from 'remark-rehype'
+import rehypeStringify from 'rehype-stringify'
+import { visit } from "unist-util-visit";
+import type { Node } from 'mdast'
+
+import GitHubSlugger from 'github-slugger';
+
+import type { Settings, Sidebar, SidebarRoute } from '@xyd-js/core'
+
+import { DocSectionSchema } from './types';
+import { markdownPlugins } from "../"
+import { mdParameters } from '../plugins/utils/mdParameters'
+
+export async function mapSettingsToDocSections(xydSettings: Settings) {
+    const pages = flatPages(xydSettings.navigation?.sidebar || [], {})
+        .map(page => ({
+            name: page,
+            path: path.join(process.cwd(), page)
+        }))
+
+    return await processSections(pages, xydSettings)
+}
+
+export async function mapContentToDocSections(
+    xydSettings: Settings,
+    route: string,
+    content: string,
+) {
+    const sections: DocSectionSchema[] = []
+    let currentSection: DocSectionSchema | null = null
+
+    const rootSection: DocSectionSchema = {
+        // id: '',
+        pageId: '',
+        pageUrl: '',
+        pageTitle: '',
+        headingLevel: 0,
+        headingTitle: '',
+        summary: '',
+        content: '',
+    }
+
+    const { remarkPlugins, rehypePlugins } = await markdownPlugins({}, xydSettings)
+
+    const processor = unified()
+        .use(remarkParse)
+        .use(remarkPlugins)
+        .use(() => {
+            return (tree: any) => {
+                let currentContent = ''
+                visit(tree, (node) => {
+                    // Skip yaml frontmatter
+                    if (node.type === 'yaml') {
+                        return
+                    }
+
+                    // Handle headings
+                    if (node.type === 'heading') {
+                        const heading = processNode(node)
+
+                        // Save previous section's content
+                        if (currentSection && currentContent) {
+                            currentSection.content = currentContent.trim()
+                            currentContent = ''
+                        }
+
+                        const slug = slugify(heading)
+
+                        if (node.depth === 1) {
+                            if (rootSection.pageId) {
+                                console.error("Multiple h1 found")
+                            }
+
+                            rootSection.pageId = slug
+                            rootSection.pageUrl = `/${route}`
+                            rootSection.pageTitle = heading
+                            rootSection.headingLevel = 1
+                            rootSection.headingTitle = heading
+                            sections.push(rootSection)
+                            currentSection = rootSection
+                        } else {
+                            const section: DocSectionSchema = {
+                                pageId: rootSection.pageId,
+                                pageUrl: `${rootSection.pageUrl}#${slug}`, // TODO: QUERY PARAMS
+                                pageTitle: rootSection.headingTitle,
+                                headingLevel: node.depth,
+                                headingTitle: heading,
+                                content: '',
+                                summary: '',
+                            }
+                            sections.push(section)
+                            currentSection = section
+                        }
+                        return
+                    }
+
+                    // Skip if we're not in a section yet
+                    if (!currentSection) {
+                        return
+                    }
+
+                    // For list items, add proper formatting
+                    if (node.type === 'listItem') {
+                        const itemContent = processNode(node)
+                        currentContent += `- ${itemContent}\n`
+                        return
+                    }
+
+                    // For links, preserve the markdown format
+                    if (node.type === 'link') {
+                        const text = processNode(node)
+                        currentContent += `[${text}](${node.url})\n`
+                        return
+                    }
+
+                    // For other nodes, just add their content
+                    const nodeContent = processNode(node)
+                    if (nodeContent) {
+                        currentContent += nodeContent + '\n'
+                    }
+                })
+
+                // Save the last section's content
+                if (currentSection && currentContent) {
+                    currentSection.content = currentContent.trim()
+                }
+            }
+        })
+        .use(remarkRehype)
+        .use(rehypePlugins)
+        .use(rehypeStringify)
+
+    await processor.process(content)
+
+    return sections
+}
+
+function processNode(node: Node): string {
+    if ('value' in node) {
+        const value = node.value as string
+
+        // TODO: check if sanitized does not remove something important
+        const { sanitizedText } = mdParameters(value)
+
+        return sanitizedText
+    }
+
+    if ('children' in node && Array.isArray(node.children)) {
+        return node.children.map((child: Node) => processNode(child)).join('')
+    }
+
+    return ''
+}
+
+
+async function processSections(pages: { name: string, path: string }[], xydSettings: Settings) {
+    return (await Promise.all(
+        pages.flatMap(async (file, i) => {
+            let filePath = ""
+            let err
+
+            try {
+                const p = file.path + ".md"
+                statSync(p)
+                filePath = p
+            } catch (error) {
+                err = error
+            }
+
+            if (!filePath) {
+                try {
+                    const p = file.path + ".mdx"
+                    statSync(p)
+                    filePath = p
+                    err = null
+                } catch (error) {
+                    err = error
+                }
+            }
+
+            if (err) {
+                console.error(err)
+                return []
+            }
+
+            const content = readFileSync(filePath, 'utf-8')
+            const sections = await mapContentToDocSections(
+                xydSettings,
+                file.name,
+                content,
+            )
+
+            return sections.flat()
+        })
+    )).flat()
+}
+
+// TODO: !!!! DRY !!!
+function flatPages(
+    sidebar: (SidebarRoute | Sidebar | string)[],
+    groups: { [key: string]: string },
+    resp: string[] = [],
+) {
+    sidebar.map(async side => {
+        if (typeof side === "string") {
+            resp.push(side)
+            return
+        }
+
+        if ("route" in side) {
+            side.pages?.map(item => {
+                return flatPages([item], groups, resp)
+            })
+            return
+        }
+
+        // Type guard to check if it's a Sidebar
+        if ("group" in side) {
+            const groupKey = side.group || "";
+            if (groups[groupKey]) {
+                const link = groups[groupKey];
+                if (link) {
+                    resp.push(link);
+                }
+            }
+
+            side.pages?.map(async page => {
+                if (typeof page === "string") {
+                    resp.push(page);
+                    return;
+                }
+
+                if ("virtual" in page && page.virtual) {
+                    resp.push(page.virtual);
+                    return;
+                }
+
+                if ("pages" in page) {
+                    return flatPages([page as Sidebar], groups, resp);
+                }
+            });
+        }
+    });
+
+    return resp;
+}
+
+function slugify(text: string) {
+    const slugger = new GitHubSlugger()
+
+    return slugger.slug(text)
+}

package/packages/md/search/types.ts

@@ -0,0 +1,36 @@
+export interface DocSectionSchema { // TODO: move to @xyd-js/plugins/search ?
+    /**
+     * Id of the page e.g. "getting-started"
+     */
+    pageId: string
+
+    /**
+     * URL of the page e.g. "/getting-started"
+     */
+    pageUrl: string
+
+    /**
+     * Title of the page e.g. "Getting Started"
+     */
+    pageTitle: string
+
+    /**
+     * Level of the heading e.g. 1, 2
+     */
+    headingLevel: number
+
+    /**
+     * Title of the heading e.g. "Making a new project"
+     */
+    headingTitle: string
+
+    /**
+     * Summary of the page e.g. "This is the summary of the page"
+     */
+    summary: string
+
+    /**
+     * Content of the section
+     */
+    content: string
+}

package/packages/vite/index.ts

@@ -0,0 +1,20 @@
+import type {Plugin} from 'rollup';
+import mdx from '@mdx-js/rollup';
+
+import {Settings} from "@xyd-js/core";
+
+import {RemarkMdxTocOptions, markdownPlugins} from "../md";
+
+export interface VitePluginInterface {
+    toc: RemarkMdxTocOptions
+    settings: Settings
+}
+
+export async function vitePlugins(options: VitePluginInterface): Promise<Plugin[]> {
+    return [
+        mdx(await markdownPlugins(
+            options.toc,
+            options.settings
+        )),
+    ];
+}

package/src/fs.ts

@@ -0,0 +1,81 @@
+import { tmpdir } from "os";
+import { join } from "path";
+import { pathToFileURL } from "url";
+import fs, { rm, writeFile } from "fs/promises"
+
+import { VFile } from "vfile"
+import { PluggableList } from "unified";
+import { compile as mdxCompile } from "@mdx-js/mdx";
+
+import { Settings } from "@xyd-js/core"
+
+export type { VarCode } from "./types"
+
+export class ContentFS {
+    constructor(
+        private readonly settings: Settings,
+        private readonly remarkPlugins: PluggableList,
+        private readonly rehypePlugins: PluggableList,
+        private readonly recmaPlugins: PluggableList,
+    ) { }
+
+    public async compile(filePath: string): Promise<string> {
+        await fs.access(filePath)
+
+        const content = await fs.readFile(filePath, "utf-8");
+
+        return await this.compileContent(content, filePath)
+    }
+
+    public async compileContent(content: string, filePath?: string): Promise<string> {
+        const vfile = new VFile({
+            path: filePath,
+            value: content,
+            contents: content
+        });
+
+        const compiled = await mdxCompile(vfile, {
+            remarkPlugins: this.remarkPlugins,
+            rehypePlugins: this.rehypePlugins,
+            recmaPlugins: this.recmaPlugins || [],
+            development: false,
+            outputFormat: 'function-body',
+            jsx: false,
+            // jsx: false,
+            // outputFormat: "program", // needed for import/export
+        });
+
+        return String(compiled)
+    }
+
+
+    public async compileContentV2(content: string, filePath?: string): Promise<string> {
+        const vfile = new VFile({
+            path: filePath,
+            value: content,
+            contents: content
+        });
+
+        const compiled = await mdxCompile(vfile, {
+            remarkPlugins: this.remarkPlugins,
+            rehypePlugins: this.rehypePlugins,
+            recmaPlugins: [],
+            development: false,
+            jsx: true,
+            outputFormat: "program", // needed for import/export
+        });
+
+        const tempPath = join(tmpdir(), `mdx-${Date.now()}.mjs`);
+        await writeFile(tempPath, String(compiled), "utf8");
+
+        return tempPath
+    }
+
+    public async readRaw(filePath: string) {
+        await fs.access(filePath)
+
+        const content = await fs.readFile(filePath, "utf-8");
+
+        return content
+    }
+}

package/src/index.ts

@@ -0,0 +1,7 @@
+export { ContentFS } from "./fs"
+
+export {
+    pageFrontMatters,
+} from "./navigation"
+
+export type { VarCode } from "./types"

package/src/navigation.ts

@@ -0,0 +1,147 @@
+import {promises as fs} from 'fs';
+import path from 'path';
+
+import React from "react";
+import remarkFrontmatter from "remark-frontmatter";
+import remarkMdxFrontmatter from "remark-mdx-frontmatter";
+import {VFile} from "vfile";
+import {compile as mdxCompile} from "@mdx-js/mdx";
+
+import {Metadata, Sidebar, MetadataMap, PageURL, VirtualPage} from "@xyd-js/core";
+
+// TODO: better algorithm + data structures - since it's on build time it's not a big deal nevertheless it should be changed in the future
+
+// pageFrontMatters gets frontmatters for given navigation
+export async function pageFrontMatters(navigation: Sidebar[], pagePathMapping: {
+    [key: string]: string
+}): Promise<MetadataMap> {
+    const frontmatters: MetadataMap = {}
+
+    const promises: Promise<any>[] = []
+
+    function mapPages(page: PageURL) {
+        if (typeof page !== "string") {
+            if ("virtual" in page) {
+                promises.push(job(page, frontmatters, pagePathMapping))
+            } else if ("pages" in page) {
+                page.pages?.forEach(mapPages)
+            }
+            return
+        }
+
+        promises.push(job(page, frontmatters, pagePathMapping))
+    }
+
+    navigation.map(async (nav: Sidebar) => {
+        if (typeof nav === "string") {
+            mapPages(nav)
+            return
+        }
+
+        nav.pages?.forEach(mapPages)
+    })
+
+    await Promise.all(promises)
+
+    // TODO: IN THE FUTURE BETTER API
+    // @ts-ignore
+    if (globalThis.__xydHasIndexPage) {
+        await job("index", frontmatters, pagePathMapping)
+    }
+
+    return frontmatters
+}
+
+function mdxExport(code: string) {
+    const scope = {
+        Fragment: React.Fragment,
+        jsxs: React.createElement,
+        jsx: React.createElement,
+        jsxDEV: React.createElement,
+    }
+    const fn = new Function(...Object.keys(scope), code)
+    return fn(scope)
+}
+
+async function getFrontmatter(filePath: string): Promise<Metadata> {
+    const body = await fs.readFile(filePath, "utf-8");
+
+    const vfile = new VFile({
+        path: filePath,
+        value: body,
+        contents: body
+    });
+
+    const compiled = await mdxCompile(vfile, {
+        remarkPlugins: [
+            remarkFrontmatter,
+            remarkMdxFrontmatter
+        ],
+        rehypePlugins: [],
+        recmaPlugins: [],
+        outputFormat: 'function-body',
+        development: false,
+        
+        // outputFormat: "program",
+        // jsx: true,
+    });
+
+    const code = String(compiled)
+
+    const {
+        reactFrontmatter, // in the future same key?
+        frontmatter
+    } = mdxExport(code)
+
+    const matter: Metadata = frontmatter
+
+    if (!matter) {
+        throw new Error(`Frontmatter not found in ${filePath}`)
+    }
+
+    let title = ""
+    if (typeof matter.title === "string") {
+        title = matter.title
+    }
+    if (reactFrontmatter) {
+        console.error("currently react frontmatter is not supported")
+    }
+
+    return matter
+}
+
+// TODO: indices map to not do like this - search for mdx if not then md
+async function job(page: string | VirtualPage, frontmatters: MetadataMap, pagePathMapping: { [key: string]: string }) {
+    let pageName = ""
+    if (typeof page === "string") {
+        pageName = page
+    } else if (page.page) {
+        pageName = page.page
+    }
+
+    // @ts-ignore TODO: IN THE FUTURE BETTER API
+    if (globalThis.__xydFrontmatterNotExists && globalThis.__xydFrontmatterNotExists[pageName]) {
+        return
+    }
+
+    if (!pageName || !pagePathMapping[pageName]) {
+        console.log(`⚠️ "${pageName}" is defined in the docs.json navigation but the file does not exist.`)
+
+        // @ts-ignore
+        if (!globalThis.__xydFrontmatterNotExists) {
+            // @ts-ignore
+            globalThis.__xydFrontmatterNotExists = {}
+        }
+
+        // @ts-ignore 
+        globalThis.__xydFrontmatterNotExists[pageName] = true
+
+        return
+    }
+
+    const filePath = path.join(process.cwd(), pagePathMapping[pageName])
+
+    const matter = await getFrontmatter(filePath)
+
+    frontmatters[pageName] = matter
+}

package/src/types.ts

@@ -0,0 +1,8 @@
+import { HighlightedCode } from "codehike/code"
+
+export type VarCode = [
+    [
+        string,
+        ...(HighlightedCode)[]
+    ] | HighlightedCode
+]

package/tsconfig.json

@@ -0,0 +1,49 @@
+{
+  "compilerOptions": {
+    "baseUrl": ".",
+    "paths": {
+      "@/mdx/*": [
+        "src/mdx/*"
+      ],
+      "@/navigation/*": [
+        "src/navigation/*"
+      ],
+      "@/utils/*": [
+        "src/utils/*"
+      ],
+      "@/vite-plugins/*": [
+        "src/vite-plugins/*"
+      ],
+      "@xyd-js/core": [
+        "../xyd-core"
+      ],
+      "@xyd-js/context": [
+        "../xyd-context/src/index.ts"
+      ]
+    },
+    "target": "ES2020",
+    "module": "ESNext",
+    "moduleResolution": "node",
+    "strict": true,
+    "esModuleInterop": true,
+    "skipLibCheck": true,
+    "forceConsistentCasingInFileNames": true,
+    "outDir": "./dist",
+    "types": [
+      "node",
+      "estree",
+      "vite"
+    ],
+    "declaration": true,
+    "declarationMap": true,
+    "incremental": true,
+    "tsBuildInfoFile": "./dist/tsconfig.tsbuildinfo",
+  },
+  "include": [
+    "src/**/*.ts"
+  ],
+  "exclude": [
+    "node_modules",
+    "dist"
+  ]
+}

package/tsup.config.ts

@@ -0,0 +1,32 @@
+import {defineConfig, Options} from 'tsup';
+
+const config: Options = {
+    entry: {
+        index: 'src/index.ts',
+        vite: 'packages/vite/index.ts',
+        md: 'packages/md/index.ts',
+    },
+    dts: {
+        entry: {
+            index: 'src/index.ts',
+            vite: 'packages/vite/index.ts',
+            md: 'packages/md/index.ts'
+        },
+        resolve: true, // Resolve external types
+    },
+    format: ['esm'], // Output both ESM and CJS formats
+    target: 'node16', // Ensure compatibility with Node.js 16
+    splitting: false, // Disable code splitting
+    sourcemap: true, // Generate source maps
+    clean: true, // Clean the output directory before each build
+    esbuildOptions: (options) => {
+        // options.platform = 'node'; // Ensure the platform is set to Node.js
+        // options.external = ['node:fs/promises']; // Mark 'node:fs/promises' as external
+        // options.loader = { '.js': 'jsx' }; // Ensure proper handling of .js files
+        options.external = [
+            'rehype-mermaid'
+        ]
+    },
+};
+
+export default defineConfig(config);

package/vitest.config.ts

@@ -0,0 +1,17 @@
+import {defineConfig} from 'vitest/config'
+
+export default defineConfig({
+    test: {
+        globals: true,
+        environment: 'node',
+        include: [
+            'src/**/*.test.ts',
+            'packages/**/*.test.ts',
+            '**/__tests__/**/*.test.ts'
+        ],
+        coverage: {
+            provider: 'v8',
+            reporter: ['text', 'json', 'html'],
+        },
+    },
+})