@xyd-js/content 0.1.0-xyd.2

package/index.ts

@@ -0,0 +1,10 @@
+export {
+    compileBySlug
+} from "./src/utils";
+
+export {
+    mdxOptions
+} from "./src/mdx/options";
+
+export {vitePlugins} from "./src/vite-plugins";
+

package/navigation.ts

@@ -0,0 +1,4 @@
+export {
+    pageFrontMatters,
+    filterNavigationByLevels
+} from "./src/navigation";

package/package.json

@@ -0,0 +1,40 @@
+{
+  "name": "@xyd-js/content",
+  "version": "0.1.0-xyd.2",
+  "description": "",
+  "main": "./dist/index.js",
+  "type": "module",
+  "exports": {
+    "./package.json": "./package.json",
+    ".": "./dist/index.js",
+    "./navigation": "./dist/navigation.js"
+  },
+  "dependencies": {
+    "@mdx-js/mdx": "^3.1.0",
+    "@mdx-js/rollup": "^3.0.1",
+    "estree-util-is-identifier-name": "^3.0.0",
+    "estree-util-value-to-estree": "^3.1.2",
+    "gray-matter": "^4.0.3",
+    "mdast": "^3.0.0",
+    "mdast-util-mdx": "^3.0.0",
+    "mdast-util-mdx-jsx": "^3.1.3",
+    "mdast-util-to-string": "^4.0.0",
+    "remark-frontmatter": "^5.0.0",
+    "remark-gfm": "^4.0.0",
+    "remark-mdx-frontmatter": "^5.0.0",
+    "unified": "^11.0.5",
+    "unist-util-visit": "^5.0.0",
+    "vfile": "^6.0.3",
+    "@xyd-js/core": "0.1.0-xyd.0"
+  },
+  "devDependencies": {
+    "@types/node": "^22.7.8",
+    "tsup": "^8.3.0"
+  },
+  "scripts": {
+    "clean": "rimraf build",
+    "prebuild": "pnpm clean",
+    "build": "tsup",
+    "watch": "tsup --watch"
+  }
+}

package/src/mdx/code.ts

@@ -0,0 +1,15 @@
+import {visit} from "unist-util-visit";
+
+export function remarkInjectCodeMeta() {
+    return (tree: any) => {
+        visit(tree, 'code', (node) => {
+            if (node.meta) {
+                node.data = node.data || {};
+                node.data.hProperties = {
+                    ...(node.data.hProperties || {}),
+                    meta: node.meta,
+                };
+            }
+        });
+    };
+}

package/src/mdx/options.ts

@@ -0,0 +1,23 @@
+import remarkFrontmatter from "remark-frontmatter";
+import remarkMdxFrontmatter from "remark-mdx-frontmatter";
+import remarkGfm from "remark-gfm";
+
+import {remarkMdxToc, RemarkMdxTocOptions} from "@/mdx/toc";
+import {remarkInjectCodeMeta} from "@/mdx/code";
+import {extractThemeSettings} from "@/mdx/themeSettings";
+import {extractPage} from "@/mdx/page";
+
+export function mdxOptions(toc: RemarkMdxTocOptions) {
+    return {
+        remarkPlugins: [
+            remarkFrontmatter,
+            remarkMdxFrontmatter,
+            remarkGfm,
+            remarkInjectCodeMeta,
+            remarkMdxToc(toc),
+            extractThemeSettings,
+            extractPage
+        ],
+        rehypePlugins: []
+    }
+}

package/src/mdx/page.ts

@@ -0,0 +1,22 @@
+import {Plugin} from 'unified';
+import {Node as UnistNode} from 'unist';
+import {visit} from 'unist-util-visit';
+
+declare global {
+    var page: boolean | null | undefined
+}
+
+export const extractPage: Plugin = () => {
+    return (tree: UnistNode) => {
+        visit(tree, 'exportNamedDeclaration', (node: any) => {
+            const declaration = node.declaration;
+            if (declaration && declaration.declarations) {
+                declaration.declarations.forEach((decl: any) => {
+                    if (decl.id.name === 'page') {
+                        global.page = decl.init as boolean;
+                    }
+                });
+            }
+        });
+    };
+};

package/src/mdx/themeSettings.ts

@@ -0,0 +1,26 @@
+import {Plugin} from 'unified';
+import {Node as UnistNode} from 'unist';
+import {visit} from 'unist-util-visit';
+
+interface ThemeSettings {
+    bigArticle: boolean;
+}
+
+declare global {
+    var themeSettings: ThemeSettings | undefined;
+}
+
+export const extractThemeSettings: Plugin = () => {
+    return (tree: UnistNode) => {
+        visit(tree, 'exportNamedDeclaration', (node: any) => {
+            const declaration = node.declaration;
+            if (declaration && declaration.declarations) {
+                declaration.declarations.forEach((decl: any) => {
+                    if (decl.id.name === 'themeSettings') {
+                        global.themeSettings = decl.init as ThemeSettings;
+                    }
+                });
+            }
+        });
+    };
+};

package/src/mdx/toc.ts

@@ -0,0 +1,132 @@
+import {Root, Heading} from "mdast";
+import {MdxjsEsm} from "mdast-util-mdx";
+import {Plugin} from "unified"; // TODO: use Plugin type
+import {MdxJsxFlowElement, MdxJsxAttribute} from "mdast-util-mdx-jsx";
+
+export type TocEntry = {
+    depth: number,
+    value: string,
+    attributes: { [key: string]: string },
+    children: TocEntry[]
+};
+
+export type CustomTag = {
+    name: RegExp,
+    depth: (name: string) => number
+};
+
+export interface RemarkMdxTocOptions {
+    name?: string,
+    customTags?: CustomTag[],
+    minDepth?: number
+}
+
+// TODO: fix any
+export const remarkMdxToc = (options: RemarkMdxTocOptions): Plugin => () => async (ast: any) => {
+    const {visit} = await import("unist-util-visit");
+    const {toString} = await import("mdast-util-to-string");
+    const {valueToEstree} = await import('estree-util-value-to-estree')
+    const {name: isIdentifierName} = await import('estree-util-is-identifier-name');
+
+    const mdast = ast as Root;
+    const name = options.name ?? "toc";
+    if (!isIdentifierName(name)) {
+        throw new Error(`Invalid name for an identifier: ${name}`);
+    }
+
+    const toc: TocEntry[] = [];
+    const flatToc: TocEntry[] = [];
+    const createEntry = (node: Heading | MdxJsxFlowElement, depth: number): TocEntry => {
+        let attributes = (node.data || {}) as TocEntry['attributes'];
+        if (node.type === "mdxJsxFlowElement") {
+            attributes = Object.fromEntries(
+                node.attributes
+                    .filter(attribute => attribute.type === 'mdxJsxAttribute' && typeof attribute.value === 'string')
+                    .map(attribute => [(attribute as MdxJsxAttribute).name, attribute.value])
+            ) as TocEntry['attributes'];
+        }
+        return {
+            depth,
+            value: toString(node, {includeImageAlt: false}),
+            attributes,
+            children: []
+        }
+    };
+
+    visit(mdast, ["heading", "mdxJsxFlowElement"], node => {
+        let depth = 0;
+        if (node.type === "mdxJsxFlowElement") {
+            let valid = false;
+            if (/^h[1-6]$/.test(node.name || "")) {
+                valid = true;
+                depth = parseInt(node.name!.substring(1));
+            } else if (options.customTags) {
+                for (const tag of options.customTags) {
+                    if (tag.name.test(node.name || "")) {
+                        valid = true;
+                        depth = tag.depth(node.name || "");
+                        break;
+                    }
+                }
+            }
+
+            if (!valid) {
+                return;
+            }
+        } else if (node.type === "heading") {
+            depth = node.depth;
+        } else {
+            return;
+        }
+
+        if (depth && (options?.minDepth && options.minDepth > depth)) {
+            return
+        }
+
+        const entry = createEntry(node, depth);
+        flatToc.push(entry);
+
+        let parent: TocEntry[] = toc;
+        for (let i = flatToc.length - 1; i >= 0; --i) {
+            const current = flatToc[i];
+            if (current.depth < entry.depth) {
+                parent = current.children;
+                break;
+            }
+        }
+        parent.push(entry);
+    });
+
+    const tocExport: MdxjsEsm = {
+        type: "mdxjsEsm",
+        value: "",
+        data: {
+            estree: {
+                type: "Program",
+                sourceType: "module",
+                body: [
+                    {
+                        type: "ExportNamedDeclaration",
+                        specifiers: [],
+                        source: null,
+                        declaration: {
+                            type: "VariableDeclaration",
+                            kind: "const",
+                            declarations: [
+                                {
+                                    type: "VariableDeclarator",
+                                    id: {
+                                        type: "Identifier",
+                                        name
+                                    },
+                                    init: valueToEstree(toc)
+                                }
+                            ]
+                        }
+                    }
+                ]
+            }
+        }
+    };
+    mdast.children.unshift(tocExport);
+};

package/src/navigation/index.ts

@@ -0,0 +1,188 @@
+import {promises as fs} from 'fs';
+import fs2, {open} from 'fs';
+import path from 'path';
+
+import React from "react";
+import remarkFrontmatter from "remark-frontmatter";
+import remarkMdxFrontmatter from "remark-mdx-frontmatter";
+import matter from 'gray-matter';
+import {VFile} from "vfile";
+import {compile as mdxCompile} from "@mdx-js/mdx";
+
+import {FrontMatter, Sidebar, PageFrontMatter, Header} from "@xyd-js/core";
+
+// TODO: better algorithm + data structures - since it's on build time it's not a big deal nevertheless it should be changed in the future
+
+// pageFrontMatters gets frontmatters for given navigation
+export async function pageFrontMatters(navigation: Sidebar[]): Promise<PageFrontMatter> {
+    const frontmatters: PageFrontMatter = {}
+
+    const promises: Promise<any>[] = []
+
+    function mapPages(page: string | Sidebar) {
+        if (typeof page !== "string") {
+            page.pages?.forEach(mapPages)
+            return
+        }
+
+        promises.push(job(page, frontmatters))
+    }
+
+    navigation.map(async (nav: Sidebar) => {
+        nav.pages?.forEach(mapPages)
+    })
+
+    await Promise.all(promises)
+
+    return frontmatters
+}
+
+// filterNavigation filter navigation items by top levels of 'header' configuration and current 'slug'
+export function filterNavigationByLevels(
+    headers: Header[],
+    slug: string
+) {
+    const topLevelTabMatcher = headers?.reduce((acc: any, header) => {
+        const tabLevel = header?.url?.split("/")?.length
+
+        if (!tabLevel) {
+            return {
+                ...acc
+            }
+        }
+
+        if (!acc[tabLevel]) {
+            return {
+                ...acc,
+                [tabLevel]: new Set().add(header?.url)
+            }
+        }
+
+        return {
+            ...acc,
+            [tabLevel]: acc[tabLevel].add(header?.url)
+        }
+    }, {}) as { [level: number]: Set<string> }
+
+    return (nav: Sidebar) => {
+        let match = false
+
+        Object.keys(topLevelTabMatcher).forEach((levelStr) => {
+            if (match) {
+                return true
+            }
+            const level = parseInt(levelStr)
+            const findThisSlug = slug.split("/").filter(s => !!s).slice(0, level).join("/")
+
+            function findMatchedPage(page: string | Sidebar) {
+                if (typeof page !== "string") {
+                    page.pages?.forEach(findMatchedPage)
+                    return
+                }
+                const findThisPage = page.split("/").filter(p => !!p).slice(0, level).join("/")
+
+                const set = topLevelTabMatcher[level]
+
+                if (set.has(findThisPage) && findThisPage === findThisSlug) {
+                    match = true
+                    return true
+                }
+            }
+
+            nav?.pages?.forEach(findMatchedPage)
+        })
+
+        return match
+    }
+}
+
+async function getFrontmatter(filePath: string, chunkSize = 1024 * 5) { // 5 KB chunk
+    return new Promise((resolve, reject) => {
+        open(filePath, 'r', (err, fd) => {
+            if (err) return reject(err);
+
+            const buffer = Buffer.alloc(chunkSize);
+            fs2.read(fd, buffer, 0, chunkSize, 0, (err, bytesRead) => {
+                if (err) return reject(err);
+
+                const uint8Array = new Uint8Array(buffer.buffer, buffer.byteOffset, bytesRead);
+                const content = new TextDecoder('utf-8').decode(uint8Array);
+                const {data: frontmatter} = matter(content); // extract frontmatter
+                resolve(frontmatter);
+
+                fs2.close(fd, () => {
+                });
+            });
+        });
+    });
+}
+
+function mdxExport(code: string) {
+    const scope = {
+        Fragment: React.Fragment,
+        jsxs: React.createElement,
+        jsx: React.createElement,
+        jsxDEV: React.createElement,
+    }
+    const fn = new Function(...Object.keys(scope), code)
+    return fn(scope)
+}
+
+async function getFrontmatterV2(filePath: string): Promise<FrontMatter> {
+    const body = await fs.readFile(filePath, "utf-8");
+
+    const vfile = new VFile({
+        path: filePath,
+        value: body,
+        contents: body
+    });
+
+    const compiled = await mdxCompile(vfile, {
+        remarkPlugins: [
+            remarkFrontmatter,
+            remarkMdxFrontmatter
+        ],
+        rehypePlugins: [],
+        recmaPlugins: [],
+        outputFormat: 'function-body',
+        development: false,
+    });
+
+    const code = String(compiled)
+
+    const {
+        reactFrontmatter, // in the future same key?
+        frontmatter
+    } = mdxExport(code)
+
+    const matter: FrontMatter = frontmatter
+
+    if (reactFrontmatter) {
+        if (typeof reactFrontmatter?.title === "function") {
+            matter.title = {
+                code: reactFrontmatter.title.toString()
+            }
+        }
+    }
+
+    return matter
+}
+
+async function job(page: string, frontmatters: PageFrontMatter) {
+    // TODO: it can be cwd because on build time it has entire path?
+    let filePath = path.join(process.cwd(), `${page}.mdx`) // TODO: support md toos
+    try {
+        await fs.access(filePath)
+    } catch (e) {
+        try {
+            const mdPath = filePath.replace(".mdx", ".md")
+            await fs.access(mdPath)
+            filePath = mdPath
+        } catch (e) {
+        }
+    }
+
+    const matter = await getFrontmatterV2(filePath)
+
+    frontmatters[page] = matter
+}

package/src/utils/index.ts

@@ -0,0 +1,43 @@
+import {promises as fs} from "fs";
+import path from "path";
+
+import {VFile} from "vfile";
+import {compile as mdxCompile} from "@mdx-js/mdx";
+
+import {mdxOptions} from "@/mdx/options";
+
+export async function compileBySlug(
+    slug: string,
+    mdx: boolean
+): Promise<string> {
+    // TODO: cwd ?
+    const filePath = path.join(process.cwd(), `${slug}.${mdx ? "mdx" : "md"}`)
+
+    await fs.access(filePath)
+
+    const content = await fs.readFile(filePath, "utf-8");
+
+    return await compile(content, filePath)
+}
+
+async function compile(content: string, filePath: string): Promise<string> {
+    const vfile = new VFile({
+        path: filePath,
+        value: content,
+        contents: content
+    });
+
+    const mdOptions = mdxOptions({
+        minDepth: 2 // TODO: configurable?
+    })
+
+    const compiled = await mdxCompile(vfile, {
+        remarkPlugins: mdOptions.remarkPlugins,
+        rehypePlugins: mdOptions.rehypePlugins,
+        recmaPlugins: [],
+        outputFormat: 'function-body',
+        development: false,
+    });
+
+    return String(compiled)
+}

package/src/vite-plugins/index.ts

@@ -0,0 +1,15 @@
+import type {Plugin} from 'rollup';
+import mdx from '@mdx-js/rollup';
+
+import {RemarkMdxTocOptions} from "@/mdx/toc";
+import {mdxOptions} from "@/mdx/options";
+
+export interface VitePluginInterface {
+    toc: RemarkMdxTocOptions
+}
+
+export function vitePlugins(options: VitePluginInterface): Plugin[] {
+    return [
+        mdx(mdxOptions(options.toc)),
+    ];
+}

package/tsconfig.json

@@ -0,0 +1,26 @@
+{
+  "compilerOptions": {
+    "baseUrl": ".",
+    "paths": {
+      "@/mdx/*": ["src/mdx/*"],
+      "@/navigation/*": ["src/navigation/*"],
+      "@/utils/*": ["src/utils/*"],
+      "@/vite-plugins/*": ["src/vite-plugins/*"]
+    },
+    "target": "ES2020",
+    "module": "ESNext",
+    "moduleResolution": "node",
+    "strict": true,
+    "esModuleInterop": true,
+    "skipLibCheck": true,
+    "forceConsistentCasingInFileNames": true,
+    "outDir": "./dist",
+    "types": ["node", "estree", "vite"],
+    "declaration": true,
+    "declarationMap": true,
+    "incremental": true,
+    "tsBuildInfoFile": "./dist/tsconfig.tsbuildinfo"
+  },
+  "include": ["src/**/*.ts"],
+  "exclude": ["node_modules", "dist"]
+}

package/tsup.config.ts

@@ -0,0 +1,27 @@
+import { defineConfig, Options } from 'tsup';
+
+const config: Options = {
+    entry: {
+        index: 'index.ts',
+        navigation: 'navigation.ts',
+    },
+    dts: {
+        entry: {
+            index: 'index.ts',
+            navigation: 'navigation.ts',
+        },
+        resolve: true, // Resolve external types
+    },
+    format: ['esm'], // Output both ESM and CJS formats
+    target: 'node16', // Ensure compatibility with Node.js 16
+    splitting: false, // Disable code splitting
+    sourcemap: false, // Generate source maps
+    clean: true, // Clean the output directory before each build
+    // esbuildOptions: (options) => {
+    //     options.platform = 'node'; // Ensure the platform is set to Node.js
+    //     options.external = ['node:fs/promises']; // Mark 'node:fs/promises' as external
+    //     options.loader = { '.js': 'jsx' }; // Ensure proper handling of .js files
+    // },
+};
+
+export default defineConfig(config);

package/vite.config.js

@@ -0,0 +1,53 @@
+import {defineConfig} from 'vite';
+
+export default defineConfig(async () => {
+    const mdx = await import('@mdx-js/rollup');
+    const remix = await import('@remix-run/dev');
+    const remarkFrontmatter = await import('remark-frontmatter');
+    const remarkMdxFrontmatter = await import('remark-mdx-frontmatter');
+    const remarkGfm = await import('remark-gfm');
+    const rehypePrettyCode = await import('rehype-pretty-code'); // TODO: for some reasons does not work
+    // const { remarkCodeHike, recmaCodeHike } = await import('codehike/mdx'); // TODO: delete because we use this inside components?
+
+    const settings = await import("./settings.json");
+
+    return {
+        optimizeDeps: {
+            include: ["react/jsx-runtime"],
+        },
+        resolve: {},
+        plugins: [
+            mdx.default({
+                // providerImportSource: '@mdx-js/react',
+                remarkPlugins: [
+                    // remarkCodeHike,
+                    remarkFrontmatter.default,
+                    remarkMdxFrontmatter.default,
+                    remarkGfm.default
+                ],
+                rehypePlugins: [
+                    // recmaCodeHike,
+                    // rehypePrettyCode.default, TODO: for some reasons does not work
+                ],
+            }),
+            remix.vitePlugin({
+                // routes(defineRoutes) {
+                //     return defineRoutes((route) => {
+                //         route(
+                //             "authentication",
+                //             "routes/_index.tsx",
+                //             {id: 'routes/__main-index'},
+                //             () => {
+                //                 route(
+                //                     "test",
+                //                     "routes/$page.tsx",
+                //                     {id: 'routes/__main-index-2'},
+                //                 );
+                //             });
+                //     });
+                // },
+            }),
+            // Add other plugins here
+        ],
+    };
+});