@docubook/core 1.0.0

package/README.md

@@ -0,0 +1,131 @@
+# @docubook/core
+
+Shared MDX compile pipeline and markdown utilities for DocuBook.
+
+## Features
+
+- **Centralized MDX Pipeline** - One shared compile flow for all DocuBook projects
+- **Managed Remark/Rehype Plugins** - Core markdown plugins are maintained by DocuBook author
+- **Content-First Workflow** - Users can focus on docs content instead of plugin maintenance
+- **Utility Helpers** - Frontmatter extraction, TOC extraction, and slug helpers included
+- **Consistent Behavior** - Same markdown rendering behavior across apps and templates
+
+## Installation
+
+Node.js ecosystem:
+
+```bash
+npm install @docubook/core
+```
+
+```bash
+pnpm add @docubook/core
+```
+
+```bash
+yarn add @docubook/core
+```
+
+Bun (independent runtime/package manager):
+
+```bash
+bun add @docubook/core
+```
+
+## Dependency Management Policy
+
+Dependencies required for markdown processing are managed in this package and updated by the DocuBook author.
+
+This means app-level users should focus on content and integration. Plugin upgrades, compatibility checks, and pipeline maintenance are handled centrally by DocuBook.
+
+### Managed Markdown Dependencies
+
+- gray-matter
+- rehype-autolink-headings
+- rehype-code-titles
+- rehype-prism-plus
+- rehype-slug
+- remark-gfm
+- unist-util-visit
+
+The most important part is the `remark` and `rehype` plugin stack, which is intentionally owned by this package to avoid dependency drift across apps.
+
+## Why This Matters
+
+- Consistent behavior across all DocuBook-based projects
+- Easier maintenance and safer upgrades
+- Less dependency duplication in app-level package.json files
+- Faster onboarding for users who only need to write docs
+
+## Usage
+
+```ts
+import {
+  parseMdx,
+  extractFrontmatter,
+  extractTocsFromRawMdx,
+} from "@docubook/core";
+
+const raw = `---\ntitle: Intro\n---\n\n## Hello`;
+
+const frontmatter = extractFrontmatter<{ title: string }>(raw);
+const toc = extractTocsFromRawMdx(raw);
+const compiled = await parseMdx<{ title: string }>(raw);
+```
+
+### File-Based Pipeline (Recommended)
+
+```ts
+import {
+  createMdxContentService,
+  readMdxFileBySlug,
+  parseMdxFile,
+  compileParsedMdxFile,
+} from "@docubook/core";
+
+type Frontmatter = {
+  title: string;
+  description: string;
+  image: string;
+  date: string;
+};
+
+const raw = await readMdxFileBySlug("getting-started/introduction");
+const parsed = parseMdxFile<Frontmatter>(raw);
+const compiled = await compileParsedMdxFile(parsed, {
+  components: {
+    // your mdx components
+  },
+});
+
+const docsService = createMdxContentService<Frontmatter>({
+  parseOptions: {
+    components: {
+      // your mdx components
+    },
+  },
+});
+
+const doc = await docsService.getCompiledForSlug("getting-started/introduction");
+```
+
+## API Overview
+
+- `parseMdx` - Compiles raw MDX with default or custom plugin options
+- `createMdxContentService` - Unified high-level API for read/parse/compile/getters
+- `readMdxFileBySlug` - Reads `slug.mdx` or `slug/index.mdx` from docs directory
+- `parseMdxFile` - Converts raw file result into `frontmatter` + `tocs` + `content`
+- `compileParsedMdxFile` - Compiles a parsed document while preserving metadata
+- `extractFrontmatter` - Parses frontmatter from raw markdown/MDX
+- `extractTocsFromRawMdx` - Extracts headings for TOC generation
+- `sluggify` - Converts heading text into URL-friendly slugs
+
+## Notes
+
+If your app uses `next-mdx-remote` directly for rendering in custom components, keep that direct dependency in the app.
+
+For compile pipeline plugins (especially `remark` and `rehype` plugins), rely on this package and avoid re-declaring them at app level unless you have a specific override requirement.
+
+## License
+
+MIT

package/package.json

@@ -0,0 +1,44 @@
+{
+    "name": "@docubook/core",
+    "version": "1.0.0",
+    "description": "Shared MDX compile pipeline and markdown utilities for DocuBook",
+    "type": "module",
+    "main": "./src/index.ts",
+    "exports": {
+        ".": "./src/index.ts"
+    },
+    "scripts": {
+        "build": "echo 'core uses source exports'"
+    },
+    "keywords": [
+        "docubook",
+        "documentation",
+        "docs framework",
+        "mdx compile",
+        "markdown utilities"
+    ],
+    "homepage": "https://docubook.pro/",
+    "repository": {
+        "type": "git",
+        "url": "git+https://github.com/DocuBook/docubook"
+    },
+    "author": "wildan.nrs",
+    "author-url": "https://wildan.dev",
+    "license": "MIT",
+    "dependencies": {
+        "gray-matter": "^4.0.3",
+        "next-mdx-remote": "^6.0.0",
+        "rehype-autolink-headings": "^7.1.0",
+        "rehype-code-titles": "^1.2.1",
+        "rehype-prism-plus": "^2.0.2",
+        "rehype-slug": "^6.0.0",
+        "remark-gfm": "^4.0.1",
+        "unist-util-visit": "^5.1.0"
+    },
+    "devDependencies": {
+        "@types/node": "^20.19.37",
+        "@types/react": "19.2.8",
+        "@types/unist": "^3.0.3",
+        "typescript": "^5.9.3"
+    }
+}

package/src/compile.ts

@@ -0,0 +1,131 @@
+import { compileMDX } from "next-mdx-remote/rsc";
+import type { Node, Parent } from "unist";
+import { visit } from "unist-util-visit";
+import remarkGfm from "remark-gfm";
+import rehypePrism from "rehype-prism-plus";
+import rehypeAutolinkHeadings from "rehype-autolink-headings";
+import rehypeSlug from "rehype-slug";
+import rehypeCodeTitles from "rehype-code-titles";
+import type { MdxCompileResult } from "./types";
+
+interface Element extends Node {
+    type: string;
+    tagName?: string;
+    properties?: Record<string, unknown> & {
+        className?: string[];
+        raw?: string;
+    };
+    children?: Node[];
+    raw?: string;
+}
+
+interface TextNode extends Node {
+    type: "text";
+    value: string;
+}
+
+type CompileMdxInput = Parameters<typeof compileMDX<Record<string, unknown>>>[0];
+type CompileMdxOptions = NonNullable<CompileMdxInput["options"]>;
+type CompilerMdxOptions = NonNullable<CompileMdxOptions["mdxOptions"]>;
+
+export type ParseMdxOptions = {
+    components?: CompileMdxInput["components"];
+    rehypePlugins?: CompilerMdxOptions["rehypePlugins"];
+    remarkPlugins?: CompilerMdxOptions["remarkPlugins"];
+};
+
+export const handleCodeTitles = () => (tree: Node) => {
+    visit(tree, "element", (node: Element, index: number | null, parent: Parent | null) => {
+        if (!parent || index === null || node.tagName !== "div") {
+            return;
+        }
+
+        const isTitleDiv = node.properties?.className?.includes("rehype-code-title");
+        if (!isTitleDiv) {
+            return;
+        }
+
+        let nextElement: Element | null = null;
+        for (let i = index + 1; i < parent.children.length; i++) {
+            const sibling = parent.children[i];
+            if (sibling.type === "element") {
+                nextElement = sibling as Element;
+                break;
+            }
+        }
+
+        if (nextElement?.tagName === "pre") {
+            const titleNode = node.children?.[0] as TextNode;
+            if (titleNode?.type === "text") {
+                if (!nextElement.properties) {
+                    nextElement.properties = {};
+                }
+                nextElement.properties["data-title"] = titleNode.value;
+                parent.children.splice(index, 1);
+                return index;
+            }
+        }
+    });
+};
+
+export const preProcess = () => (tree: Node) => {
+    visit(tree, (node: Node) => {
+        const element = node as Element;
+        if (element?.type === "element" && element?.tagName === "pre" && element.children) {
+            const [codeEl] = element.children as Element[];
+            if (codeEl.tagName !== "code" || !codeEl.children?.[0]) return;
+
+            const textNode = codeEl.children[0] as TextNode;
+            if (textNode.type === "text" && textNode.value) {
+                element.raw = textNode.value;
+            }
+        }
+    });
+};
+
+export const postProcess = () => (tree: Node) => {
+    visit(tree, "element", (node: Node) => {
+        const element = node as Element;
+        if (element?.type === "element" && element?.tagName === "pre") {
+            if (element.properties && element.raw) {
+                element.properties.raw = element.raw;
+            }
+        }
+    });
+};
+
+export function createDefaultRehypePlugins(): unknown[] {
+    return [
+        preProcess,
+        rehypeCodeTitles,
+        handleCodeTitles,
+        rehypePrism,
+        rehypeSlug,
+        rehypeAutolinkHeadings,
+        postProcess,
+    ];
+}
+
+export function createDefaultRemarkPlugins(): unknown[] {
+    return [remarkGfm];
+}
+
+export async function parseMdx<Frontmatter>(
+    rawMdx: string,
+    options: ParseMdxOptions = {}
+): Promise<MdxCompileResult<Frontmatter>> {
+    const rehypePlugins = options.rehypePlugins ?? (createDefaultRehypePlugins() as CompilerMdxOptions["rehypePlugins"]);
+    const remarkPlugins = options.remarkPlugins ?? (createDefaultRemarkPlugins() as CompilerMdxOptions["remarkPlugins"]);
+
+    return await compileMDX<Frontmatter>({
+        source: rawMdx,
+        options: {
+            parseFrontmatter: true,
+            mdxOptions: {
+                rehypePlugins,
+                remarkPlugins,
+            },
+        },
+        components: options.components,
+    });
+}

package/src/content.ts

@@ -0,0 +1,129 @@
+import path from "path";
+import { promises as fs } from "fs";
+import { extractFrontmatter, extractTocsFromRawMdx } from "./extract";
+import { parseMdx, type ParseMdxOptions } from "./compile";
+import type { MdxCompileResult, TocItem } from "./types";
+
+type CacheFn = <T extends (...args: any[]) => any>(fn: T) => T;
+
+export type ReadMdxFileResult = {
+    content: string;
+    filePath: string;
+};
+
+export type ParsedMdxFile<Frontmatter, T extends TocItem = TocItem> = {
+    content: string;
+    filePath: string;
+    frontmatter: Frontmatter;
+    tocs: T[];
+};
+
+export type CompiledMdxFile<Frontmatter, T extends TocItem = TocItem> = MdxCompileResult<Frontmatter> & {
+    filePath: string;
+    tocs: T[];
+};
+
+type ReadMdxBySlugOptions = {
+    rootDir?: string;
+    docsDir?: string;
+};
+
+export async function readMdxFileBySlug(slug: string, options: ReadMdxBySlugOptions = {}): Promise<ReadMdxFileResult> {
+    const docsDir = options.docsDir ?? "docs";
+    // Keep file-system path operations ignored by Turbopack tracing.
+    // The runtime path is constrained to docsDir and slug candidates below.
+    const docsRoot = options.rootDir
+        ? path.join(/*turbopackIgnore: true*/ options.rootDir, docsDir)
+        : path.join(/*turbopackIgnore: true*/ process.cwd(), docsDir);
+    const paths = [
+        path.join(/*turbopackIgnore: true*/ docsRoot, `${slug}.mdx`),
+        path.join(/*turbopackIgnore: true*/ docsRoot, slug, "index.mdx"),
+    ];
+
+    for (const p of paths) {
+        try {
+            const content = await fs.readFile(/*turbopackIgnore: true*/ p, "utf-8");
+            return {
+                content,
+                filePath: `${docsDir}/${path.relative(/*turbopackIgnore: true*/ docsRoot, p)}`,
+            };
+        } catch {
+            // ignore and try next candidate
+        }
+    }
+
+    throw new Error(`Could not find mdx file for slug: ${slug}`);
+}
+
+type ParseMdxFileOptions<T extends TocItem> = {
+    tocsExtractor?: (rawMdx: string) => T[];
+};
+
+export function parseMdxFile<Frontmatter, T extends TocItem = TocItem>(
+    raw: ReadMdxFileResult,
+    options: ParseMdxFileOptions<T> = {}
+): ParsedMdxFile<Frontmatter, T> {
+    const tocsExtractor = options.tocsExtractor ?? ((mdx) => extractTocsFromRawMdx(mdx) as T[]);
+
+    return {
+        content: raw.content,
+        filePath: raw.filePath,
+        frontmatter: extractFrontmatter<Frontmatter>(raw.content),
+        tocs: tocsExtractor(raw.content),
+    };
+}
+
+export async function compileParsedMdxFile<Frontmatter, T extends TocItem = TocItem>(
+    parsed: ParsedMdxFile<Frontmatter, T>,
+    options: ParseMdxOptions = {}
+): Promise<CompiledMdxFile<Frontmatter, T>> {
+    const compiled = await parseMdx<Frontmatter>(parsed.content, options);
+
+    return {
+        ...compiled,
+        frontmatter: parsed.frontmatter,
+        filePath: parsed.filePath,
+        tocs: parsed.tocs,
+    };
+}
+
+export type CreateMdxContentServiceOptions<Frontmatter, T extends TocItem = TocItem> = {
+    parseOptions?: ParseMdxOptions;
+    readOptions?: ReadMdxBySlugOptions;
+    tocsExtractor?: (rawMdx: string) => T[];
+    cacheFn?: CacheFn;
+};
+
+export function createMdxContentService<Frontmatter, T extends TocItem = TocItem>(
+    options: CreateMdxContentServiceOptions<Frontmatter, T> = {}
+) {
+    const identityCache: CacheFn = (fn) => fn;
+    const cacheFn = options.cacheFn ?? identityCache;
+
+    const getParsedForSlug = cacheFn(async (slug: string): Promise<ParsedMdxFile<Frontmatter, T>> => {
+        const raw = await readMdxFileBySlug(slug, options.readOptions);
+        return parseMdxFile<Frontmatter, T>(raw, { tocsExtractor: options.tocsExtractor });
+    });
+
+    const getCompiledForSlug = cacheFn(async (slug: string): Promise<CompiledMdxFile<Frontmatter, T>> => {
+        const parsed = await getParsedForSlug(slug);
+        return compileParsedMdxFile<Frontmatter, T>(parsed, options.parseOptions);
+    });
+
+    async function getFrontmatterForSlug(slug: string): Promise<Frontmatter> {
+        const parsed = await getParsedForSlug(slug);
+        return parsed.frontmatter;
+    }
+
+    async function getTocsForSlug(slug: string): Promise<T[]> {
+        const parsed = await getParsedForSlug(slug);
+        return parsed.tocs;
+    }
+
+    return {
+        getParsedForSlug,
+        getCompiledForSlug,
+        getFrontmatterForSlug,
+        getTocsForSlug,
+    };
+}

package/src/extract.ts

@@ -0,0 +1,44 @@
+import matter from "gray-matter";
+import type { TocItem } from "./types";
+
+export function sluggify(text: string): string {
+    const slug = text.toLowerCase().replace(/\s+/g, "-");
+    return slug.replace(/[^a-z0-9-]/g, "");
+}
+
+export function extractTocsFromRawMdx(rawMdx: string): TocItem[] {
+    // Match code blocks, markdown headings, and <Release version="x.y.z" />.
+    const combinedRegex = /(```[\s\S]*?```)|^(#{2,4})\s(.+)$|<Release[^>]*version="([^"]+)"/gm;
+    const extractedHeadings: TocItem[] = [];
+
+    let match: RegExpExecArray | null;
+    while ((match = combinedRegex.exec(rawMdx)) !== null) {
+        if (match[1]) continue;
+
+        if (match[2]) {
+            const headingLevel = match[2].length;
+            const headingText = match[3].trim();
+            extractedHeadings.push({
+                level: headingLevel,
+                text: headingText,
+                href: `#${sluggify(headingText)}`,
+            });
+            continue;
+        }
+
+        if (match[4]) {
+            const version = match[4];
+            extractedHeadings.push({
+                level: 2,
+                text: `v${version}`,
+                href: `#${version}`,
+            });
+        }
+    }
+
+    return extractedHeadings;
+}
+
+export function extractFrontmatter<Frontmatter>(content: string): Frontmatter {
+    return matter(content).data as Frontmatter;
+}

package/src/index.ts

@@ -0,0 +1,23 @@
+export type { TocItem, MdxCompileResult } from "./types";
+export {
+    parseMdx,
+    preProcess,
+    postProcess,
+    handleCodeTitles,
+    createDefaultRehypePlugins,
+    createDefaultRemarkPlugins,
+} from "./compile";
+export { extractFrontmatter, extractTocsFromRawMdx, sluggify } from "./extract";
+export type { ParseMdxOptions } from "./compile";
+export {
+    readMdxFileBySlug,
+    parseMdxFile,
+    compileParsedMdxFile,
+    createMdxContentService,
+} from "./content";
+export type {
+    ReadMdxFileResult,
+    ParsedMdxFile,
+    CompiledMdxFile,
+    CreateMdxContentServiceOptions,
+} from "./content";

package/src/types.ts

@@ -0,0 +1,13 @@
+import type { ReactNode } from "react";
+
+export type TocItem = {
+    level: number;
+    text: string;
+    href: string;
+};
+
+export type MdxCompileResult<Frontmatter> = {
+    content: ReactNode;
+    frontmatter: Frontmatter;
+    scope?: Record<string, unknown>;
+};

package/tsconfig.json

@@ -0,0 +1,15 @@
+{
+    "compilerOptions": {
+        "target": "ES2020",
+        "module": "ESNext",
+        "moduleResolution": "Bundler",
+        "strict": true,
+        "skipLibCheck": true,
+        "declaration": true,
+        "declarationMap": true,
+        "jsx": "preserve"
+    },
+    "include": [
+        "src/**/*"
+    ]
+}