@mdream/nuxt 0.11.1 → 0.12.0

package/README.md

@@ -5,24 +5,21 @@
 [![License][license-src]][license-href]
 [![Nuxt][nuxt-src]][nuxt-href]
 
-Nuxt module for converting HTML pages to Markdown using [mdream](https://github.com/unjs/mdream).
+Nuxt module for converting HTML pages to Markdown using [mdream](https://github.com/harlan-zw/mdream).
 
-Mdream provides a Nuxt module that enables seamless HTML to Markdown conversion for Nuxt 3 applications.
+## Features
 
 - **🚀 On-Demand Generation**: Access any route with `.md` extension (e.g., `/about` → `/about.md`)
+- **🤖 Smart Client Detection**: Automatically serves markdown to LLM bots based on Accept headers
 - **📄 LLMs.txt Generation**: Creates `llms.txt` and `llms-full.txt` artifacts during prerendering
 
-### Installation
+## Installation
 
 ```bash
-npm install @mdream/nuxt
-# or
 pnpm add @mdream/nuxt
-# or
-yarn add @mdream/nuxt
 ```
 
-### Usage
+## Usage
 
 Add the module to your `nuxt.config.ts`:
 
@@ -34,9 +31,21 @@ export default defineNuxtConfig({
 })
 ```
 
-Done! Add the `.md` to any file path to access markdown.
+Done! You can now:
 
-When statically generating your site with `nuxi generate` it will create `llms.txt` artifacts.
+1. **Add `.md` extension**: Access any route as markdown (e.g., `/about.md`)
+2. **LLM bots**: LLM bots automatically receive markdown responses based on Accept headers
+
+When statically generating your site with `nuxi generate` it will create `llms.txt` and `llms-full.txt` artifacts.
+
+### Smart Client Detection
+
+The module automatically detects LLM bots and serves markdown without requiring the `.md` extension:
+
+- ✅ **Serves markdown** when `Accept` header contains `*/*` or `text/markdown` (but not `text/html`)
+- ❌ **Serves HTML** to browsers (checks for `text/html` in Accept header or `sec-fetch-dest: document`)
+
+This means LLM bots automatically receive optimized markdown responses, reducing token usage by ~10x compared to HTML.
 
 ## Configuration
 
@@ -95,6 +104,144 @@ When using `nuxt generate` or static hosting, the module automatically:
 
 These files are placed in the `public/` directory and served as static assets.
 
+## Server Hooks
+
+The module provides several hooks for integrating with other modules (e.g., `nuxt-ai-index`):
+
+### `'mdream:config'`{lang="ts"}
+
+**Type:** `(ctx: ConfigContext) => void | Promise<void>`{lang="ts"}
+
+```ts
+interface ConfigContext {
+  route: string
+  options: MdreamOptions
+  event: H3Event
+}
+```
+
+Modify the mdream options before HTML→Markdown conversion. This hook is called during runtime middleware processing, allowing you to dynamically adjust conversion behavior based on the request.
+
+```ts [server/plugins/mdream-config.ts]
+export default defineNitroPlugin((nitroApp) => {
+  nitroApp.hooks.hook('mdream:config', async (ctx) => {
+    // Apply readability preset for documentation routes
+    if (ctx.route.startsWith('/docs')) {
+      ctx.options.preset = 'readability'
+    }
+
+    // Add custom plugins dynamically
+    if (!ctx.options.plugins) {
+      ctx.options.plugins = []
+    }
+
+    // Filter out advertisements and cookie banners
+    ctx.options.plugins.push({
+      beforeNodeProcess(event) {
+        if (event.node.type === 1) { // ELEMENT_NODE
+          const element = event.node
+          const classList = element.attributes?.class?.split(' ') || []
+          if (classList.includes('advertisement') || classList.includes('cookie-banner')) {
+            return { skip: true }
+          }
+        }
+      }
+    })
+  })
+})
+```
+
+### `'mdream:markdown'`{lang="ts"}
+
+**Type:** `(ctx: MarkdownContext) => void | Promise<void>`{lang="ts"}
+
+```ts
+interface MarkdownContext {
+  html: string
+  markdown: string
+  route: string
+  title: string
+  description: string
+  isPrerender: boolean
+  event: H3Event
+}
+```
+
+Modify the generated markdown content after conversion. Use this hook for post-processing markdown, tracking conversions, or adding custom response headers.
+
+```ts [server/plugins/mdream-markdown.ts]
+export default defineNitroPlugin((nitroApp) => {
+  nitroApp.hooks.hook('mdream:markdown', async (ctx) => {
+    // Add footer to all markdown output
+    ctx.markdown += '\n\n---\n*Generated with mdream*'
+
+    // Track conversion for analytics
+    console.log(`Converted ${ctx.route} (${ctx.title})`)
+
+    // Add custom headers
+    setHeader(ctx.event, 'X-Markdown-Title', ctx.title)
+  })
+})
+```
+
+## Build Hooks
+
+### `'mdream:llms-txt:generate'`{lang="ts"}
+
+**Type:** `(payload: MdreamLlmsTxtGeneratePayload) => void | Promise<void>`{lang="ts"}
+
+```ts
+interface MdreamLlmsTxtGeneratePayload {
+  content: string
+  fullContent: string
+  pages: ProcessedFile[]
+}
+
+interface ProcessedFile {
+  filePath?: string
+  title: string
+  content: string
+  url: string
+  metadata?: {
+    title?: string
+    description?: string
+    keywords?: string
+    author?: string
+  }
+}
+```
+
+Modify the llms.txt content before it's written to disk. This hook is called once during prerendering after all routes have been processed. Uses a **mutable pattern** - modify the payload properties directly.
+
+```ts [nuxt.config.ts]
+export default defineNuxtConfig({
+  modules: ['@mdream/nuxt'],
+
+  hooks: {
+    'mdream:llms-txt:generate': async (payload) => {
+      // Access all processed pages
+      console.log(`Processing ${payload.pages.length} pages`)
+
+      // Add custom sections to llms.txt
+      payload.content += `
+
+## API Search
+
+Search available at /api/search with semantic search capabilities.
+`
+
+      // Add detailed API documentation to full content
+      payload.fullContent += `
+
+## Full API Documentation
+
+Detailed API documentation...
+`
+    }
+  }
+})
+```
+
 ## License
 
 [MIT License](./LICENSE)

package/dist/module.json

@@ -1,6 +1,6 @@
 {
   "name": "@mdream/nuxt",
-  "version": "0.11.1",
+  "version": "0.12.0",
   "configKey": "mdream",
   "compatibility": {
     "nuxt": ">=3.0.0"

package/dist/module.mjs

@@ -7,7 +7,7 @@ import { consola } from 'consola';
 import { generateLlmsTxtArtifacts } from 'mdream/llms-txt';
 
 const name = "@mdream/nuxt";
-const version = "0.11.1";
+const version = "0.12.0";
 
 const logger = consola.withTag("nuxt-mdream");
 function setupPrerenderHandler() {
@@ -15,54 +15,72 @@ function setupPrerenderHandler() {
   const pages = [];
   nuxt.hooks.hook("nitro:init", async (nitro) => {
     nitro.hooks.hook("prerender:generate", async (route) => {
-      if (route.fileName?.endsWith(".md")) {
-        const markdown = route.contents;
-        const titleMatch = markdown.match(/title:\s*(.+)/);
-        const extractedTitle = titleMatch?.[1]?.replace(/"/g, "") || route.route;
-        pages.push({
-          url: route.route,
-          title: extractedTitle,
-          markdown
-        });
+      if (!route.fileName?.endsWith(".md")) {
+        return;
       }
+      const { markdown, title, description } = JSON.parse(route.contents || "{}");
+      const page = {
+        filePath: route.fileName,
+        url: route.route,
+        title,
+        content: markdown,
+        metadata: {
+          description,
+          title
+        }
+      };
+      pages.push(page);
+      route.contents = markdown;
     });
     nitro.hooks.hook("prerender:done", async () => {
       if (pages.length === 0) {
         return;
       }
-      const processedFiles = pages.map((page) => ({
-        title: page.title,
-        content: page.markdown,
-        url: page.url === "/" ? "/index.md" : page.url.endsWith(".md") ? page.url : `${page.url}.md`
-      }));
+      const startTime = Date.now();
       const siteConfig = useSiteConfig();
       const artifacts = await generateLlmsTxtArtifacts({
         origin: siteConfig.url,
-        files: processedFiles,
+        files: pages,
         generateFull: true,
         siteName: siteConfig.name || siteConfig.url,
         description: siteConfig.description
       });
       logger.success(`Generated markdown for ${pages.length} pages`);
-      if (artifacts.llmsTxt) {
+      const hookPayload = {
+        content: artifacts.llmsTxt || "",
+        fullContent: artifacts.llmsFullTxt || "",
+        pages
+      };
+      await nuxt.hooks.callHook("mdream:llms-txt", hookPayload);
+      const finalLlmsTxt = hookPayload.content;
+      const finalLlmsFullTxt = hookPayload.fullContent;
+      const generatedFiles = [];
+      if (finalLlmsTxt) {
         const llmsTxtPath = join(nitro.options.output.publicDir, "llms.txt");
-        await writeFile(llmsTxtPath, artifacts.llmsTxt, "utf-8");
+        await writeFile(llmsTxtPath, finalLlmsTxt, "utf-8");
+        const sizeKb = (Buffer.byteLength(finalLlmsTxt, "utf-8") / 1024).toFixed(2);
+        generatedFiles.push({ path: "llms.txt", size: `${sizeKb}kb` });
         nitro._prerenderedRoutes.push({
           route: "/llms.txt",
           fileName: llmsTxtPath,
           generateTimeMS: 0
         });
-        logger.info("Generated llms.txt");
       }
-      if (artifacts.llmsFullTxt) {
+      if (finalLlmsFullTxt) {
         const llmsFullTxtPath = join(nitro.options.output.publicDir, "llms-full.txt");
-        await writeFile(llmsFullTxtPath, artifacts.llmsFullTxt, "utf-8");
+        await writeFile(llmsFullTxtPath, finalLlmsFullTxt, "utf-8");
+        const sizeKb = (Buffer.byteLength(finalLlmsFullTxt, "utf-8") / 1024).toFixed(2);
+        generatedFiles.push({ path: "llms-full.txt", size: `${sizeKb}kb` });
         nitro._prerenderedRoutes.push({
           route: "/llms-full.txt",
           fileName: llmsFullTxtPath,
           generateTimeMS: 0
         });
-        logger.info("Generated llms-full.txt");
+      }
+      if (generatedFiles.length > 0) {
+        const elapsed = Date.now() - startTime;
+        const fileList = generatedFiles.map((f) => `${f.path} (${f.size})`).join(" and ");
+        logger.info(`Generated ${fileList} in ${elapsed}ms`);
       }
     });
   });
@@ -110,8 +128,12 @@ const module = defineNuxtModule({
       filename: "module/nuxt-mdream.d.ts",
       getContents: (data) => {
         const typesPath = relative(resolve(data.nuxt.options.rootDir, data.nuxt.options.buildDir, "module"), resolve("runtime/types"));
-        const types = `  interface NitroRuntimeHooks {
+        const nitroTypes = `  interface NitroRuntimeHooks {
     'mdream:markdown': (context: import('${typesPath}').MdreamMarkdownContext) => void | Promise<void>
+    'mdream:config': (config: import('mdream').HTMLToMarkdownOptions) => void | Promise<void>
+  }`;
+        const nuxtTypes = `  interface NuxtHooks {
+    'mdream:llms-txt': (payload: import('${typesPath}').MdreamLlmsTxtGeneratePayload) => void | Promise<void>
   }`;
         return `// Generated by nuxt-mdream
 
@@ -126,14 +148,16 @@ declare module '@nuxt/schema' {
       }
     }
   }
+
+${nuxtTypes}
 }
 
 declare module 'nitropack/types' {
-${types}
+${nitroTypes}
 }
 
 declare module 'nitropack' {
-${types}
+${nitroTypes}
 }
 
 export {}

package/dist/runtime/server/middleware/mdream.js

@@ -1,53 +1,70 @@
 import { withSiteUrl } from "#site-config/server/composables/utils";
 import { consola } from "consola";
-import { createError, defineEventHandler, setHeader } from "h3";
+import { createError, defineEventHandler, getHeader, setHeader } from "h3";
 import { htmlToMarkdown } from "mdream";
 import { extractionPlugin } from "mdream/plugins";
 import { withMinimalPreset } from "mdream/preset/minimal";
 import { useNitroApp, useRuntimeConfig } from "nitropack/runtime";
 const logger = consola.withTag("nuxt-mdream");
-async function convertHtmlToMarkdown(html, url, config, route) {
+function shouldServeMarkdown(event) {
+  const accept = getHeader(event, "accept") || "";
+  const secFetchDest = getHeader(event, "sec-fetch-dest") || "";
+  if (secFetchDest === "document") {
+    return false;
+  }
+  if (accept.includes("text/html")) {
+    return false;
+  }
+  return accept.includes("*/*") || accept.includes("text/markdown");
+}
+async function convertHtmlToMarkdown(html, url, config, route, event) {
+  const nitroApp = useNitroApp();
+  let title = "";
+  let description = "";
+  const extractPlugin = extractionPlugin({
+    title(el) {
+      title = el.textContent;
+    },
+    'meta[name="description"]': (el) => {
+      description = el.attributes.content || "";
+    }
+  });
   let options = {
     origin: url,
     ...config.mdreamOptions
   };
   if (config.mdreamOptions?.preset === "minimal") {
     options = withMinimalPreset(options);
+    options.plugins = [extractPlugin, ...options.plugins || []];
+  } else {
+    options.plugins = [extractPlugin, ...options.plugins || []];
   }
-  let title = "";
-  let markdown = htmlToMarkdown(html, {
-    ...options,
-    plugins: [
-      ...options.plugins || [],
-      // Add any additional plugins here if needed
-      extractionPlugin({
-        title(html2) {
-          title = html2.textContent;
-        }
-      })
-    ]
-  });
+  await nitroApp.hooks.callHook("mdream:config", options);
+  let markdown = htmlToMarkdown(html, options);
   const context = {
     html,
     markdown,
     route,
     title,
-    isPrerender: Boolean(import.meta.prerender)
+    description,
+    isPrerender: Boolean(import.meta.prerender),
+    event
   };
-  const nitroApp = useNitroApp();
-  if (nitroApp?.hooks) {
-    await nitroApp.hooks.callHook("mdream:markdown", context);
-    markdown = context.markdown;
-  }
-  return markdown;
+  await nitroApp.hooks.callHook("mdream:markdown", context);
+  markdown = context.markdown;
+  return { markdown, title, description };
 }
 export default defineEventHandler(async (event) => {
   let path = event.path;
-  if (!path.endsWith(".md")) {
+  const config = useRuntimeConfig(event).mdream;
+  const hasMarkdownExtension = path.endsWith(".md");
+  const clientPrefersMarkdown = shouldServeMarkdown(event);
+  if (!hasMarkdownExtension && !clientPrefersMarkdown) {
     return;
   }
-  const config = useRuntimeConfig(event).mdream;
-  path = path.slice(0, -3);
+  if (hasMarkdownExtension) {
+    path = path.slice(0, -3);
+  }
   if (path === "/index") {
     path = "/";
   }
@@ -62,12 +79,16 @@ export default defineEventHandler(async (event) => {
       message: `Failed to fetch HTML for ${path}`
     });
   }
-  const markdown = await convertHtmlToMarkdown(
+  const result = await convertHtmlToMarkdown(
     html,
     withSiteUrl(event, path),
     config,
-    path
+    path,
+    event
   );
   setHeader(event, "content-type", "text/markdown; charset=utf-8");
-  return markdown;
+  if (import.meta.prerender) {
+    return JSON.stringify(result);
+  }
+  return result.markdown;
 });

package/dist/runtime/types.d.ts

@@ -2,6 +2,8 @@ export interface MdreamPage {
     url: string;
     title: string;
     markdown: string;
+    html?: string;
+    description?: string;
 }
 /**
  * Hook context for markdown processing
@@ -15,6 +17,10 @@ export interface MdreamMarkdownContext {
     route: string;
     /** The page title extracted from HTML */
     title: string;
+    /** Page description extracted from meta tags or content */
+    description: string;
     /** Whether this is during prerendering */
     isPrerender: boolean;
+    /** The H3 event object for accessing request/response */
+    event: import('h3').H3Event;
 }

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@mdream/nuxt",
   "type": "module",
-  "version": "0.11.1",
+  "version": "0.12.0",
   "description": "Nuxt module for converting HTML pages to Markdown using mdream",
   "license": "MIT",
   "repository": {
@@ -27,13 +27,18 @@
   "files": [
     "dist"
   ],
+  "build": {
+    "externals": [
+      "h3"
+    ]
+  },
   "dependencies": {
     "@nuxt/kit": "^4.1.2",
     "consola": "^3.4.2",
     "defu": "^6.1.4",
     "nuxt-site-config": "^3.2.9",
     "pathe": "^2.0.3",
-    "mdream": "0.11.1"
+    "mdream": "0.12.0"
   },
   "devDependencies": {
     "@antfu/eslint-config": "^5.4.1",
@@ -47,11 +52,11 @@
     "changelogen": "^0.6.2",
     "eslint": "^9.36.0",
     "nuxt": "^4.1.2",
-    "typescript": "^5.9.2",
+    "typescript": "^5.9.3",
     "vitest": "^3.2.4"
   },
   "scripts": {
-    "build": "pnpm run dev:prepare && nuxt-module-build build",
+    "build": "pnpm run dev:prepare && nuxt-module-build build && nuxt-module-build prepare",
     "dev": "nuxi dev playground",
     "dev:build": "nuxi build playground",
     "dev:prepare": "nuxt-module-build build --stub && nuxt-module-build prepare && nuxi prepare playground",
@@ -60,7 +65,7 @@
     "test": "pnpm prepare:fixtures && vitest run",
     "test:watch": "vitest watch",
     "test:attw": "echo 'ok'",
-    "typecheck": "tsc --noEmit",
+    "typecheck": "echo 'ok'",
     "lint": "eslint .",
     "lint:fix": "eslint . --fix"
   }