@mdream/nuxt 0.8.0

package/LICENSE.md

@@ -0,0 +1,9 @@
+MIT License
+
+Copyright (c) 2025 Harlan Wilton
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

package/README.md

@@ -0,0 +1,113 @@
+# @mdream/nuxt
+
+[![npm version][npm-version-src]][npm-version-href]
+[![npm downloads][npm-downloads-src]][npm-downloads-href]
+[![License][license-src]][license-href]
+[![Nuxt][nuxt-src]][nuxt-href]
+
+Nuxt module for converting HTML pages to Markdown using [mdream](https://github.com/unjs/mdream).
+
+Mdream provides a Nuxt module that enables seamless HTML to Markdown conversion for Nuxt 3 applications.
+
+- **🚀 On-Demand Generation**: Access any route with `.md` extension (e.g., `/about` → `/about.md`)
+- **📄 LLMs.txt Generation**: Creates `llms.txt` and `llms-full.txt` artifacts during prerendering
+
+### Installation
+
+```bash
+npm install @mdream/nuxt mdream
+# or
+pnpm add @mdream/nuxt mdream
+# or
+yarn add @mdream/nuxt mdream
+```
+
+### Usage
+
+Add the module to your `nuxt.config.ts`:
+
+```ts
+export default defineNuxtConfig({
+  modules: [
+    '@mdream/nuxt'
+  ],
+})
+```
+
+Done! Add the `.md` to any file path to access markdown.
+
+When statically generating your site with `nuxi generate` it will create `llms.txt` artifacts.
+
+## Configuration
+
+Configure the module in your `nuxt.config.ts`:
+
+```ts
+export default defineNuxtConfig({
+  modules: ['@mdream/nuxt'],
+
+  mdream: {
+    // Enable/disable the module
+    enabled: true,
+
+    // Pass options to mdream
+    mdreamOptions: {
+      // mdream conversion options
+    },
+
+    // Cache configuration (production only)
+    cache: {
+      maxAge: 3600, // 1 hour
+      swr: true // Stale-while-revalidate
+    }
+  }
+})
+```
+
+### Options
+
+- `enabled` (boolean): Enable or disable the module. Default: `true`
+- `mdreamOptions` (object): Options passed to mdream's `htmlToMarkdown` function
+- `cache.maxAge` (number): Cache duration in seconds. Default: `3600` (1 hour)
+- `cache.swr` (boolean): Enable stale-while-revalidate. Default: `true`
+
+## Robot Meta Tag Support
+
+The module respects the `robots` meta tag. Pages with `noindex` will return a 404 error when accessed as markdown:
+
+```vue
+<script setup>
+useHead({
+  meta: [
+    { name: 'robots', content: 'noindex' }
+  ]
+})
+</script>
+```
+
+## Static Generation
+
+When using `nuxt generate` or static hosting, the module automatically:
+
+1. Generates `.md` files for all pages
+2. Creates `llms.txt` with page listings
+3. Creates `llms-full.txt` with full content
+
+These files are placed in the `public/` directory and served as static assets.
+
+## License
+
+[MIT License](./LICENSE)
+
+<!-- Badges -->
+[npm-version-src]: https://img.shields.io/npm/v/@mdream/nuxt/latest.svg?style=flat&colorA=020420&colorB=00DC82
+[npm-version-href]: https://npmjs.com/package/@mdream/nuxt
+
+[npm-downloads-src]: https://img.shields.io/npm/dm/@mdream/nuxt.svg?style=flat&colorA=020420&colorB=00DC82
+[npm-downloads-href]: https://npm.chart.dev/@mdream/nuxt
+
+[license-src]: https://img.shields.io/npm/l/@mdream/nuxt.svg?style=flat&colorA=020420&colorB=00DC82
+[license-href]: https://npmjs.com/package/@mdream/nuxt
+
+[nuxt-src]: https://img.shields.io/badge/Nuxt-020420?logo=nuxt.js
+[nuxt-href]: https://nuxt.com

package/dist/module.d.mts

@@ -0,0 +1,38 @@
+import * as _nuxt_schema from '@nuxt/schema';
+import { HTMLToMarkdownOptions } from 'mdream';
+
+interface ModuleOptions {
+    /**
+     * Enable/disable the module
+     * @default true
+     */
+    enabled?: boolean;
+    /**
+     * Options to pass to mdream htmlToMarkdown function
+     */
+    mdreamOptions?: HTMLToMarkdownOptions & {
+        /**
+         * Preset to apply to the htmlToMarkdown function
+         */
+        preset?: 'minimal';
+    };
+    /**
+     * Cache configuration
+     */
+    cache?: {
+        /**
+         * Cache duration in seconds
+         * @default 3600 (1 hour)
+         */
+        maxAge?: number;
+        /**
+         * Enable stale-while-revalidate
+         * @default true
+         */
+        swr?: boolean;
+    };
+}
+
+declare const _default: _nuxt_schema.NuxtModule<ModuleOptions, ModuleOptions, false>;
+
+export { _default as default };

package/dist/module.json

@@ -0,0 +1,12 @@
+{
+  "name": "@mdream/nuxt",
+  "version": "0.8.0",
+  "configKey": "mdream",
+  "compatibility": {
+    "nuxt": "^3.0.0"
+  },
+  "builder": {
+    "@nuxt/module-builder": "1.0.1",
+    "unbuild": "3.5.0"
+  }
+}

package/dist/module.mjs

@@ -0,0 +1,178 @@
+import { useNuxt, defineNuxtModule, createResolver, addTypeTemplate, addServerHandler } from '@nuxt/kit';
+import { defu } from 'defu';
+import { useSiteConfig, installNuxtSiteConfig } from 'nuxt-site-config/kit';
+import { join, dirname, relative, resolve } from 'node:path';
+import { mkdir, writeFile } from 'node:fs/promises';
+import { consola } from 'consola';
+import { htmlToMarkdown, generateLlmsTxtArtifacts } from 'mdream';
+
+const name = "@mdream/nuxt";
+const version = "0.8.0";
+
+const logger = consola.withTag("nuxt-mdream");
+function isIndexable(html) {
+  const robotsMatch = html.match(/<meta\s+name=["']robots["']\s+content=["']([^"']+)["']/i);
+  if (robotsMatch) {
+    const content = robotsMatch[1].toLowerCase();
+    return !content.includes("noindex");
+  }
+  return true;
+}
+function setupPrerenderHandler(config, nuxt = useNuxt()) {
+  const pages = [];
+  nuxt.hooks.hook("nitro:init", async (nitro) => {
+    nitro.hooks.hook("prerender:generate", async (route) => {
+      if (!route.fileName?.endsWith(".html") || !route.contents) {
+        return;
+      }
+      if (["/200.html", "/404.html"].includes(route.route)) {
+        return;
+      }
+      const html = route.contents;
+      if (!isIndexable(html)) {
+        return;
+      }
+      try {
+        const markdown = htmlToMarkdown(html, {
+          origin: route.route,
+          ...config.mdreamOptions
+        });
+        const titleMatch = html.match(/<title[^>]*>([^<]+)<\/title>/i);
+        const title = titleMatch ? titleMatch[1].trim() : route.route;
+        pages.push({
+          url: route.route,
+          title,
+          markdown
+        });
+        const mdPath = route.route === "/" ? "/index.md" : `${route.route}.md`;
+        const outputPath = join(nitro.options.output.publicDir, mdPath);
+        await mkdir(dirname(outputPath), { recursive: true });
+        await writeFile(outputPath, markdown, "utf-8");
+      } catch (error) {
+        logger.warn(`Failed to convert ${route.route} to markdown:`, error);
+      }
+    });
+    nitro.hooks.hook("prerender:done", async () => {
+      if (pages.length === 0) {
+        return;
+      }
+      try {
+        const processedFiles = pages.map((page) => ({
+          title: page.title,
+          content: page.markdown,
+          url: page.url === "/" ? "/index.md" : `${page.url}.md`
+        }));
+        const siteConfig = useSiteConfig();
+        const artifacts = await generateLlmsTxtArtifacts({
+          files: processedFiles,
+          generateFull: true,
+          siteName: siteConfig.name || siteConfig.url,
+          description: siteConfig.description
+        });
+        if (artifacts.llmsTxt) {
+          const llmsTxtPath = join(nitro.options.output.publicDir, "llms.txt");
+          await writeFile(llmsTxtPath, artifacts.llmsTxt, "utf-8");
+        }
+        if (artifacts.llmsFullTxt) {
+          const llmsFullTxtPath = join(nitro.options.output.publicDir, "llms-full.txt");
+          await writeFile(llmsFullTxtPath, artifacts.llmsFullTxt, "utf-8");
+        }
+        logger.success(`Generated markdown for ${pages.length} pages`);
+        if (artifacts.llmsTxt) {
+          logger.info("Generated llms.txt");
+        }
+        if (artifacts.llmsFullTxt) {
+          logger.info("Generated llms-full.txt");
+        }
+      } catch (error) {
+        logger.error("Failed to generate llms.txt artifacts:", error);
+      }
+    });
+  });
+}
+
+const module = defineNuxtModule({
+  meta: {
+    name,
+    version,
+    configKey: "mdream",
+    compatibility: {
+      nuxt: "^3.0.0"
+    }
+  },
+  defaults: {
+    enabled: true,
+    mdreamOptions: {},
+    cache: {
+      maxAge: 3600,
+      // 1 hour
+      swr: true
+    }
+  },
+  async setup(options, nuxt) {
+    if (!options.enabled) {
+      return;
+    }
+    const resolver = createResolver(import.meta.url);
+    await installNuxtSiteConfig();
+    const runtimeConfig = {
+      enabled: options.enabled,
+      mdreamOptions: options.mdreamOptions,
+      cache: defu(options.cache, {
+        maxAge: 3600,
+        swr: true
+      })
+    };
+    nuxt.options.runtimeConfig.mdream = defu(
+      nuxt.options.runtimeConfig.mdream,
+      runtimeConfig
+    );
+    addTypeTemplate({
+      filename: "module/nuxt-mdream.d.ts",
+      getContents: (data) => {
+        const typesPath = relative(resolve(data.nuxt.options.rootDir, data.nuxt.options.buildDir, "module"), resolve("runtime/types"));
+        const types = `  interface NitroRuntimeHooks {
+    'mdream:markdown': (context: import('${typesPath}').MdreamMarkdownContext) => void | Promise<void>
+  }`;
+        return `// Generated by nuxt-mdream
+
+declare module '@nuxt/schema' {
+  interface RuntimeConfig {
+    mdream: {
+      enabled: boolean
+      mdreamOptions: Record<string, any>
+      cache: {
+        maxAge: number
+        swr: boolean
+      }
+    }
+  }
+}
+
+declare module 'nitropack/types' {
+${types}
+}
+
+declare module 'nitropack' {
+${types}
+}
+
+export {}
+`;
+      }
+    }, {
+      nitro: true,
+      nuxt: true
+    });
+    addServerHandler({
+      middleware: true,
+      handler: resolver.resolve("./runtime/server/middleware/mdream")
+    });
+    const isStatic = nuxt.options.nitro.static || nuxt.options._generate || false;
+    if (isStatic || nuxt.options.nitro.prerender?.routes?.length) {
+      setupPrerenderHandler(runtimeConfig, nuxt);
+    }
+  }
+});
+
+export { module as default };

package/dist/runtime/server/middleware/mdream.d.ts

@@ -0,0 +1,2 @@
+declare const _default: import("h3").EventHandler<import("h3").EventHandlerRequest, Promise<string | undefined>>;
+export default _default;

package/dist/runtime/server/middleware/mdream.js

@@ -0,0 +1,102 @@
+import { consola } from "consola";
+import { createError, defineEventHandler, getRequestURL, setHeader } from "h3";
+import { htmlToMarkdown } from "mdream";
+import { withMinimalPreset } from "mdream/preset/minimal";
+import { useNitroApp, useRuntimeConfig } from "nitropack/runtime";
+const logger = consola.withTag("nuxt-mdream");
+function isIndexable(html) {
+  const robotsMatch = html.match(/<meta\s+name=["']robots["']\s+content=["']([^"']+)["']/i);
+  if (robotsMatch) {
+    const content = robotsMatch[1].toLowerCase();
+    return !content.includes("noindex");
+  }
+  return true;
+}
+async function convertHtmlToMarkdown(html, url, config, route, title) {
+  let options = {
+    origin: url,
+    ...config.mdreamOptions
+  };
+  if (config.mdreamOptions?.preset === "minimal") {
+    options = withMinimalPreset(options);
+  }
+  let markdown = htmlToMarkdown(html, options);
+  const context = {
+    html,
+    markdown,
+    route,
+    title,
+    isPrerender: false
+  };
+  try {
+    const nitroApp = useNitroApp();
+    if (nitroApp?.hooks) {
+      await nitroApp.hooks.callHook("mdream:markdown", context);
+      markdown = context.markdown;
+    }
+  } catch (error) {
+    logger.debug("Could not call mdream:markdown hook:", error);
+  }
+  return markdown;
+}
+export default defineEventHandler(async (event) => {
+  const requestUrl = getRequestURL(event);
+  let htmlPath = requestUrl.pathname;
+  if (!htmlPath.endsWith(".md")) {
+    return;
+  }
+  const config = useRuntimeConfig(event).mdream;
+  htmlPath = htmlPath.slice(0, -3);
+  if (htmlPath === "/index") {
+    htmlPath = "/";
+  }
+  try {
+    const fullUrl = new URL(htmlPath, requestUrl.origin).toString();
+    const response = await fetch(fullUrl, {
+      headers: {
+        // Forward important headers
+        "user-agent": event.headers.get("user-agent") || "",
+        "accept-language": event.headers.get("accept-language") || ""
+      }
+    });
+    if (!response.ok) {
+      throw createError({
+        statusCode: response.status,
+        statusMessage: response.statusText
+      });
+    }
+    const html = await response.text();
+    if (!isIndexable(html)) {
+      throw createError({
+        statusCode: 404,
+        statusMessage: "Page not indexable"
+      });
+    }
+    const titleMatch = html.match(/<title[^>]*>([^<]+)<\/title>/i);
+    const title = titleMatch ? titleMatch[1].trim() : htmlPath;
+    const markdown = await convertHtmlToMarkdown(
+      html,
+      requestUrl.origin + htmlPath,
+      config,
+      htmlPath,
+      title
+    );
+    setHeader(event, "content-type", "text/markdown; charset=utf-8");
+    return markdown;
+  } catch (error) {
+    if (error.statusCode) {
+      throw error;
+    }
+    if (error.status === 404) {
+      throw createError({
+        statusCode: 404,
+        statusMessage: "Page not found"
+      });
+    }
+    logger.error("Failed to generate markdown for", htmlPath, error);
+    throw createError({
+      statusCode: 500,
+      statusMessage: `Failed to generate markdown: ${error.message || "Unknown error"}`
+    });
+  }
+});

package/dist/runtime/server/tsconfig.json

@@ -0,0 +1,3 @@
+{
+  "extends": "../../../.nuxt/tsconfig.server.json"
+}

package/dist/runtime/types.d.ts

@@ -0,0 +1,20 @@
+export interface MdreamPage {
+    url: string;
+    title: string;
+    markdown: string;
+}
+/**
+ * Hook context for markdown processing
+ */
+export interface MdreamMarkdownContext {
+    /** The original HTML content */
+    html: string;
+    /** The generated markdown content */
+    markdown: string;
+    /** The route being processed */
+    route: string;
+    /** The page title extracted from HTML */
+    title: string;
+    /** Whether this is during prerendering */
+    isPrerender: boolean;
+}

package/dist/types.d.mts

@@ -0,0 +1,7 @@
+import type { NuxtModule } from '@nuxt/schema'
+
+import type { default as Module } from './module.mjs'
+
+export type ModuleOptions = typeof Module extends NuxtModule<infer O> ? Partial<O> : Record<string, any>
+
+export { default } from './module.mjs'

package/package.json

@@ -0,0 +1,65 @@
+{
+  "name": "@mdream/nuxt",
+  "type": "module",
+  "version": "0.8.0",
+  "description": "Nuxt module for converting HTML pages to Markdown using mdream",
+  "license": "MIT",
+  "exports": {
+    ".": {
+      "types": "./dist/types.d.mts",
+      "import": "./dist/module.mjs"
+    }
+  },
+  "main": "./dist/module.mjs",
+  "types": "./dist/types.d.mts",
+  "typesVersions": {
+    "*": {
+      ".": [
+        "./dist/types.d.mts"
+      ]
+    }
+  },
+  "files": [
+    "dist"
+  ],
+  "peerDependencies": {
+    "nuxt": "^3.0.0"
+  },
+  "dependencies": {
+    "@nuxt/kit": "^3.17.6",
+    "consola": "^3.4.2",
+    "defu": "^6.1.4",
+    "nuxt-site-config": "^3.2.2",
+    "pathe": "^2.0.3",
+    "mdream": "0.8.0"
+  },
+  "devDependencies": {
+    "@antfu/eslint-config": "^4.16.2",
+    "@arethetypeswrong/cli": "^0.18.2",
+    "@nuxt/devtools": "latest",
+    "@nuxt/module-builder": "^1.0.1",
+    "@nuxt/schema": "^3.17.6",
+    "@nuxt/test-utils": "^3.19.2",
+    "@types/node": "latest",
+    "bumpp": "^10.2.0",
+    "changelogen": "^0.6.2",
+    "eslint": "^9.30.1",
+    "nuxt": "^3.17.6",
+    "typescript": "^5.8.3",
+    "vitest": "^3.2.4"
+  },
+  "scripts": {
+    "build": "pnpm run dev:prepare && nuxt-module-build build",
+    "dev": "nuxi dev playground",
+    "dev:build": "nuxi build playground",
+    "dev:prepare": "nuxt-module-build build --stub && nuxt-module-build prepare && nuxi prepare playground",
+    "prepare:fixtures": "nuxi prepare test/fixtures/basic",
+    "release": "pnpm build && bumpp && pnpm -r publish",
+    "test": "pnpm prepare:fixtures && vitest run",
+    "test:watch": "vitest watch",
+    "test:attw": "attw --pack",
+    "typecheck": "tsc --noEmit",
+    "lint": "eslint .",
+    "lint:fix": "eslint . --fix"
+  }
+}