@mdream/nuxt 0.8.1 → 0.8.2

package/README.md

@@ -15,11 +15,11 @@ Mdream provides a Nuxt module that enables seamless HTML to Markdown conversion
 ### Installation
 
 ```bash
-npm install @mdream/nuxt mdream
+npm install @mdream/nuxt
 # or
-pnpm add @mdream/nuxt mdream
+pnpm add @mdream/nuxt
 # or
-yarn add @mdream/nuxt mdream
+yarn add @mdream/nuxt
 ```
 
 ### Usage

package/dist/module.json

@@ -1,6 +1,6 @@
 {
   "name": "@mdream/nuxt",
-  "version": "0.8.0",
+  "version": "0.8.1",
   "configKey": "mdream",
   "compatibility": {
     "nuxt": ">=3.0.0"

package/dist/module.mjs

@@ -1,91 +1,68 @@
-import { join, dirname, relative, resolve } from 'node:path';
-import { useNuxt, defineNuxtModule, createResolver, addTypeTemplate, addServerHandler } from '@nuxt/kit';
+import { join, relative, resolve } from 'node:path';
+import { useNuxt, defineNuxtModule, createResolver, addTypeTemplate, addServerHandler, addPlugin } from '@nuxt/kit';
 import { defu } from 'defu';
 import { useSiteConfig, installNuxtSiteConfig } from 'nuxt-site-config/kit';
-import { mkdir, writeFile } from 'node:fs/promises';
+import { writeFile } from 'node:fs/promises';
 import { consola } from 'consola';
-import { htmlToMarkdown, generateLlmsTxtArtifacts } from 'mdream';
+import { generateLlmsTxtArtifacts } from 'mdream';
 
 const name = "@mdream/nuxt";
-const version = "0.8.0";
+const version = "0.8.1";
 
 const logger = consola.withTag("nuxt-mdream");
-function isIndexable(html) {
-  const robotsMatch = html.match(/<meta\s+name=["']robots["']\s+content=["']([^"']+)["']/i);
-  if (robotsMatch) {
-    const content = String(robotsMatch[1]).toLowerCase();
-    return !content.includes("noindex");
-  }
-  return true;
-}
-function setupPrerenderHandler(config, nuxt = useNuxt()) {
+function setupPrerenderHandler() {
+  const nuxt = useNuxt();
   const pages = [];
   nuxt.hooks.hook("nitro:init", async (nitro) => {
     nitro.hooks.hook("prerender:generate", async (route) => {
-      if (!route.fileName?.endsWith(".html") || !route.contents) {
-        return;
-      }
-      if (["/200.html", "/404.html"].includes(route.route)) {
-        return;
-      }
-      const html = route.contents;
-      if (!isIndexable(html)) {
-        return;
-      }
-      try {
-        const markdown = htmlToMarkdown(html, {
-          origin: route.route,
-          ...config.mdreamOptions
-        });
-        const titleMatch = html.match(/<title[^>]*>([^<]+)<\/title>/i);
-        const title = titleMatch ? String(titleMatch[1]).trim() : route.route;
+      if (route.fileName?.endsWith(".md")) {
+        const markdown = route.contents;
+        const titleMatch = markdown.match(/title:\s*(.+)/);
+        const extractedTitle = titleMatch?.[1]?.replace(/"/g, "") || route.route;
         pages.push({
           url: route.route,
-          title,
+          title: extractedTitle,
           markdown
         });
-        const mdPath = route.route === "/" ? "/index.md" : `${route.route}.md`;
-        const outputPath = join(nitro.options.output.publicDir, mdPath);
-        await mkdir(dirname(outputPath), { recursive: true });
-        await writeFile(outputPath, markdown, "utf-8");
-      } catch (error) {
-        logger.warn(`Failed to convert ${route.route} to markdown:`, error);
       }
     });
     nitro.hooks.hook("prerender:done", async () => {
       if (pages.length === 0) {
         return;
       }
-      try {
-        const processedFiles = pages.map((page) => ({
-          title: page.title,
-          content: page.markdown,
-          url: page.url === "/" ? "/index.md" : `${page.url}.md`
-        }));
-        const siteConfig = useSiteConfig();
-        const artifacts = await generateLlmsTxtArtifacts({
-          files: processedFiles,
-          generateFull: true,
-          siteName: siteConfig.name || siteConfig.url,
-          description: siteConfig.description
+      const processedFiles = pages.map((page) => ({
+        title: page.title,
+        content: page.markdown,
+        url: page.url === "/" ? "/index.md" : `${page.url}.md`
+      }));
+      const siteConfig = useSiteConfig();
+      const artifacts = await generateLlmsTxtArtifacts({
+        origin: siteConfig.url,
+        files: processedFiles,
+        generateFull: true,
+        siteName: siteConfig.name || siteConfig.url,
+        description: siteConfig.description
+      });
+      logger.success(`Generated markdown for ${pages.length} pages`);
+      if (artifacts.llmsTxt) {
+        const llmsTxtPath = join(nitro.options.output.publicDir, "llms.txt");
+        await writeFile(llmsTxtPath, artifacts.llmsTxt, "utf-8");
+        nitro._prerenderedRoutes.push({
+          route: "/llms.txt",
+          fileName: llmsTxtPath,
+          generateTimeMS: 0
         });
-        if (artifacts.llmsTxt) {
-          const llmsTxtPath = join(nitro.options.output.publicDir, "llms.txt");
-          await writeFile(llmsTxtPath, artifacts.llmsTxt, "utf-8");
-        }
-        if (artifacts.llmsFullTxt) {
-          const llmsFullTxtPath = join(nitro.options.output.publicDir, "llms-full.txt");
-          await writeFile(llmsFullTxtPath, artifacts.llmsFullTxt, "utf-8");
-        }
-        logger.success(`Generated markdown for ${pages.length} pages`);
-        if (artifacts.llmsTxt) {
-          logger.info("Generated llms.txt");
-        }
-        if (artifacts.llmsFullTxt) {
-          logger.info("Generated llms-full.txt");
-        }
-      } catch (error) {
-        logger.error("Failed to generate llms.txt artifacts:", error);
+        logger.info("Generated llms.txt");
+      }
+      if (artifacts.llmsFullTxt) {
+        const llmsFullTxtPath = join(nitro.options.output.publicDir, "llms-full.txt");
+        await writeFile(llmsFullTxtPath, artifacts.llmsFullTxt, "utf-8");
+        nitro._prerenderedRoutes.push({
+          route: "/llms-full.txt",
+          fileName: llmsFullTxtPath,
+          generateTimeMS: 0
+        });
+        logger.info("Generated llms-full.txt");
       }
     });
   });
@@ -102,7 +79,9 @@ const module = defineNuxtModule({
   },
   defaults: {
     enabled: true,
-    mdreamOptions: {},
+    mdreamOptions: {
+      preset: "minimal"
+    },
     cache: {
       maxAge: 3600,
       // 1 hour
@@ -168,9 +147,12 @@ export {}
       middleware: true,
       handler: resolver.resolve("./runtime/server/middleware/mdream")
     });
+    if (nuxt.options.build) {
+      addPlugin({ mode: "server", src: resolver.resolve("./runtime/nuxt/plugins/prerender") });
+    }
     const isStatic = nuxt.options.nitro.static || nuxt.options._generate || false;
     if (isStatic || nuxt.options.nitro.prerender?.routes?.length) {
-      setupPrerenderHandler(runtimeConfig, nuxt);
+      setupPrerenderHandler();
     }
   }
 });

package/dist/runtime/nuxt/plugins/prerender.d.ts

@@ -0,0 +1,2 @@
+declare const _default: import("nuxt/app").Plugin<Record<string, unknown>> & import("nuxt/app").ObjectPlugin<Record<string, unknown>>;
+export default _default;

package/dist/runtime/nuxt/plugins/prerender.js

@@ -0,0 +1,20 @@
+import { defineNuxtPlugin, prerenderRoutes } from "nuxt/app";
+export default defineNuxtPlugin({
+  setup(nuxtApp) {
+    if (!import.meta.prerender) {
+      return;
+    }
+    nuxtApp.hooks.hook("app:rendered", (ctx) => {
+      let url = ctx.ssrContext?.url || "";
+      if (url.endsWith(".md")) {
+        return;
+      }
+      if (url.endsWith("/")) {
+        url = `${url}index.md`;
+      } else {
+        url = `${url}.md`;
+      }
+      prerenderRoutes(url);
+    });
+  }
+});

package/dist/runtime/server/middleware/mdream.d.ts

@@ -1,2 +1,2 @@
-declare const _default: import("h3").EventHandler<import("h3").EventHandlerRequest, Promise<string | undefined>>;
+declare const _default: import("h3").EventHandler<import("h3").EventHandlerRequest, Promise<string | import("h3").H3Error<unknown> | undefined>>;
 export default _default;

package/dist/runtime/server/middleware/mdream.js

@@ -1,18 +1,12 @@
+import { withSiteUrl } from "#site-config/server/composables/utils";
 import { consola } from "consola";
-import { createError, defineEventHandler, getRequestURL, setHeader } from "h3";
+import { createError, defineEventHandler, setHeader } from "h3";
 import { htmlToMarkdown } from "mdream";
+import { extractionPlugin } from "mdream/plugins";
 import { withMinimalPreset } from "mdream/preset/minimal";
 import { useNitroApp, useRuntimeConfig } from "nitropack/runtime";
 const logger = consola.withTag("nuxt-mdream");
-function isIndexable(html) {
-  const robotsMatch = html.match(/<meta\s+name=["']robots["']\s+content=["']([^"']+)["']/i);
-  if (robotsMatch) {
-    const content = String(robotsMatch[1]).toLowerCase();
-    return !content.includes("noindex");
-  }
-  return true;
-}
-async function convertHtmlToMarkdown(html, url, config, route, title) {
+async function convertHtmlToMarkdown(html, url, config, route) {
   let options = {
     origin: url,
     ...config.mdreamOptions
@@ -20,83 +14,60 @@ async function convertHtmlToMarkdown(html, url, config, route, title) {
   if (config.mdreamOptions?.preset === "minimal") {
     options = withMinimalPreset(options);
   }
-  let markdown = htmlToMarkdown(html, options);
+  let title = "";
+  let markdown = htmlToMarkdown(html, {
+    ...options,
+    plugins: [
+      ...options.plugins || [],
+      // Add any additional plugins here if needed
+      extractionPlugin({
+        title(html2) {
+          title = html2.textContent;
+        }
+      })
+    ]
+  });
   const context = {
     html,
     markdown,
     route,
     title,
-    isPrerender: false
+    isPrerender: Boolean(import.meta.prerender)
   };
-  try {
-    const nitroApp = useNitroApp();
-    if (nitroApp?.hooks) {
-      await nitroApp.hooks.callHook("mdream:markdown", context);
-      markdown = context.markdown;
-    }
-  } catch (error) {
-    logger.debug("Could not call mdream:markdown hook:", error);
+  const nitroApp = useNitroApp();
+  if (nitroApp?.hooks) {
+    await nitroApp.hooks.callHook("mdream:markdown", context);
+    markdown = context.markdown;
   }
   return markdown;
 }
 export default defineEventHandler(async (event) => {
-  const requestUrl = getRequestURL(event);
-  let htmlPath = requestUrl.pathname;
-  if (!htmlPath.endsWith(".md")) {
+  let path = event.path;
+  if (!path.endsWith(".md")) {
     return;
   }
   const config = useRuntimeConfig(event).mdream;
-  htmlPath = htmlPath.slice(0, -3);
-  if (htmlPath === "/index") {
-    htmlPath = "/";
+  path = path.slice(0, -3);
+  if (path === "/index") {
+    path = "/";
   }
+  let html;
   try {
-    const fullUrl = new URL(htmlPath, requestUrl.origin).toString();
-    const response = await fetch(fullUrl, {
-      headers: {
-        // Forward important headers
-        "user-agent": event.headers.get("user-agent") || "",
-        "accept-language": event.headers.get("accept-language") || ""
-      }
-    });
-    if (!response.ok) {
-      throw createError({
-        statusCode: response.status,
-        statusMessage: response.statusText
-      });
-    }
-    const html = await response.text();
-    if (!isIndexable(html)) {
-      throw createError({
-        statusCode: 404,
-        statusMessage: "Page not indexable"
-      });
-    }
-    const titleMatch = html.match(/<title[^>]*>([^<]+)<\/title>/i);
-    const title = titleMatch ? String(titleMatch[1]).trim() : htmlPath;
-    const markdown = await convertHtmlToMarkdown(
-      html,
-      requestUrl.origin + htmlPath,
-      config,
-      htmlPath,
-      title
-    );
-    setHeader(event, "content-type", "text/markdown; charset=utf-8");
-    return markdown;
-  } catch (error) {
-    if (error.statusCode) {
-      throw error;
-    }
-    if (error.status === 404) {
-      throw createError({
-        statusCode: 404,
-        statusMessage: "Page not found"
-      });
-    }
-    logger.error("Failed to generate markdown for", htmlPath, error);
-    throw createError({
+    html = await globalThis.$fetch(path);
+  } catch (e) {
+    logger.error(`Failed to fetch HTML for ${path}`, e);
+    return createError({
       statusCode: 500,
-      statusMessage: `Failed to generate markdown: ${error.message || "Unknown error"}`
+      statusMessage: "Internal Server Error",
+      message: `Failed to fetch HTML for ${path}`
     });
   }
+  const markdown = await convertHtmlToMarkdown(
+    html,
+    withSiteUrl(event, path),
+    config,
+    path
+  );
+  setHeader(event, "content-type", "text/markdown; charset=utf-8");
+  return markdown;
 });

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@mdream/nuxt",
   "type": "module",
-  "version": "0.8.1",
+  "version": "0.8.2",
   "description": "Nuxt module for converting HTML pages to Markdown using mdream",
   "license": "MIT",
   "exports": {
@@ -26,25 +26,25 @@
     "nuxt": "^3.0.0"
   },
   "dependencies": {
-    "@nuxt/kit": "^4.0.0",
+    "@nuxt/kit": "^4.0.1",
     "consola": "^3.4.2",
     "defu": "^6.1.4",
     "nuxt-site-config": "^3.2.2",
     "pathe": "^2.0.3",
-    "mdream": "0.8.1"
+    "mdream": "0.8.2"
   },
   "devDependencies": {
     "@antfu/eslint-config": "^4.17.0",
     "@arethetypeswrong/cli": "^0.18.2",
     "@nuxt/devtools": "latest",
     "@nuxt/module-builder": "^1.0.1",
-    "@nuxt/schema": "^4.0.0",
+    "@nuxt/schema": "^4.0.1",
     "@nuxt/test-utils": "^3.19.2",
     "@types/node": "latest",
     "bumpp": "^10.2.0",
     "changelogen": "^0.6.2",
     "eslint": "^9.31.0",
-    "nuxt": "^4.0.0",
+    "nuxt": "^4.0.1",
     "typescript": "^5.8.3",
     "vitest": "^3.2.4"
   },