nuxt-ai-ready 0.3.2 → 0.3.3

package/dist/module.json

@@ -4,7 +4,7 @@
     "nuxt": ">=4.0.0"
   },
   "configKey": "aiReady",
-  "version": "0.3.2",
+  "version": "0.3.3",
   "builder": {
     "@nuxt/module-builder": "1.0.2",
     "unbuild": "3.6.1"

package/dist/module.mjs

@@ -184,8 +184,7 @@ function setupPrerenderHandler(llmsTxtConfig, timestampsConfig) {
         pagesStream = createWriteStream(pagesPath, { encoding: "utf-8" });
         pagesStream.write("pages[999999]{route,title,description,headings,chunkIds,updatedAt}:\n");
       }
-      const { chunks, title, description, headings, updatedAt: metaUpdatedAt } = JSON.parse(route.contents || "{}");
-      const markdown = chunks.map((c) => c.content).join("\n\n");
+      const { markdown, chunks, title, description, headings, updatedAt: metaUpdatedAt } = JSON.parse(route.contents || "{}");
       let pageTimestamp = {
         updatedAt: (/* @__PURE__ */ new Date()).toISOString()
       };

package/dist/runtime/server/middleware/mdream.js

@@ -121,7 +121,7 @@ export default defineEventHandler(async (event) => {
     return;
   }
   if (import.meta.prerender) {
-    const result2 = await convertHtmlToMarkdownChunks(
+    const result2 = convertHtmlToMarkdownChunks(
       html,
       withSiteUrl(event, path),
       config.mdreamOptions

package/dist/runtime/server/utils.d.ts

@@ -1,8 +1,9 @@
 import type { ModulePublicRuntimeConfig } from '../../module.js';
-export declare function convertHtmlToMarkdownChunks(html: string, url: string, mdreamOptions: ModulePublicRuntimeConfig['mdreamOptions']): Promise<{
-    headings: Record<string, string[]>;
+export declare function convertHtmlToMarkdownChunks(html: string, url: string, mdreamOptions: ModulePublicRuntimeConfig['mdreamOptions']): {
     updatedAt?: string | undefined;
+    markdown: string;
     chunks: import("mdream").MarkdownChunk[];
     title: string;
     description: string;
-}>;
+    headings: Record<string, string[]>;
+};

package/dist/runtime/server/utils.js

@@ -1,9 +1,17 @@
-import { TagIdMap } from "mdream";
+import { htmlToMarkdown, TagIdMap } from "mdream";
 import { extractionPlugin } from "mdream/plugins";
 import { withMinimalPreset } from "mdream/preset/minimal";
-import { htmlToMarkdownSplitChunksStream } from "mdream/splitter";
+import { htmlToMarkdownSplitChunks } from "mdream/splitter";
 import { estimateTokenCount } from "tokenx";
-export async function convertHtmlToMarkdownChunks(html, url, mdreamOptions) {
+function stripFrontmatter(text) {
+  if (!text.startsWith("---\n"))
+    return text;
+  const endIdx = text.indexOf("\n---", 4);
+  if (endIdx === -1)
+    return text;
+  return text.slice(endIdx + 4).trimStart();
+}
+export function convertHtmlToMarkdownChunks(html, url, mdreamOptions) {
   let title = "";
   let description = "";
   let updatedAt;
@@ -14,7 +22,6 @@ export async function convertHtmlToMarkdownChunks(html, url, mdreamOptions) {
     'meta[name="description"]': (el) => {
       description = el.attributes.content || "";
     },
-    // Extract timestamp from various meta tag formats
     'meta[property="article:modified_time"], meta[name="last-modified"], meta[name="updated"], meta[property="og:updated_time"], meta[name="lastmod"]': (el) => {
       if (!updatedAt && el.attributes.content) {
         updatedAt = el.attributes.content;
@@ -31,33 +38,43 @@ export async function convertHtmlToMarkdownChunks(html, url, mdreamOptions) {
   } else {
     options.plugins = [extractPlugin, ...options.plugins || []];
   }
-  const chunksStream = htmlToMarkdownSplitChunksStream(html, {
+  const rawMarkdown = htmlToMarkdown(html, options);
+  const markdown = stripFrontmatter(rawMarkdown);
+  const rawChunks = htmlToMarkdownSplitChunks(html, {
     ...options,
     headersToSplitOn: [TagIdMap.h1, TagIdMap.h2, TagIdMap.h3],
     origin: url,
-    chunkSize: 256,
+    chunkSize: 512,
     stripHeaders: false,
     lengthFunction(text) {
       return estimateTokenCount(text);
     }
   });
-  const chunks = [];
-  for await (const chunk of chunksStream) {
-    chunks.push(chunk);
-  }
+  const chunks = rawChunks.filter((chunk, idx) => {
+    if (idx === 0 && chunk.content.startsWith("---\n")) {
+      const endIdx = chunk.content.indexOf("\n---", 4);
+      if (endIdx !== -1) {
+        chunk.content = chunk.content.slice(endIdx + 4).trimStart();
+        return chunk.content.length > 0;
+      }
+    }
+    return true;
+  });
+  const headings = chunks.reduce((set, chunk) => {
+    Object.entries(chunk.metadata?.headers || {}).forEach(([k, v]) => {
+      if (!set[k])
+        set[k] = [];
+      if (v && !set[k].includes(v))
+        set[k].push(v);
+    });
+    return set;
+  }, {});
   return {
+    markdown,
     chunks,
     title,
     description,
-    ...updatedAt && { updatedAt },
-    headings: chunks.reduce((set, m) => {
-      Object.entries(m.metadata?.headers || {}).forEach(([k, v]) => {
-        if (!set[k])
-          set[k] = [];
-        if (v && !set[k].includes(v))
-          set[k].push(v);
-      });
-      return set;
-    }, {})
+    headings,
+    ...updatedAt && { updatedAt }
   };
 }

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "nuxt-ai-ready",
   "type": "module",
-  "version": "0.3.2",
+  "version": "0.3.3",
   "description": "Best practice AI & LLM discoverability for Nuxt sites.",
   "author": {
     "name": "Harlan Wilton",
@@ -59,7 +59,7 @@
     "@nuxtjs/i18n": "^10.2.1",
     "@nuxtjs/mcp-toolkit": "^0.5.1",
     "@nuxtjs/robots": "^5.6.1",
-    "@nuxtjs/sitemap": "^7.4.8",
+    "@nuxtjs/sitemap": "^7.4.9",
     "@vitest/coverage-v8": "^4.0.15",
     "@vueuse/nuxt": "^14.1.0",
     "better-sqlite3": "^12.5.0",
@@ -76,7 +76,7 @@
     "vitest": "^4.0.15",
     "vue": "^3.5.25",
     "vue-router": "^4.6.3",
-    "vue-tsc": "^3.1.6",
+    "vue-tsc": "^3.1.7",
     "zod": "^4.1.13"
   },
   "resolutions": {