nuxt-ai-ready 0.0.3 → 0.1.2

package/dist/module.d.mts

@@ -1,36 +1,8 @@
 import * as _nuxt_schema from '@nuxt/schema';
-import { ModuleOptions, BulkChunk } from '../dist/runtime/types.js';
+import { BulkChunk, ModuleOptions } from '../dist/runtime/types.js';
 export { BulkChunk, ModuleOptions } from '../dist/runtime/types.js';
-import { ProcessedFile } from 'mdream/llms-txt';
-
-/**
- * Hook payload for mdream:llms-txt
- * Called after mdream has generated llms.txt, before writing to disk
- *
- * IMPORTANT: This uses a mutable pattern. Hooks should modify the content
- * and fullContent properties directly rather than returning values.
- *
- * @example
- * nuxt.hooks.hook('mdream:llms-txt', async (payload) => {
- *   payload.content += '\n\n## Custom Section\n\nAdded by hook!'
- *   payload.fullContent += '\n\n## Custom Section (Full)\n\nAdded by hook!'
- * })
- */
-interface LlmsTxtGeneratePayload {
-    /** Current llms.txt content - modify this directly */
-    content: string;
-    /** Current llms-full.txt content - modify this directly */
-    fullContent: string;
-    /** All routes with their metadata (read-only) */
-    pages: ProcessedFile[];
-}
 
 interface ModuleHooks {
-    /**
-     * Hook to modify llms.txt content before final output
-     * Other modules can append their own API endpoints here
-     */
-    'ai-ready:llms-txt': (payload: LlmsTxtGeneratePayload) => void | Promise<void>;
     /**
      * Hook to add routes to the AI ready
      * Other modules can register their own API routes
@@ -58,5 +30,10 @@ interface ModulePublicRuntimeConfig {
 }
 declare const _default: _nuxt_schema.NuxtModule<ModuleOptions, ModuleOptions, false>;
 
+declare module '@nuxt/schema' {
+    interface NuxtHooks extends ModuleHooks {
+    }
+}
+
 export { _default as default };
 export type { ModuleHooks, ModulePublicRuntimeConfig };

package/dist/module.json

@@ -4,7 +4,7 @@
     "nuxt": ">=4.0.0"
   },
   "configKey": "aiReady",
-  "version": "0.0.3",
+  "version": "0.1.2",
   "builder": {
     "@nuxt/module-builder": "1.0.2",
     "unbuild": "3.6.1"

package/dist/module.mjs

@@ -6,26 +6,39 @@ import { TagIdMap } from 'mdream';
 import { extractionPlugin } from 'mdream/plugins';
 import { htmlToMarkdownSplitChunksStream } from 'mdream/splitter';
 import { useSiteConfig, installNuxtSiteConfig, withSiteUrl } from 'nuxt-site-config/kit';
+import { isPathFile } from 'nuxt-site-config/urls';
 import { relative, resolve, dirname } from 'pathe';
 import { readPackageJSON } from 'pkg-types';
 import { estimateTokenCount } from 'tokenx';
-import { writeFile } from 'node:fs/promises';
+import { stat } from 'node:fs/promises';
 import { join } from 'node:path';
-import { generateLlmsTxtArtifacts } from 'mdream/llms-txt';
-import { normalizeLlmsTxtConfig } from '../dist/runtime/llms-txt.js';
+import { createLlmsTxtStream } from 'mdream/llms-txt';
 
 const logger = useLogger("nuxt-ai-ready");
 
 function setupPrerenderHandler() {
   const nuxt = useNuxt();
-  const pages = [];
   nuxt.hooks.hook("nitro:init", async (nitro) => {
+    let writer = null;
+    let pageCount = 0;
+    const startTime = Date.now();
     nitro.hooks.hook("prerender:generate", async (route) => {
       if (!route.fileName?.endsWith(".md")) {
         return;
       }
+      if (!writer) {
+        const siteConfig = useSiteConfig();
+        const stream = createLlmsTxtStream({
+          siteName: siteConfig.name || siteConfig.url,
+          description: siteConfig.description,
+          origin: siteConfig.url,
+          generateFull: true,
+          outputDir: nitro.options.output.publicDir
+        });
+        writer = stream.getWriter();
+      }
       const { markdown, title, description } = JSON.parse(route.contents || "{}");
-      const page = {
+      await writer.write({
         filePath: route.fileName,
         url: route.route,
         title,
@@ -34,68 +47,37 @@ function setupPrerenderHandler() {
           description,
           title
         }
-      };
-      pages.push(page);
+      });
+      pageCount++;
       route.contents = markdown;
     });
     nitro.hooks.hook("prerender:done", async () => {
-      if (pages.length === 0) {
+      if (!writer) {
         return;
       }
-      const startTime = Date.now();
-      const siteConfig = useSiteConfig();
-      const artifacts = await generateLlmsTxtArtifacts({
-        origin: siteConfig.url,
-        files: pages,
-        generateFull: true,
-        siteName: siteConfig.name || siteConfig.url,
-        description: siteConfig.description
-      });
-      logger.success(`Generated markdown for ${pages.length} pages`);
-      const hookPayload = {
-        content: artifacts.llmsTxt || "",
-        fullContent: artifacts.llmsFullTxt || "",
-        pages
-      };
-      const llmsTxtConfig = nuxt.options.runtimeConfig["nuxt-ai-ready"].llmsTxt;
-      const normalizedContent = normalizeLlmsTxtConfig(llmsTxtConfig);
-      if (normalizedContent) {
-        hookPayload.content = `${hookPayload.content}
-
-${normalizedContent}
-`;
-      }
-      await nuxt.hooks.callHook("ai-ready:llms-txt", hookPayload);
-      const finalLlmsTxt = hookPayload.content;
-      const finalLlmsFullTxt = hookPayload.fullContent;
-      const generatedFiles = [];
-      if (finalLlmsTxt) {
-        const llmsTxtPath = join(nitro.options.output.publicDir, "llms.txt");
-        await writeFile(llmsTxtPath, finalLlmsTxt, "utf-8");
-        const sizeKb = (Buffer.byteLength(finalLlmsTxt, "utf-8") / 1024).toFixed(2);
-        generatedFiles.push({ path: "llms.txt", size: `${sizeKb}kb` });
-        nitro._prerenderedRoutes.push({
+      await writer.close();
+      const llmsTxtPath = join(nitro.options.output.publicDir, "llms.txt");
+      const llmsFullTxtPath = join(nitro.options.output.publicDir, "llms-full.txt");
+      const [llmsStats, llmsFullStats] = await Promise.all([
+        stat(llmsTxtPath),
+        stat(llmsFullTxtPath)
+      ]);
+      nitro._prerenderedRoutes.push(
+        {
           route: "/llms.txt",
           fileName: llmsTxtPath,
           generateTimeMS: 0
-        });
-      }
-      if (finalLlmsFullTxt) {
-        const llmsFullTxtPath = join(nitro.options.output.publicDir, "llms-full.txt");
-        await writeFile(llmsFullTxtPath, finalLlmsFullTxt, "utf-8");
-        const sizeKb = (Buffer.byteLength(finalLlmsFullTxt, "utf-8") / 1024).toFixed(2);
-        generatedFiles.push({ path: "llms-full.txt", size: `${sizeKb}kb` });
-        nitro._prerenderedRoutes.push({
+        },
+        {
           route: "/llms-full.txt",
           fileName: llmsFullTxtPath,
           generateTimeMS: 0
-        });
-      }
-      if (generatedFiles.length > 0) {
-        const elapsed = Date.now() - startTime;
-        const fileList = generatedFiles.map((f) => `${f.path} (${f.size})`).join(" and ");
-        logger.info(`Generated ${fileList} in ${elapsed}ms`);
-      }
+        }
+      );
+      const elapsed = Date.now() - startTime;
+      const llmsKb = (llmsStats.size / 1024).toFixed(2);
+      const llmsFullKb = (llmsFullStats.size / 1024).toFixed(2);
+      logger.info(`Generated llms.txt (${llmsKb}kb) and llms-full.txt (${llmsFullKb}kb) from ${pageCount} pages in ${elapsed}ms`);
     });
   });
 }
@@ -114,16 +96,7 @@ const module = defineNuxtModule({
   },
   moduleDependencies: {
     "@nuxtjs/robots": {
-      version: ">=5.6.0",
-      defaults: {
-        groups: [
-          {
-            userAgent: "*",
-            contentUsage: ["train-ai=y"],
-            contentSignal: ["ai-train=yes", "search=yes", "ai-input=yes"]
-          }
-        ]
-      }
+      version: ">=5.6.0"
     },
     "nuxt-site-config": {
       version: ">=3"
@@ -168,6 +141,13 @@ const module = defineNuxtModule({
       resolve$1("./runtime/server/utils"),
       resolve$1("./runtime/server/mcp")
     );
+    if (typeof config.contentSignal === "object") {
+      nuxt.options.robots.groups.push({
+        userAgent: "*",
+        contentUsage: [`train-ai=${config.contentSignal.aiTrain ? "y" : "n"}`],
+        contentSignal: [`ai-train=${config.contentSignal.aiTrain ? "yes" : "no"}`, `search=${config.contentSignal.search ? "yes" : "no"}`, `ai-input=${config.contentSignal.aiInput ? "yes" : "no"}`]
+      });
+    }
     addTypeTemplate({
       filename: "module/nuxt-ai-ready.d.ts",
       getContents: (data) => {
@@ -195,7 +175,7 @@ export {}
     if (config.bulkRoute !== false) {
       const resolvedBulkRoute = withSiteUrl(config.bulkRoute);
       defaultLlmsTxtSections.push({
-        title: "AI Tools",
+        title: "LLM Tools",
         links: [
           {
             title: "Bulk Data",
@@ -228,7 +208,7 @@ Returns JSONL (newline-delimited JSON) with all indexed content.`
         defaultLlmsTxtSections[0].links.push(mcpLink);
       } else {
         defaultLlmsTxtSections.push({
-          title: "AI Tools - API Endpoints",
+          title: "LLM Tools",
           links: [mcpLink]
         });
       }
@@ -295,6 +275,9 @@ Returns JSONL (newline-delimited JSON) with all indexed content.`
           if (typeof route._sitemap !== "undefined" && !route._sitemap) {
             return;
           }
+          if (isPathFile(route.route)) {
+            return;
+          }
           let title = "";
           let description = "";
           const headings = [];

package/dist/runtime/types.d.ts

@@ -44,6 +44,24 @@ export interface ModuleOptions {
      * Structured llms.txt configuration
      */
     llmsTxt?: LlmsTxtConfig;
+    /**
+     * Content Signal Directives
+     */
+    contentSignal?: false | {
+        /**
+         * Allow Training or fine-tuning AI models.
+         */
+        aiTrain?: boolean;
+        /**
+         * Allow building a search index and providing search results (e.g., returning hyperlinks and short excerpts from your website's contents).
+         * Search does not include providing AI-generated search summaries.
+         */
+        search?: boolean;
+        /**
+         * Inputting content into one or more AI models (e.g., retrieval augmented generation, grounding, or other real-time taking of content for generative AI search answers).
+         */
+        aiInput?: boolean;
+    };
 }
 /**
  * Individual chunk entry in bulk.jsonl (one per chunk)

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "nuxt-ai-ready",
   "type": "module",
-  "version": "0.0.3",
+  "version": "0.1.2",
   "description": "Best practice AI & LLM discoverability for Nuxt sites.",
   "author": {
     "name": "Harlan Wilton",
@@ -35,7 +35,7 @@
     "@nuxt/kit": "4.2.1",
     "consola": "^3.4.2",
     "defu": "^6.1.4",
-    "mdream": "^0.14.0",
+    "mdream": "^0.15.0",
     "minimatch": "^10.1.1",
     "nuxt-site-config": "^3.2.11",
     "pathe": "^2.0.3",
@@ -55,10 +55,10 @@
     "@nuxtjs/color-mode": "^4.0.0",
     "@nuxtjs/eslint-config-typescript": "^12.1.0",
     "@nuxtjs/i18n": "^10.2.1",
-    "@nuxtjs/mcp-toolkit": "^0.4.1",
+    "@nuxtjs/mcp-toolkit": "^0.5.1",
     "@nuxtjs/robots": "^5.6.0",
     "@nuxtjs/sitemap": "^7.4.7",
-    "@vitest/coverage-v8": "^4.0.14",
+    "@vitest/coverage-v8": "^4.0.15",
     "@vueuse/nuxt": "^14.1.0",
     "better-sqlite3": "^12.5.0",
     "bumpp": "^10.3.2",
@@ -71,7 +71,7 @@
     "playwright-core": "^1.57.0",
     "postgres": "^3.4.7",
     "typescript": "^5.9.3",
-    "vitest": "^4.0.14",
+    "vitest": "^4.0.15",
     "vue": "^3.5.25",
     "vue-router": "^4.6.3",
     "vue-tsc": "^3.1.5",