nuxt-ai-ready 0.1.5 → 0.2.0

package/README.md

@@ -21,11 +21,11 @@ Nuxt AI Ready converts your indexable pages into clean markdown that AI systems
 
 ## Features
 
-- 📄 **llms.txt Generation**: Auto-generates `llms.txt` and `llms-full.txt` at build time
+- 📄 **LLM Optimized Output**: Generate `llms.txt` as well as `llms.toon` (tabular, token efficient) artifacts 
 - 🚀 **On-Demand Markdown**: Any route available as `.md` (e.g., `/about` → `/about.md`)
 - 🤖 **Smart Bot Detection**: Serves markdown to AI crawlers automatically
 - 📡 **Content Signals**: Help AI systems understand how to use your pages
-- 📦 **RAG-Ready Output**: Chunked content for semantic search and AI chat pipelines
+- 📦 **Bulk Chunk Export**: Exported token optimized chunks ready for RAG and semantic search
 - ⚡ **MCP Integration**: Let AI agents query your site directly
 
 ## Installation

package/dist/module.d.mts

@@ -22,7 +22,6 @@ interface ModuleHooks {
     }) => void | Promise<void>;
 }
 interface ModulePublicRuntimeConfig {
-    bulkRoute: string | false;
     debug: boolean;
     version: string;
     mdreamOptions: ModuleOptions['mdreamOptions'];

package/dist/module.json

@@ -4,7 +4,7 @@
     "nuxt": ">=4.0.0"
   },
   "configKey": "aiReady",
-  "version": "0.1.5",
+  "version": "0.2.0",
   "builder": {
     "@nuxt/module-builder": "1.0.2",
     "unbuild": "3.6.1"

package/dist/module.mjs

@@ -1,29 +1,46 @@
 import { useLogger, useNuxt, defineNuxtModule, createResolver, addTypeTemplate, hasNuxtModule, addServerHandler, addPlugin } from '@nuxt/kit';
 import defu from 'defu';
 import { useSiteConfig, installNuxtSiteConfig, withSiteUrl } from 'nuxt-site-config/kit';
-import { relative as relative$1 } from 'pathe';
+import { relative } from 'pathe';
 import { readPackageJSON } from 'pkg-types';
 import { createHash } from 'node:crypto';
 import { mkdirSync, createWriteStream } from 'node:fs';
-import { stat } from 'node:fs/promises';
-import { join, dirname, relative } from 'node:path';
+import { stat, open } from 'node:fs/promises';
+import { join, dirname } from 'node:path';
+import { encodeLines } from '@toon-format/toon';
 import { createLlmsTxtStream } from 'mdream/llms-txt';
 
 const logger = useLogger("nuxt-ai-ready");
 
 function generateVectorId(route, chunkIdx) {
-  const hash = createHash("sha256").update(route).digest("hex").substring(0, 48);
+  const hash = createHash("sha256").update(route).digest("hex").substring(0, 8);
   return `${hash}-${chunkIdx}`;
 }
-function setupPrerenderHandler() {
+async function updateFirstLine(filePath, newFirstLine) {
+  const fh = await open(filePath, "r+");
+  try {
+    const buffer = Buffer.alloc(1024);
+    await fh.read(buffer, 0, 1024, 0);
+    const content = buffer.toString("utf-8");
+    const firstLineEnd = content.indexOf("\n");
+    const oldFirstLine = content.substring(0, firstLineEnd);
+    const paddedLine = newFirstLine.padEnd(oldFirstLine.length, " ");
+    await fh.write(paddedLine, 0, "utf-8");
+  } finally {
+    await fh.close();
+  }
+}
+function setupPrerenderHandler(llmsTxtConfig) {
   const nuxt = useNuxt();
   nuxt.hooks.hook("nitro:init", async (nitro) => {
     let writer = null;
-    let bulkStream = null;
-    let bulkStreamEntries = 0;
+    let chunksStream = null;
+    let pagesStream = null;
+    let chunksProcessed = 0;
     let pageCount = 0;
     const startTime = Date.now();
-    const bulkPath = join(nitro.options.output.publicDir, "content.jsonl");
+    const pagesChunksPath = join(nitro.options.output.publicDir, "llms-full.toon");
+    const pagesPath = join(nitro.options.output.publicDir, "llms.toon");
     nitro.hooks.hook("prerender:generate", async (route) => {
       if (!route.fileName?.endsWith(".md")) {
         return;
@@ -39,19 +56,21 @@ function setupPrerenderHandler() {
           description: siteConfig.description,
           origin: siteConfig.url,
           generateFull: true,
-          outputDir: nitro.options.output.publicDir
+          outputDir: nitro.options.output.publicDir,
+          sections: llmsTxtConfig.sections,
+          notes: llmsTxtConfig.notes
         });
         writer = stream.getWriter();
-      }
-      if (!bulkStream) {
-        mkdirSync(dirname(bulkPath), { recursive: true });
-        bulkStream = createWriteStream(bulkPath);
-        logger.info(`Bulk JSONL stream created at ${relative(nuxt.options.rootDir, bulkPath)}`);
+        mkdirSync(dirname(pagesChunksPath), { recursive: true });
+        mkdirSync(dirname(pagesPath), { recursive: true });
+        chunksStream = createWriteStream(pagesChunksPath, "utf-8");
+        chunksStream.write("pageChunks[999999]{id,route,content}:\n");
+        pagesStream = createWriteStream(pagesPath, "utf-8");
+        pagesStream.write("pages[999999]{route,title,description,headings,chunkIds}:\n");
       }
       const { chunks, title, description, headings } = JSON.parse(route.contents || "{}");
       const markdown = chunks.map((c) => c.content).join("\n\n");
       await writer.write({
-        filePath: route.fileName,
         url: pageRoute,
         title,
         content: markdown,
@@ -62,32 +81,55 @@ function setupPrerenderHandler() {
       });
       pageCount++;
       logger.debug(`Processing ${chunks.length} chunks for route: ${pageRoute}`);
+      const chunkIds = [];
       for (let idx = 0; idx < chunks.length; idx++) {
         const chunk = chunks[idx];
         if (!chunk)
           continue;
+        const chunkId = generateVectorId(pageRoute, idx);
+        chunkIds.push(chunkId);
         const bulkChunk = {
-          id: generateVectorId(pageRoute, idx),
+          id: chunkId,
           route: pageRoute,
-          chunkIndex: idx,
-          content: chunk.content,
-          headers: chunk.metadata?.headers,
-          loc: chunk.metadata?.loc,
-          title,
-          description
+          content: chunk.content
         };
         await nuxt.hooks.callHook("ai-ready:chunk", {
           chunk: bulkChunk,
           route: pageRoute,
           title,
           description,
-          headings
+          headings: Object.entries(headings).flatMap(
+            ([tag, texts]) => texts.map((text) => ({ [tag]: text }))
+          )
         });
-        bulkStream.write(`${JSON.stringify(bulkChunk)}
+        if (chunksStream) {
+          const lines = Array.from(encodeLines({ pageChunks: [bulkChunk] }));
+          if (lines[1]) {
+            chunksStream.write(`${lines[1]}
 `);
-        bulkStreamEntries++;
+          }
+        }
+        chunksProcessed++;
       }
       logger.debug(`Completed ${chunks.length} chunks for ${pageRoute}`);
+      const pageDoc = {
+        route: pageRoute,
+        title,
+        description,
+        // Convert headings object to readable string format (h1:Title|h2:Subtitle,...)
+        headings: headings && Object.keys(headings).length ? Object.entries(headings).flatMap(
+          ([tag, texts]) => texts.map((text) => `${tag}:${text}`)
+        ).join("|") : "",
+        // Join chunkIds array to comma-separated string
+        chunkIds: chunkIds.join(",")
+      };
+      if (pagesStream) {
+        const lines = Array.from(encodeLines({ pages: [pageDoc] }));
+        if (lines[1]) {
+          pagesStream.write(`${lines[1]}
+`);
+        }
+      }
       route.contents = markdown;
     });
     nitro.hooks.hook("prerender:done", async () => {
@@ -95,10 +137,24 @@ function setupPrerenderHandler() {
         return;
       }
       await writer.close();
-      if (bulkStream) {
-        bulkStream.end();
-        logger.info(`Bulk JSONL stream closed with ${bulkStreamEntries} chunks`);
+      if (chunksStream) {
+        await new Promise((resolve, reject) => {
+          chunksStream.on("error", reject);
+          chunksStream.on("finish", resolve);
+          chunksStream.end();
+        });
       }
+      if (pagesStream) {
+        await new Promise((resolve, reject) => {
+          pagesStream.on("error", reject);
+          pagesStream.on("finish", resolve);
+          pagesStream.end();
+        });
+      }
+      await updateFirstLine(pagesChunksPath, `pageChunks[${chunksProcessed}]{id,route,content}:`);
+      await updateFirstLine(pagesPath, `pages[${pageCount}]{route,title,description,headings,chunkIds}:`);
+      logger.info(`Wrote llms-full.toon with ${chunksProcessed} chunks`);
+      logger.info(`Wrote llms.toon with ${pageCount} pages`);
       const llmsTxtPath = join(nitro.options.output.publicDir, "llms.txt");
       const llmsFullTxtPath = join(nitro.options.output.publicDir, "llms-full.txt");
       const files = [
@@ -111,26 +167,31 @@ function setupPrerenderHandler() {
           route: "/llms-full.txt",
           fileName: llmsFullTxtPath,
           generateTimeMS: 0
+        },
+        {
+          route: "/llms-full.toon",
+          fileName: pagesChunksPath,
+          generateTimeMS: 0
+        },
+        {
+          route: "/llms.toon",
+          fileName: pagesPath,
+          generateTimeMS: 0
         }
       ];
-      if (bulkStream) {
-        files.push({
-          route: "/content.jsonl",
-          fileName: bulkPath,
-          generateTimeMS: 0
-        });
-      }
-      const [llmsStats, llmsFullStats, bulkStats] = await Promise.all([
+      const [llmsStats, llmsFullStats, pagesChunksStats, pagesStats] = await Promise.all([
         stat(llmsTxtPath),
         stat(llmsFullTxtPath),
-        bulkStream ? stat(bulkPath) : Promise.resolve(null)
+        stat(pagesChunksPath),
+        stat(pagesPath)
       ]);
       nitro._prerenderedRoutes.push(...files);
       const elapsed = Date.now() - startTime;
       const llmsKb = (llmsStats.size / 1024).toFixed(2);
       const llmsFullKb = (llmsFullStats.size / 1024).toFixed(2);
-      const bulkKb = bulkStats ? (bulkStats.size / 1024).toFixed(2) : "0";
-      logger.info(`Generated llms.txt (${llmsKb}kb), llms-full.txt (${llmsFullKb}kb), and content.jsonl (${bulkKb}kb) from ${pageCount} pages (${bulkStreamEntries} chunks) in ${elapsed}ms`);
+      const pagesChunksKb = (pagesChunksStats.size / 1024).toFixed(2);
+      const pagesKb = (pagesStats.size / 1024).toFixed(2);
+      logger.info(`Generated llms.txt (${llmsKb}kb), llms-full.txt (${llmsFullKb}kb), llms-full.toon (${pagesChunksKb}kb), and llms.toon (${pagesKb}kb) from ${pageCount} pages (${chunksProcessed} chunks) in ${elapsed}ms`);
     });
   });
 }
@@ -159,7 +220,6 @@ const module = defineNuxtModule({
     return {
       enabled: true,
       debug: false,
-      bulkRoute: "/content.jsonl",
       mdreamOptions: {
         preset: "minimal"
       },
@@ -200,7 +260,7 @@ const module = defineNuxtModule({
     addTypeTemplate({
       filename: "module/nuxt-ai-ready.d.ts",
       getContents: (data) => {
-        const typesPath = relative$1(resolve(data.nuxt.options.rootDir, data.nuxt.options.buildDir, "module"), resolve("runtime/types"));
+        const typesPath = relative(resolve(data.nuxt.options.rootDir, data.nuxt.options.buildDir, "module"), resolve("runtime/types"));
         const nitroTypes = `  interface NitroRuntimeHooks {
     'ai-ready:markdown': (context: import('${typesPath}').MarkdownContext) => void | Promise<void>
     'ai-ready:mdreamConfig': (config: import('mdream').HTMLToMarkdownOptions) => void | Promise<void>
@@ -221,23 +281,27 @@ export {}
       nitro: true
     });
     const defaultLlmsTxtSections = [];
-    if (config.bulkRoute !== false) {
-      const resolvedBulkRoute = withSiteUrl(config.bulkRoute);
-      defaultLlmsTxtSections.push({
-        title: "LLM Tools",
-        links: [
-          {
-            title: "Bulk Data",
-            href: resolvedBulkRoute,
-            description: `\`\`\`bash
-curl "${resolvedBulkRoute}"
-\`\`\`
+    const pagesRoute = withSiteUrl("llms.toon");
+    const pagesChunksRoute = withSiteUrl("llms-full.toon");
+    defaultLlmsTxtSections.push({
+      title: "LLM Resources",
+      links: [
+        {
+          title: "Pages Minimal",
+          href: pagesRoute,
+          description: `Page-level metadata in TOON format (token-efficient JSON encoding, see https://toonformat.dev). Contains: route, title, description, headings, chunkIds. Use with llms-full.toon for complete content. Fields: { route, title, description, headings, chunkIds }.
 
-Returns JSONL (newline-delimited JSON) with all indexed content.`
-          }
-        ]
-      });
-    }
+  <code lang="bash">curl "${pagesRoute}"</code>`
+        },
+        {
+          title: "Page Chunks",
+          href: pagesChunksRoute,
+          description: `Individual content chunks in TOON format for RAG/embeddings. Contains: id, route, content. Fields: { id, route, content }. Join with llms.toon using route to get title/description/headings metadata. Chunk index inferred from id suffix (e.g., "hash-0", "hash-1").
+
+  <code lang="bash">curl "${pagesChunksRoute}"</code>`
+        }
+      ]
+    });
     const hasMCP = hasNuxtModule("@nuxtjs/mcp-toolkit");
     if (hasMCP) {
       nuxt.hook("mcp:definitions:paths", (paths) => {
@@ -245,15 +309,25 @@ Returns JSONL (newline-delimited JSON) with all indexed content.`
         paths.tools = paths.tools || [];
         paths.resources = paths.resources || [];
         paths.prompts = paths.prompts || [];
-        paths.tools.push(`${mcpRuntimeDir}/tools`);
-        paths.resources.push(`${mcpRuntimeDir}/resources`);
-        paths.prompts.push(`${mcpRuntimeDir}/prompts`);
+        const mcpConfig = config.mcp || {};
+        const toolsConfig = mcpConfig.tools ?? {};
+        const resourcesConfig = mcpConfig.resources ?? {};
+        if (toolsConfig.listPages !== false) {
+          paths.tools.push(`${mcpRuntimeDir}/tools/list-pages.ts`);
+        }
+        if (resourcesConfig.pages !== false) {
+          paths.resources.push(`${mcpRuntimeDir}/resources/pages.ts`);
+        }
+        if (resourcesConfig.pagesChunks !== false) {
+          paths.resources.push(`${mcpRuntimeDir}/resources/pages-chunks.ts`);
+        }
       });
       const mcpLink = {
         title: "MCP",
-        href: withSiteUrl(nuxt.options.mcp?.route || "/mcp")
+        href: withSiteUrl(nuxt.options.mcp?.route || "/mcp"),
+        description: "Model Context Protocol server endpoint for AI agent integration."
       };
-      if (config.bulkRoute !== false && defaultLlmsTxtSections[0]) {
+      if (defaultLlmsTxtSections[0]) {
         defaultLlmsTxtSections[0].links.push(mcpLink);
       } else {
         defaultLlmsTxtSections.push({
@@ -269,10 +343,16 @@ Returns JSONL (newline-delimited JSON) with all indexed content.`
       ],
       notes: config.llmsTxt.notes
     } : { sections: defaultLlmsTxtSections };
+    const llmsTxtPayload = {
+      sections: mergedLlmsTxt.sections || [],
+      notes: typeof mergedLlmsTxt.notes === "string" ? [mergedLlmsTxt.notes] : mergedLlmsTxt.notes || []
+    };
+    await nuxt.callHook("ai-ready:llms-txt", llmsTxtPayload);
+    mergedLlmsTxt.sections = llmsTxtPayload.sections;
+    mergedLlmsTxt.notes = llmsTxtPayload.notes.length > 0 ? llmsTxtPayload.notes : void 0;
     nuxt.options.runtimeConfig["nuxt-ai-ready"] = {
       version: version || "0.0.0",
       debug: config.debug || false,
-      bulkRoute: config.bulkRoute,
       mdreamOptions: config.mdreamOptions || {},
       markdownCacheHeaders: defu(config.markdownCacheHeaders, {
         maxAge: 3600,
@@ -292,16 +372,11 @@ Returns JSONL (newline-delimited JSON) with all indexed content.`
     }
     const isStatic = nuxt.options.nitro.static || nuxt.options._generate || false;
     if (isStatic || nuxt.options.nitro.prerender?.routes?.length) {
-      setupPrerenderHandler();
-    }
-    if (config.bulkRoute !== false) {
-      nuxt.options.nitro.routeRules = nuxt.options.nitro.routeRules || {};
-      nuxt.options.nitro.routeRules[config.bulkRoute] = {
-        headers: {
-          "Content-Type": "application/x-ndjson; charset=utf-8"
-        }
-      };
+      setupPrerenderHandler(mergedLlmsTxt);
     }
+    nuxt.options.nitro.routeRules = nuxt.options.nitro.routeRules || {};
+    nuxt.options.nitro.routeRules["/llms.toon"] = { headers: { "Content-Type": "text/toon; charset=utf-8" } };
+    nuxt.options.nitro.routeRules["/llms-full.toon"] = { headers: { "Content-Type": "text/toon; charset=utf-8" } };
   }
 });
 

package/dist/runtime/server/mcp/resources/pages-chunks.d.ts

@@ -0,0 +1,17 @@
+declare const _default: {
+    uri: string;
+    name: string;
+    description: string;
+    metadata: {
+        mimeType: string;
+    };
+    cache: "1h";
+    handler(uri: URL): Promise<{
+        contents: {
+            uri: string;
+            mimeType: string;
+            text: string;
+        }[];
+    }>;
+};
+export default _default;

package/dist/runtime/server/mcp/resources/pages-chunks.js

@@ -0,0 +1,22 @@
+export default {
+  uri: "resource://nuxt-ai-ready/pages-chunks",
+  name: "All Page Chunks",
+  description: "Chunk-level content (id, route, content) in TOON format for RAG/embeddings. Join with pages resource using id field - match chunk.id with page.chunkIds[] to get title, description, headings. TOON is token-efficient JSON encoding (see https://toonformat.dev)",
+  metadata: {
+    mimeType: "text/plain"
+  },
+  cache: "1h",
+  async handler(uri) {
+    const response = await fetch("/llms-full.toon");
+    if (!response.ok)
+      throw new Error(`Failed to fetch chunks: ${response.statusText}`);
+    const text = await response.text();
+    return {
+      contents: [{
+        uri: uri.toString(),
+        mimeType: "text/plain",
+        text
+      }]
+    };
+  }
+};

package/dist/runtime/server/mcp/resources/pages.d.ts

@@ -1,2 +1,17 @@
-declare const _default: any;
+declare const _default: {
+    uri: string;
+    name: string;
+    description: string;
+    metadata: {
+        mimeType: string;
+    };
+    cache: "1h";
+    handler(uri: URL): Promise<{
+        contents: {
+            uri: string;
+            mimeType: string;
+            text: string;
+        }[];
+    }>;
+};
 export default _default;

package/dist/runtime/server/mcp/resources/pages.js

@@ -1,23 +1,22 @@
-import { defineMcpResource, jsonResult } from "#imports";
-import { streamBulkDocuments } from "../../utils/db.js";
-export default defineMcpResource({
-  uri: "pages://list",
+export default {
+  uri: "resource://nuxt-ai-ready/pages",
   name: "All Pages",
-  description: "Complete list of all indexed pages with basic metadata (route, title, description)",
-  mimeType: "application/json",
-  handler: async () => {
-    const pages = [];
-    for await (const doc of streamBulkDocuments()) {
-      pages.push({
-        route: doc.route,
-        title: doc.title,
-        description: doc.description,
-        id: doc.id
-      });
-    }
-    return jsonResult({
-      total: pages.length,
-      pages
-    });
+  description: "Page-level metadata (route, title, description, markdown, headings, chunkIds) in TOON format. Each page includes chunkIds[] array to join with pages-chunks resource for chunk-level content. TOON is token-efficient JSON encoding (see https://toonformat.dev)",
+  metadata: {
+    mimeType: "text/plain"
+  },
+  cache: "1h",
+  async handler(uri) {
+    const response = await fetch("/llms.toon");
+    if (!response.ok)
+      throw new Error(`Failed to fetch pages: ${response.statusText}`);
+    const text = await response.text();
+    return {
+      contents: [{
+        uri: uri.toString(),
+        mimeType: "text/plain",
+        text
+      }]
+    };
   }
-});
+};

package/dist/runtime/server/mcp/tools/list-pages.d.ts

@@ -1,2 +1,86 @@
-declare const _default: any;
+import { z } from 'zod';
+/**
+ * Lists all pages by fetching and returning TOON-encoded data
+ * TOON (Token-Oriented Object Notation) is a compact encoding that minimizes tokens for LLM input
+ * See https://toonformat.dev
+ */
+declare const _default: {
+    name: string;
+    description: string;
+    inputSchema: {
+        mode: z.ZodDefault<z.ZodEnum<{
+            minimal: "minimal";
+            chunks: "chunks";
+        }>>;
+    };
+    cache: "1h";
+    handler({ mode }: import("@modelcontextprotocol/sdk/server/zod-compat.js").ShapeOutput<Readonly<{
+        [k: string]: z.core.$ZodType<unknown, unknown, z.core.$ZodTypeInternals<unknown, unknown>>;
+    }>>): Promise<{
+        [x: string]: unknown;
+        content: ({
+            type: "text";
+            text: string;
+            _meta?: {
+                [x: string]: unknown;
+            } | undefined;
+        } | {
+            type: "image";
+            data: string;
+            mimeType: string;
+            _meta?: {
+                [x: string]: unknown;
+            } | undefined;
+        } | {
+            type: "audio";
+            data: string;
+            mimeType: string;
+            _meta?: {
+                [x: string]: unknown;
+            } | undefined;
+        } | {
+            uri: string;
+            name: string;
+            type: "resource_link";
+            description?: string | undefined;
+            mimeType?: string | undefined;
+            _meta?: {
+                [x: string]: unknown;
+            } | undefined;
+            icons?: {
+                src: string;
+                mimeType?: string | undefined;
+                sizes?: string[] | undefined;
+            }[] | undefined;
+            title?: string | undefined;
+        } | {
+            type: "resource";
+            resource: {
+                uri: string;
+                text: string;
+                mimeType?: string | undefined;
+                _meta?: {
+                    [x: string]: unknown;
+                } | undefined;
+            } | {
+                uri: string;
+                blob: string;
+                mimeType?: string | undefined;
+                _meta?: {
+                    [x: string]: unknown;
+                } | undefined;
+            };
+            _meta?: {
+                [x: string]: unknown;
+            } | undefined;
+        })[];
+        _meta?: {
+            [x: string]: unknown;
+        } | undefined;
+        structuredContent?: {
+            [x: string]: unknown;
+        } | undefined;
+        isError?: boolean | undefined;
+    }>;
+};
 export default _default;

package/dist/runtime/server/mcp/tools/list-pages.js

@@ -1,79 +1,18 @@
-import { defineMcpTool, jsonResult } from "#imports";
-import { streamBulkDocuments } from "../../utils/db.js";
-export default defineMcpTool({
+import { z } from "zod";
+import { toonResult } from "../utils.js";
+const schema = {
+  mode: z.enum(["chunks", "minimal"]).default("minimal").describe("Return individual content chunks (chunks) or page-level metadata (minimal)")
+};
+export default {
   name: "list_pages",
-  description: `Lists all indexed pages from the site with configurable output fields.
-
-WHEN TO USE: Use this tool when you need to DISCOVER or SEARCH for pages. Common scenarios:
-- "What pages are available?" - browse all pages
-- "Find pages about X topic" - search by title/description
-- "Show me the site structure" - explore content organization
-- "What documentation exists?" - discover available content
-
-WHEN NOT TO USE: If you already know the exact page route, use get_page directly.
-
-WORKFLOW: This tool returns page metadata (route, title, description, etc.). After finding relevant pages, use get_page to retrieve full content.
-
-FIELD OPTIONS: Control which fields to include in the output:
-- route: Page URL path (always included)
-- title: Page title
-- description: Page meta description
-- headings: Document structure (h1, h2, h3, etc.)
-- markdown: Full markdown content (warning: can be large, avoid unless needed)
-- id: Document identifier
-- chunkIds: Associated chunk identifiers`,
-  parameters: {
-    type: "object",
-    properties: {
-      fields: {
-        type: "array",
-        description: "Fields to include in output. Defaults to [route, title, description]",
-        items: {
-          type: "string",
-          enum: ["route", "title", "description", "headings", "markdown", "id", "chunkIds"]
-        },
-        default: ["route", "title", "description"]
-      },
-      search: {
-        type: "string",
-        description: "Optional search term to filter pages by title or description"
-      },
-      limit: {
-        type: "number",
-        description: "Maximum number of pages to return",
-        minimum: 1,
-        maximum: 1e3,
-        default: 100
-      }
-    }
-  },
-  // @ts-expect-error untyped
-  handler: async ({ fields = ["route", "title", "description"], search, limit = 100 }) => {
-    const searchLower = search?.toLowerCase();
-    const result = [];
-    let total = 0;
-    let filtered = 0;
-    for await (const doc of streamBulkDocuments()) {
-      total++;
-      if (searchLower) {
-        const matches = doc.title?.toLowerCase().includes(searchLower) || doc.description?.toLowerCase().includes(searchLower) || doc.route?.toLowerCase().includes(searchLower);
-        if (!matches)
-          continue;
-      }
-      filtered++;
-      if (result.length < limit) {
-        const projected = { route: doc.route };
-        fields.forEach((field) => {
-          if (field !== "route" && field in doc)
-            projected[field] = doc[field];
-        });
-        result.push(projected);
-      }
-    }
-    return jsonResult({
-      total,
-      filtered,
-      pages: result
-    });
+  description: 'Lists all available pages in TOON format (token-efficient). Use "chunks" mode to get individual content chunks, or "minimal" for page-level metadata.',
+  inputSchema: schema,
+  cache: "1h",
+  async handler({ mode }) {
+    const response = await fetch(mode === "chunks" ? "/llms-full.toon" : "/llms.toon");
+    if (!response.ok)
+      throw new Error(`Failed to fetch pages: ${response.statusText}`);
+    const toon = await response.text();
+    return toonResult(toon);
   }
-});
+};

package/dist/runtime/server/mcp/utils.d.ts

@@ -0,0 +1,3 @@
+import type { CallToolResult } from '@modelcontextprotocol/sdk/types.js';
+export declare function jsonResult(data: any, pretty?: boolean): CallToolResult;
+export declare function toonResult(toon: string): CallToolResult;

package/dist/runtime/server/mcp/utils.js

@@ -0,0 +1,7 @@
+export function jsonResult(data, pretty = true) {
+  const text = pretty ? JSON.stringify(data, null, 2) : JSON.stringify(data);
+  return { content: [{ type: "text", text }] };
+}
+export function toonResult(toon) {
+  return { content: [{ type: "text", text: toon }] };
+}

package/dist/runtime/server/middleware/mdream.js

@@ -1,12 +1,11 @@
 import { withSiteUrl } from "#site-config/server/composables/utils";
 import { createError, defineEventHandler, getHeader, setHeader } from "h3";
-import { htmlToMarkdown, TagIdMap } from "mdream";
+import { htmlToMarkdown } from "mdream";
 import { extractionPlugin } from "mdream/plugins";
 import { withMinimalPreset } from "mdream/preset/minimal";
-import { htmlToMarkdownSplitChunksStream } from "mdream/splitter";
 import { useNitroApp, useRuntimeConfig } from "nitropack/runtime";
-import { estimateTokenCount } from "tokenx";
 import { logger } from "../logger.js";
+import { convertHtmlToMarkdownChunks } from "../utils.js";
 function shouldServeMarkdown(event) {
   const accept = getHeader(event, "accept") || "";
   const secFetchDest = getHeader(event, "sec-fetch-dest") || "";
@@ -62,50 +61,6 @@ async function convertHtmlToMarkdown(html, url, config, route, event) {
   markdown = context.markdown;
   return { markdown, title, description, headings };
 }
-async function convertHtmlToMarkdownChunks(html, url, config) {
-  let title = "";
-  let description = "";
-  const headings = [];
-  const extractPlugin = extractionPlugin({
-    title(el) {
-      title = el.textContent;
-    },
-    'meta[name="description"]': (el) => {
-      description = el.attributes.content || "";
-    },
-    "h1, h2, h3, h4, h5, h6": (el) => {
-      const text = el.textContent?.trim();
-      const level = el.name.toLowerCase();
-      if (text)
-        headings.push({ [level]: text });
-    }
-  });
-  let options = {
-    origin: url,
-    ...config.mdreamOptions
-  };
-  if (config.mdreamOptions?.preset === "minimal") {
-    options = withMinimalPreset(options);
-    options.plugins = [extractPlugin, ...options.plugins || []];
-  } else {
-    options.plugins = [extractPlugin, ...options.plugins || []];
-  }
-  const chunksStream = htmlToMarkdownSplitChunksStream(html, {
-    ...options,
-    headersToSplitOn: [TagIdMap.h1, TagIdMap.h2, TagIdMap.h3],
-    origin: url,
-    chunkSize: 256,
-    stripHeaders: false,
-    lengthFunction(text) {
-      return estimateTokenCount(text);
-    }
-  });
-  const chunks = [];
-  for await (const chunk of chunksStream) {
-    chunks.push(chunk);
-  }
-  return { chunks, title, description, headings };
-}
 export default defineEventHandler(async (event) => {
   let path = event.path;
   const config = useRuntimeConfig(event)["nuxt-ai-ready"];
@@ -169,7 +124,7 @@ export default defineEventHandler(async (event) => {
     const result2 = await convertHtmlToMarkdownChunks(
       html,
       withSiteUrl(event, path),
-      config
+      config.mdreamOptions
     );
     return JSON.stringify(result2);
   }

package/dist/runtime/server/utils.d.ts

@@ -0,0 +1,7 @@
+import type { ModulePublicRuntimeConfig } from '../../module.js';
+export declare function convertHtmlToMarkdownChunks(html: string, url: string, mdreamOptions: ModulePublicRuntimeConfig['mdreamOptions']): Promise<{
+    chunks: import("mdream").MarkdownChunk[];
+    title: string;
+    description: string;
+    headings: Record<string, string[]>;
+}>;

package/dist/runtime/server/utils.js

@@ -0,0 +1,50 @@
+import { TagIdMap } from "mdream";
+import { extractionPlugin } from "mdream/plugins";
+import { withMinimalPreset } from "mdream/preset/minimal";
+import { htmlToMarkdownSplitChunksStream } from "mdream/splitter";
+import { estimateTokenCount } from "tokenx";
+export async function convertHtmlToMarkdownChunks(html, url, mdreamOptions) {
+  let title = "";
+  let description = "";
+  const extractPlugin = extractionPlugin({
+    title(el) {
+      title = el.textContent;
+    },
+    'meta[name="description"]': (el) => {
+      description = el.attributes.content || "";
+    }
+  });
+  let options = {
+    origin: url,
+    ...mdreamOptions
+  };
+  if (mdreamOptions?.preset === "minimal") {
+    options = withMinimalPreset(options);
+    options.plugins = [extractPlugin, ...options.plugins || []];
+  } else {
+    options.plugins = [extractPlugin, ...options.plugins || []];
+  }
+  const chunksStream = htmlToMarkdownSplitChunksStream(html, {
+    ...options,
+    headersToSplitOn: [TagIdMap.h1, TagIdMap.h2, TagIdMap.h3],
+    origin: url,
+    chunkSize: 256,
+    stripHeaders: false,
+    lengthFunction(text) {
+      return estimateTokenCount(text);
+    }
+  });
+  const chunks = [];
+  for await (const chunk of chunksStream) {
+    chunks.push(chunk);
+  }
+  return { chunks, title, description, headings: chunks.reduce((set, m) => {
+    Object.entries(m.metadata?.headers || {}).forEach(([k, v]) => {
+      if (!set[k])
+        set[k] = [];
+      if (v && !set[k].includes(v))
+        set[k].push(v);
+    });
+    return set;
+  }, {}) };
+}

package/dist/runtime/types.d.ts

@@ -11,11 +11,6 @@ export interface ModuleOptions {
      * @default false
      */
     debug?: boolean;
-    /**
-     * Bulk data API (JSONL streaming)
-     * @default '/_ai-ready/bulk'
-     */
-    bulkRoute: string | false;
     /**
      * Options to pass to mdream htmlToMarkdown function
      */
@@ -62,25 +57,61 @@ export interface ModuleOptions {
          */
         aiInput?: boolean;
     };
+    /**
+     * MCP (Model Context Protocol) configuration
+     * Control which tools and resources are exposed via MCP
+     * @default All enabled when @nuxtjs/mcp-toolkit is installed
+     */
+    mcp?: {
+        /**
+         * Enable/disable specific MCP tools
+         * @default All tools enabled
+         */
+        tools?: {
+            /** Get page by route - fetches markdown content for specific page */
+            listPages?: boolean;
+        };
+        /**
+         * Enable/disable specific MCP resources
+         * @default All resources enabled
+         */
+        resources?: {
+            /** pages://list - all pages without markdown content */
+            pages?: boolean;
+            /** pages://chunks - individual content chunks from all pages */
+            pagesChunks?: boolean;
+        };
+    };
 }
 /**
- * Individual chunk entry in bulk.jsonl (one per chunk)
- * Consumers can reassemble by route if needed
+ * Individual chunk entry in llms-full.toon (one per chunk)
+ * Used for RAG, embeddings, and semantic search
+ * Optimized for token efficiency - join with llms.toon for title/description
+ * Chunk index can be inferred from id suffix (e.g., "hash-0", "hash-1")
+ * Tabular TOON format (primitives only)
  */
 export interface BulkChunk {
     id: string;
     route: string;
-    chunkIndex: number;
     content: string;
-    headers?: Record<string, string>;
-    loc?: {
-        lines: {
-            from: number;
-            to: number;
-        };
-    };
+}
+/**
+ * Page-level entry in llms.toon (one per page)
+ * Used for page discovery, listing, and metadata queries
+ */
+export interface BulkDocument {
+    /** Page route/path */
+    route: string;
+    /** Page title */
     title: string;
+    /** Page description */
     description: string;
+    /** Full markdown content reassembled from chunks */
+    markdown: string;
+    /** Page headings structure (e.g., [{ "h1": "Title" }, { "h2": "Subtitle" }]) */
+    headings: Array<Record<string, string>>;
+    /** All chunk IDs for this page (first ID can be used as document ID) */
+    chunkIds: string[];
 }
 /**
  * Hook context for markdown processing (Nitro runtime hook)

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "nuxt-ai-ready",
   "type": "module",
-  "version": "0.1.5",
+  "version": "0.2.0",
   "description": "Best practice AI & LLM discoverability for Nuxt sites.",
   "author": {
     "name": "Harlan Wilton",
@@ -33,9 +33,10 @@
   ],
   "dependencies": {
     "@nuxt/kit": "4.2.1",
+    "@toon-format/toon": "^2.0.1",
     "consola": "^3.4.2",
     "defu": "^6.1.4",
-    "mdream": "^0.15.0",
+    "mdream": "^0.15.1",
     "minimatch": "^10.1.1",
     "nuxt-site-config": "^3.2.11",
     "pathe": "^2.0.3",

package/dist/runtime/server/mcp/prompts/explain-concept.d.ts

@@ -1,2 +0,0 @@
-declare const _default: any;
-export default _default;

package/dist/runtime/server/mcp/prompts/explain-concept.js

@@ -1,63 +0,0 @@
-import { defineMcpPrompt } from "#imports";
-import { streamBulkDocuments } from "../../utils/db.js";
-export default defineMcpPrompt({
-  name: "explain_concept",
-  description: "Get a detailed explanation of a concept by finding and reading relevant pages",
-  arguments: [
-    {
-      name: "concept",
-      description: "The concept or feature to explain",
-      required: true
-    },
-    {
-      name: "level",
-      description: "Explanation level: beginner, intermediate, or advanced",
-      required: false
-    }
-  ],
-  // @ts-expect-error untyped
-  handler: async ({ concept, level = "intermediate" }) => {
-    const searchLower = concept.toLowerCase();
-    const seenRoutes = /* @__PURE__ */ new Set();
-    const relevantPages = [];
-    for await (const doc of streamBulkDocuments()) {
-      if (seenRoutes.has(doc.route))
-        continue;
-      const matches = doc.title?.toLowerCase().includes(searchLower) || doc.description?.toLowerCase().includes(searchLower) || doc.route?.toLowerCase().includes(searchLower);
-      if (matches) {
-        seenRoutes.add(doc.route);
-        relevantPages.push({
-          route: doc.route,
-          title: doc.title,
-          description: doc.description
-        });
-        if (relevantPages.length >= 10)
-          break;
-      }
-    }
-    return {
-      messages: [
-        {
-          role: "user",
-          content: {
-            type: "text",
-            text: `Please explain "${concept}" at a ${level} level.
-
-Here are the relevant pages found: ${JSON.stringify(relevantPages, null, 2)}
-
-Please:
-1. Use get_page to read the most relevant pages (top 2-3)
-2. Synthesize the information to create an explanation that:
-   - Provides clear definitions
-   - Includes practical examples from the pages
-   - Explains use cases
-   - Mentions related concepts
-   - References the specific pages used
-
-Tailor the explanation for a ${level} audience.`
-          }
-        }
-      ]
-    };
-  }
-});

package/dist/runtime/server/mcp/prompts/find-information.d.ts

@@ -1,2 +0,0 @@
-declare const _default: any;
-export default _default;

package/dist/runtime/server/mcp/prompts/find-information.js

@@ -1,58 +0,0 @@
-import { defineMcpPrompt } from "#imports";
-import { streamBulkDocuments } from "../../utils/db.js";
-export default defineMcpPrompt({
-  name: "find_information",
-  description: "Find information about a specific topic by searching site pages and retrieving relevant content",
-  arguments: [
-    {
-      name: "topic",
-      description: "Topic, feature, or question to find information about",
-      required: true
-    },
-    {
-      name: "detail",
-      description: "Level of detail needed: summary, detailed, or comprehensive",
-      required: false
-    }
-  ],
-  // @ts-expect-error untyped
-  handler: async ({ topic, detail = "detailed" }) => {
-    const searchLower = topic.toLowerCase();
-    const seenRoutes = /* @__PURE__ */ new Set();
-    const relevantPages = [];
-    for await (const doc of streamBulkDocuments()) {
-      if (seenRoutes.has(doc.route))
-        continue;
-      const matches = doc.title?.toLowerCase().includes(searchLower) || doc.description?.toLowerCase().includes(searchLower) || doc.route?.toLowerCase().includes(searchLower);
-      if (matches) {
-        seenRoutes.add(doc.route);
-        relevantPages.push({
-          route: doc.route,
-          title: doc.title,
-          description: doc.description
-        });
-        if (relevantPages.length >= 10)
-          break;
-      }
-    }
-    return {
-      messages: [
-        {
-          role: "user",
-          content: {
-            type: "text",
-            text: `Help me find information about: "${topic}"
-
-Here are the relevant pages found: ${JSON.stringify(relevantPages, null, 2)}
-
-Please:
-1. Review the page titles and descriptions to identify the most relevant ones
-2. Use get_page to retrieve full content of the top 2-3 most relevant pages
-3. ${detail === "summary" ? "Provide a concise summary (2-3 paragraphs)" : detail === "comprehensive" ? "Provide a comprehensive explanation with all details and examples from the pages" : "Provide a detailed explanation covering the key points"}
-4. Always cite which pages the information came from`
-          }
-        }
-      ]
-    };
-  }
-});

package/dist/runtime/server/mcp/prompts/search-content.d.ts

@@ -1,2 +0,0 @@
-declare const _default: any;
-export default _default;

package/dist/runtime/server/mcp/prompts/search-content.js

@@ -1,59 +0,0 @@
-import { defineMcpPrompt } from "#imports";
-import { streamBulkDocuments } from "../../utils/db.js";
-export default defineMcpPrompt({
-  name: "browse_pages",
-  description: "Browse and discover pages by topic, with results ready for exploration",
-  arguments: [
-    {
-      name: "topic",
-      description: "Topic or keyword to search for in page titles/descriptions",
-      required: true
-    },
-    {
-      name: "maxResults",
-      description: "Maximum number of pages to retrieve",
-      required: false
-    }
-  ],
-  // @ts-expect-error untyped
-  handler: async ({ topic, maxResults = 10 }) => {
-    const searchLower = topic.toLowerCase();
-    const seenRoutes = /* @__PURE__ */ new Set();
-    const filteredPages = [];
-    let total = 0;
-    for await (const doc of streamBulkDocuments()) {
-      total++;
-      if (seenRoutes.has(doc.route))
-        continue;
-      const matches = doc.title?.toLowerCase().includes(searchLower) || doc.description?.toLowerCase().includes(searchLower) || doc.route?.toLowerCase().includes(searchLower);
-      if (matches) {
-        seenRoutes.add(doc.route);
-        filteredPages.push({
-          route: doc.route,
-          title: doc.title,
-          description: doc.description
-        });
-        if (filteredPages.length >= maxResults)
-          break;
-      }
-    }
-    return {
-      messages: [
-        {
-          role: "user",
-          content: {
-            type: "text",
-            text: `Help the user find pages about: "${topic}"
-
-Here are ${filteredPages.length} pages found (out of ${total} total pages): ${JSON.stringify(filteredPages, null, 2)}
-
-Please:
-1. Review the filtered results and identify the most relevant pages
-2. If specific pages look relevant, use get_page to retrieve their full content
-3. Summarize findings and reference the source pages`
-          }
-        }
-      ]
-    };
-  }
-});

package/dist/runtime/server/mcp/resources/all-content.d.ts

@@ -1,2 +0,0 @@
-declare const _default: any;
-export default _default;

package/dist/runtime/server/mcp/resources/all-content.js

@@ -1,14 +0,0 @@
-import { defineMcpResource, textResult } from "#imports";
-import { streamBulkDocuments } from "../../utils/db.js";
-export default defineMcpResource({
-  uri: "content://all",
-  name: "All Site Content",
-  description: "Complete indexed site content in JSONL format (newline-delimited JSON)",
-  mimeType: "application/x-ndjson",
-  handler: async () => {
-    const lines = [];
-    for await (const doc of streamBulkDocuments())
-      lines.push(JSON.stringify(doc));
-    return textResult(lines.join("\n"));
-  }
-});

package/dist/runtime/server/mcp/tools/get-page.d.ts

@@ -1,2 +0,0 @@
-declare const _default: any;
-export default _default;

package/dist/runtime/server/mcp/tools/get-page.js

@@ -1,43 +0,0 @@
-import { defineMcpTool, errorResult, jsonResult } from "#imports";
-import { streamBulkDocuments } from "../../utils/db.js";
-export default defineMcpTool({
-  name: "get_page",
-  description: `Retrieves the full content and details of a specific page by its route.
-
-WHEN TO USE: Use this tool when you know the EXACT route to a page. Common scenarios:
-- User asks for a specific page: "Get the /about page"
-- You found a relevant route from list_pages and want full content
-- You need complete page details including markdown content
-
-WHEN NOT TO USE: If you don't know the exact route, use list_pages first to discover available pages.
-
-OUTPUT: Returns complete page data including:
-- route: Page URL path
-- title: Page title
-- description: Page meta description
-- markdown: Full markdown content
-- headings: Document structure
-- id: Document identifier
-- chunkIds: Associated chunk identifiers`,
-  parameters: {
-    type: "object",
-    properties: {
-      route: {
-        type: "string",
-        description: 'The exact route/path to the page (e.g., "/docs/getting-started", "/about", "/blog/my-post")'
-      }
-    },
-    required: ["route"]
-  },
-  // @ts-expect-error untyped
-  handler: async ({ route }) => {
-    const normalizedRoute = route.startsWith("/") ? route : `/${route}`;
-    const cleanRoute = normalizedRoute.replace(/\/$/, "") || "/";
-    for await (const doc of streamBulkDocuments()) {
-      const docRoute = doc.route?.replace(/\/$/, "") || "/";
-      if (docRoute === cleanRoute || doc.route === route)
-        return jsonResult(doc);
-    }
-    return errorResult(`Page not found: ${route}. Use list_pages to discover available pages.`);
-  }
-});

package/dist/runtime/server/utils/db.d.ts

@@ -1,8 +0,0 @@
-import type { BulkChunk } from '../../types.js';
-declare module 'nitropack' {
-    interface NitroApp {
-        _bulkDocuments?: Promise<BulkChunk[]>;
-    }
-}
-export declare function streamBulkDocuments(): AsyncGenerator<BulkChunk>;
-export declare function useBulkDocuments(): Promise<BulkChunk[]>;

package/dist/runtime/server/utils/db.js

@@ -1,48 +0,0 @@
-import { useNitroApp, useRuntimeConfig } from "nitropack/runtime";
-import { logger } from "../logger.js";
-export async function* streamBulkDocuments() {
-  const config = useRuntimeConfig();
-  const bulkRoute = config["nuxt-ai-ready"]?.bulkRoute;
-  const response = await fetch(bulkRoute).catch((err) => {
-    logger.warn("Documents loading failed:", err);
-    throw err;
-  });
-  if (!response.ok || !response.body)
-    throw new Error(`Failed to fetch bulk documents: ${response.statusText}`);
-  const reader = response.body.getReader();
-  const decoder = new TextDecoder();
-  let buffer = "";
-  try {
-    while (true) {
-      const { done, value } = await reader.read();
-      if (done)
-        break;
-      buffer += decoder.decode(value, { stream: true });
-      let newlineIndex = buffer.indexOf("\n");
-      while (newlineIndex !== -1) {
-        const line = buffer.slice(0, newlineIndex).trim();
-        buffer = buffer.slice(newlineIndex + 1);
-        if (line)
-          yield JSON.parse(line);
-        newlineIndex = buffer.indexOf("\n");
-      }
-    }
-    if (buffer.trim())
-      yield JSON.parse(buffer.trim());
-  } finally {
-    reader.releaseLock();
-  }
-}
-export async function useBulkDocuments() {
-  const nitroApp = useNitroApp();
-  if (nitroApp._bulkDocuments)
-    return await nitroApp._bulkDocuments;
-  logger.debug("Lazy loading bulk documents...");
-  nitroApp._bulkDocuments = (async () => {
-    const documents = [];
-    for await (const chunk of streamBulkDocuments())
-      documents.push(chunk);
-    return documents;
-  })();
-  return await nitroApp._bulkDocuments;
-}