nuxt-ai-ready 0.3.8 → 0.4.0

package/dist/runtime/types.d.ts

@@ -65,40 +65,18 @@ export interface ModuleOptions {
     mcp?: {
         /** Enable MCP tools (list-pages) @default true */
         tools?: boolean;
-        /** Enable MCP resources (pages, pages-chunks) @default true */
+        /** Enable MCP resources (pages) @default true */
         resources?: boolean;
     };
     /**
-     * Content timestamp tracking configuration
+     * Cache duration for llms.txt in seconds (runtime generation)
+     * Set to 0 to disable caching
+     * @default 600 (10 minutes)
      */
-    timestamps?: {
-        /**
-         * Enable timestamp tracking
-         * @default false
-         */
-        enabled?: boolean;
-        /**
-         * Path to store content hash manifest
-         * @default 'node_modules/.cache/nuxt-seo/ai-index/content-hashes.json'
-         */
-        manifestPath?: string;
-    };
-}
-/**
- * Individual chunk entry in llms-full.toon (one per chunk)
- * Used for RAG, embeddings, and semantic search
- * Optimized for token efficiency - join with llms.toon for title/description
- * Chunk index can be inferred from id suffix (e.g., "hash-0", "hash-1")
- * Tabular TOON format (primitives only)
- */
-export interface BulkChunk {
-    id: string;
-    route: string;
-    content: string;
+    cacheMaxAgeSeconds?: number;
 }
 /**
- * Page-level entry in llms.toon (one per page)
- * Used for page discovery, listing, and metadata queries
+ * Page-level entry for discovery and metadata queries
  */
 export interface BulkDocument {
     /** Page route/path */
@@ -107,12 +85,10 @@ export interface BulkDocument {
     title: string;
     /** Page description */
     description: string;
-    /** Full markdown content reassembled from chunks */
+    /** Full markdown content */
     markdown: string;
     /** Page headings structure (e.g., [{ "h1": "Title" }, { "h2": "Subtitle" }]) */
     headings: Array<Record<string, string>>;
-    /** All chunk IDs for this page (first ID can be used as document ID) */
-    chunkIds: string[];
     /** ISO 8601 timestamp of last content update */
     updatedAt?: string;
 }
@@ -123,13 +99,13 @@ export interface BulkDocument {
  * You can modify the markdown content before it's returned to the client.
  *
  * @example Modify markdown content
- * nitroApp.hooks.hook('mdream:markdown', async (context) => {
+ * nitroApp.hooks.hook('ai-ready:markdown', async (context) => {
  *   // Add a footer to all markdown
  *   context.markdown += '\n\n---\n*Generated with mdream*'
  * })
  *
  * @example Track conversions and add headers
- * nitroApp.hooks.hook('mdream:markdown', async (context) => {
+ * nitroApp.hooks.hook('ai-ready:markdown', async (context) => {
  *   console.log(`Converted ${context.route} (${context.title})`)
  *
  *   // Add custom headers
@@ -186,33 +162,30 @@ export interface LlmsTxtConfig {
     notes?: string | string[];
 }
 /**
- * Hook context for chunk processing (Nitro build-time hook)
+ * Hook context for page markdown processing (Nuxt build-time hook)
  *
- * Called during prerender for each generated chunk, allowing modules
- * to implement RAG tooling (e.g., vector embeddings, search indexing)
+ * Called once per page during prerender when markdown content has changed.
+ * Allows integrations to process page content (e.g., for embedding generation).
  *
- * @example Process chunks for vector search
- * nitro.hooks.hook('ai-ready:chunk', async (context) => {
- *   const embedding = await generateEmbedding(context.chunk.content)
- *   await vectorDb.insert({
- *     id: context.chunk.id,
- *     embedding,
- *     metadata: {
- *       route: context.route,
- *       title: context.title,
- *     }
- *   })
+ * @example Process page markdown
+ * nuxt.hooks.hook('ai-ready:page:markdown', async (context) => {
+ *   console.log(`Processing ${context.route}: ${context.title}`)
+ *   // Generate embeddings, update search index, etc.
  * })
  */
-export interface ChunkContext {
-    /** The chunk data that will be written to bulk JSONL */
-    chunk: BulkChunk;
+export interface PageMarkdownContext {
     /** The route being processed (e.g., '/about') */
     route: string;
+    /** The markdown content */
+    markdown: string;
     /** Page title extracted from HTML */
     title: string;
     /** Page description from meta tags */
     description: string;
-    /** Headings extracted from the page */
-    headings: Array<Record<string, string>>;
+}
+declare module 'nitropack/types' {
+    interface NitroRuntimeHooks {
+        'ai-ready:markdown': (context: MarkdownContext) => void | Promise<void>;
+        'ai-ready:mdreamConfig': (config: import('mdream').HTMLToMarkdownOptions) => void | Promise<void>;
+    }
 }

package/dist/types.d.mts

@@ -5,7 +5,7 @@ declare module '@nuxt/schema' {
   interface PublicRuntimeConfig extends ModulePublicRuntimeConfig {}
 }
 
-export { type BulkChunk, type ModuleOptions } from '../dist/runtime/types.js'
+export { type ModuleOptions } from '../dist/runtime/types.js'
 
 export { default } from './module.mjs'
 

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "nuxt-ai-ready",
   "type": "module",
-  "version": "0.3.8",
+  "version": "0.4.0",
   "description": "Best practice AI & LLM discoverability for Nuxt sites.",
   "author": {
     "name": "Harlan Wilton",
@@ -31,25 +31,26 @@
   "files": [
     "dist"
   ],
+  "peerDependencies": {
+    "@nuxtjs/sitemap": "^7.5.0"
+  },
   "dependencies": {
     "@clack/prompts": "^0.11.0",
     "@nuxt/kit": "4.2.2",
-    "@toon-format/toon": "^2.1.0",
     "consola": "^3.4.2",
     "defu": "^6.1.4",
+    "fuse.js": "^7.1.0",
     "mdream": "^0.15.3",
-    "minimatch": "^10.1.1",
-    "nuxt-site-config": "3.2.11",
+    "nuxt-site-config": "3.2.14",
     "ofetch": "^1.5.1",
     "pathe": "^2.0.3",
     "pkg-types": "^2.3.0",
     "std-env": "3.10.0",
-    "tokenx": "^1.2.1",
     "ufo": "^1.6.1",
     "unstorage": "^1.17.3"
   },
   "devDependencies": {
-    "@antfu/eslint-config": "^6.6.1",
+    "@antfu/eslint-config": "^6.7.1",
     "@arethetypeswrong/cli": "^0.18.2",
     "@headlessui/vue": "^1.7.23",
     "@nuxt/content": "^3.9.0",
@@ -60,29 +61,29 @@
     "@nuxtjs/eslint-config-typescript": "^12.1.0",
     "@nuxtjs/i18n": "^10.2.1",
     "@nuxtjs/mcp-toolkit": "^0.5.2",
-    "@nuxtjs/robots": "^5.6.3",
-    "@nuxtjs/sitemap": "^7.4.9",
-    "@vitest/coverage-v8": "^4.0.15",
+    "@nuxtjs/robots": "^5.6.7",
+    "@nuxtjs/sitemap": "7.5.0",
+    "@vitest/coverage-v8": "^4.0.16",
     "@vueuse/nuxt": "^14.1.0",
     "better-sqlite3": "^12.5.0",
     "bumpp": "^10.3.2",
-    "eslint": "^9.39.1",
+    "eslint": "^9.39.2",
     "execa": "^9.6.1",
     "happy-dom": "^20.0.11",
     "nuxt": "^4.2.2",
-    "nuxt-site-config": "3.2.11",
+    "nuxt-site-config": "3.2.14",
     "playwright": "^1.57.0",
     "playwright-core": "^1.57.0",
     "postgres": "^3.4.7",
     "typescript": "^5.9.3",
-    "vitest": "^4.0.15",
+    "vitest": "^4.0.16",
     "vue": "^3.5.25",
-    "vue-router": "^4.6.3",
+    "vue-router": "^4.6.4",
     "vue-tsc": "^3.1.8",
-    "zod": "^4.1.13"
+    "zod": "^4.2.1"
   },
   "resolutions": {
-    "nuxt-ai-ready": "wo rkspace:*"
+    "nuxt-ai-ready": "workspace:*"
   },
   "scripts": {
     "lint": "eslint . --fix",

package/dist/runtime/llms-txt.js

@@ -1,35 +0,0 @@
-function normalizeLink(link) {
-  const parts = [];
-  parts.push(`- [${link.title}](${link.href})`);
-  if (link.description) {
-    parts.push(`  ${link.description}`);
-  }
-  return parts.join("\n");
-}
-function normalizeSection(section) {
-  const parts = [];
-  parts.push(`## ${section.title}`);
-  parts.push("");
-  if (section.description) {
-    const descriptions = Array.isArray(section.description) ? section.description : [section.description];
-    parts.push(...descriptions);
-    parts.push("");
-  }
-  if (section.links?.length) {
-    parts.push(...section.links.map(normalizeLink));
-  }
-  return parts.join("\n");
-}
-export function normalizeLlmsTxtConfig(config) {
-  const parts = [];
-  if (config.sections?.length) {
-    parts.push(...config.sections.map(normalizeSection));
-  }
-  if (config.notes) {
-    parts.push("## Notes");
-    parts.push("");
-    const notes = Array.isArray(config.notes) ? config.notes : [config.notes];
-    parts.push(...notes);
-  }
-  return parts.join("\n\n");
-}

package/dist/runtime/server/mcp/dev/tools/list-pages.js

@@ -1,10 +0,0 @@
-import { getDevPages, jsonResult } from "../utils.js";
-export default {
-  name: "list_pages",
-  description: "Lists all available pages with their routes. In dev mode, returns JSON from sitemap/routes (TOON format unavailable until build).",
-  inputSchema: {},
-  async handler() {
-    const pages = await getDevPages();
-    return jsonResult(pages);
-  }
-};

package/dist/runtime/server/mcp/dev/utils.js

@@ -1,34 +0,0 @@
-import routes from "#ai-ready/routes.mjs";
-import { useRuntimeConfig } from "nitropack/runtime";
-export { jsonResult } from "../utils.js";
-function routeToRegex(routePath) {
-  const pattern = routePath.replace(/:[^/]+\(\.\*\)\*?/g, ".*").replace(/:[^/]+/g, "[^/]+");
-  return new RegExp(`^${pattern}$`);
-}
-function matchRoute(path, routeRecords) {
-  for (const r of routeRecords) {
-    if (routeToRegex(r.path).test(path))
-      return r;
-  }
-}
-export async function getDevPages() {
-  const config = useRuntimeConfig()["nuxt-ai-ready"];
-  if (!config.hasSitemap)
-    return routes.map((r) => ({ route: r.path, name: r.name, meta: r.meta }));
-  const { parseSitemapXml } = await import("@nuxtjs/sitemap/utils");
-  const sitemapRes = await fetch("/sitemap.xml");
-  if (!sitemapRes.ok)
-    return routes.map((r) => ({ route: r.path, name: r.name, meta: r.meta }));
-  const xml = await sitemapRes.text();
-  const { urls } = await parseSitemapXml(xml);
-  return urls.map((entry) => {
-    const pathname = typeof entry === "string" ? new URL(entry).pathname : new URL(entry.loc).pathname;
-    const matched = matchRoute(pathname, routes);
-    return {
-      route: pathname,
-      ...typeof entry !== "string" && entry.lastmod && { lastmod: entry.lastmod },
-      ...matched?.name && { name: matched.name },
-      ...matched?.meta && { meta: matched.meta }
-    };
-  });
-}

package/dist/runtime/server/mcp/prod/resources/pages-chunks.js

@@ -1,25 +0,0 @@
-import { getNitroOrigin } from "#site-config/server/composables";
-import { useEvent } from "nitropack/runtime";
-export default {
-  uri: "resource://nuxt-ai-ready/pages-chunks",
-  name: "All Page Chunks",
-  description: "Chunk-level content (id, route, content) in TOON format for RAG/embeddings. Join with pages resource using id field - match chunk.id with page.chunkIds[] to get title, description, headings. TOON is token-efficient JSON encoding (see https://toonformat.dev)",
-  metadata: {
-    mimeType: "text/toon"
-  },
-  cache: "1h",
-  async handler(uri) {
-    const event = useEvent();
-    const nitroOrigin = getNitroOrigin(event);
-    const text = await $fetch(`/llms-full.toon`, {
-      baseURL: nitroOrigin
-    });
-    return {
-      contents: [{
-        uri: uri.toString(),
-        mimeType: "text/toon",
-        text
-      }]
-    };
-  }
-};

package/dist/runtime/server/mcp/prod/resources/pages.js

@@ -1,25 +0,0 @@
-import { getNitroOrigin } from "#site-config/server/composables";
-import { useEvent } from "nitropack/runtime";
-export default {
-  uri: "resource://nuxt-ai-ready/pages",
-  name: "All Pages",
-  description: "Page-level metadata (route, title, description, headings, chunkIds, updatedAt) in TOON format. Each page includes chunkIds to join with pages-chunks resource for chunk-level content. TOON is token-efficient JSON encoding (see https://toonformat.dev)",
-  metadata: {
-    mimeType: "text/toon"
-  },
-  cache: "1h",
-  async handler(uri) {
-    const event = useEvent();
-    const nitroOrigin = getNitroOrigin(event);
-    const text = await $fetch(`/llms.toon`, {
-      baseURL: nitroOrigin
-    });
-    return {
-      contents: [{
-        uri: uri.toString(),
-        mimeType: "text/toon",
-        text
-      }]
-    };
-  }
-};

package/dist/runtime/server/mcp/prod/tools/list-pages.js

@@ -1,21 +0,0 @@
-import { getNitroOrigin } from "#site-config/server/composables";
-import { useEvent } from "nitropack/runtime";
-import { z } from "zod";
-import { toonResult } from "../../utils.js";
-const schema = {
-  mode: z.enum(["chunks", "minimal"]).default("minimal").describe("Return individual content chunks (chunks) or page-level metadata (minimal)")
-};
-export default {
-  name: "list_pages",
-  description: 'Lists all available pages in TOON format (token-efficient). Use "chunks" mode to get individual content chunks, or "minimal" for page-level metadata.',
-  inputSchema: schema,
-  cache: "1h",
-  async handler({ mode }) {
-    const event = useEvent();
-    const nitroOrigin = getNitroOrigin(event);
-    const text = await $fetch(mode === "chunks" ? "/llms-full.toon" : "/llms.toon", {
-      baseURL: nitroOrigin
-    });
-    return toonResult(text);
-  }
-};

package/dist/runtime/server/mcp/utils.d.ts

@@ -1,3 +0,0 @@
-import type { CallToolResult } from '@modelcontextprotocol/sdk/types.js';
-export declare function jsonResult(data: any, pretty?: boolean): CallToolResult;
-export declare function toonResult(toon: string): CallToolResult;

package/dist/runtime/server/mcp/utils.js

@@ -1,7 +0,0 @@
-export function jsonResult(data, pretty = true) {
-  const text = pretty ? JSON.stringify(data, null, 2) : JSON.stringify(data);
-  return { content: [{ type: "text", text }] };
-}
-export function toonResult(toon) {
-  return { content: [{ type: "text", text: toon }] };
-}

package/dist/runtime/server/middleware/mdream.js

@@ -1,148 +0,0 @@
-import { withSiteUrl } from "#site-config/server/composables/utils";
-import { createError, defineEventHandler, getHeader, setHeader } from "h3";
-import { htmlToMarkdown } from "mdream";
-import { extractionPlugin } from "mdream/plugins";
-import { withMinimalPreset } from "mdream/preset/minimal";
-import { useNitroApp, useRuntimeConfig } from "nitropack/runtime";
-import { logger } from "../logger.js";
-import { convertHtmlToMarkdownChunks } from "../utils.js";
-function normalizeWhitespace(text) {
-  return text.replace(/\u00A0/g, " ");
-}
-function shouldServeMarkdown(event) {
-  const accept = getHeader(event, "accept") || "";
-  const secFetchDest = getHeader(event, "sec-fetch-dest") || "";
-  if (secFetchDest === "document") {
-    return false;
-  }
-  if (accept.includes("text/html")) {
-    return false;
-  }
-  return accept.includes("*/*") || accept.includes("text/markdown");
-}
-async function convertHtmlToMarkdown(html, url, config, route, event) {
-  const nitroApp = useNitroApp();
-  let title = "";
-  let description = "";
-  const headings = [];
-  const extractPlugin = extractionPlugin({
-    title(el) {
-      title = el.textContent;
-    },
-    'meta[name="description"]': (el) => {
-      description = el.attributes.content || "";
-    },
-    "h1, h2, h3, h4, h5, h6": (el) => {
-      const text = el.textContent?.trim();
-      const level = el.name.toLowerCase();
-      if (text)
-        headings.push({ [level]: text });
-    }
-  });
-  let options = {
-    origin: url,
-    ...config.mdreamOptions
-  };
-  if (config.mdreamOptions?.preset === "minimal") {
-    options = withMinimalPreset(options);
-    options.plugins = [extractPlugin, ...options.plugins || []];
-  } else {
-    options.plugins = [extractPlugin, ...options.plugins || []];
-  }
-  await nitroApp.hooks.callHook("ai-ready:mdreamConfig", options);
-  let markdown = htmlToMarkdown(html, options);
-  const context = {
-    html,
-    markdown,
-    route,
-    title,
-    description,
-    isPrerender: Boolean(import.meta.prerender),
-    event
-  };
-  await nitroApp.hooks.callHook("ai-ready:markdown", context);
-  markdown = normalizeWhitespace(context.markdown);
-  return { markdown, title: normalizeWhitespace(title), description: normalizeWhitespace(description), headings };
-}
-export default defineEventHandler(async (event) => {
-  let path = event.path;
-  const config = useRuntimeConfig(event)["nuxt-ai-ready"];
-  if (path.startsWith("/api") || path.startsWith("/_") || path.startsWith("/@")) {
-    return;
-  }
-  const lastSegment = path.split("/").pop() || "";
-  const hasExtension = lastSegment.includes(".");
-  const extension = hasExtension ? lastSegment.substring(lastSegment.lastIndexOf(".")) : "";
-  if (hasExtension && extension !== ".md") {
-    return;
-  }
-  const hasMarkdownExtension = path.endsWith(".md");
-  const clientPrefersMarkdown = shouldServeMarkdown(event);
-  if (!hasMarkdownExtension && !clientPrefersMarkdown) {
-    return;
-  }
-  if (hasMarkdownExtension) {
-    path = path.slice(0, -3);
-  }
-  if (path === "/index") {
-    path = "/";
-  }
-  let html;
-  try {
-    const response = await event.fetch(path);
-    if (!response.ok) {
-      if (hasMarkdownExtension) {
-        return createError({
-          statusCode: response.status,
-          statusMessage: response.statusText,
-          message: `Failed to fetch HTML for ${path}`
-        });
-      }
-      return;
-    }
-    const contentType = response.headers.get("content-type") || "";
-    if (!contentType.includes("text/html")) {
-      if (hasMarkdownExtension) {
-        return createError({
-          statusCode: 415,
-          statusMessage: "Unsupported Media Type",
-          message: `Expected text/html but got ${contentType} for ${path}`
-        });
-      }
-      return;
-    }
-    html = await response.text();
-  } catch (e) {
-    logger.error(`Failed to fetch HTML for ${path}`, e);
-    if (hasMarkdownExtension) {
-      return createError({
-        statusCode: 500,
-        statusMessage: "Internal Server Error",
-        message: `Failed to fetch HTML for ${path}`
-      });
-    }
-    return;
-  }
-  if (import.meta.prerender) {
-    const result2 = convertHtmlToMarkdownChunks(
-      html,
-      withSiteUrl(event, path),
-      config.mdreamOptions
-    );
-    return JSON.stringify(result2);
-  }
-  const result = await convertHtmlToMarkdown(
-    html,
-    withSiteUrl(event, path),
-    config,
-    path,
-    event
-  );
-  setHeader(event, "content-type", "text/markdown; charset=utf-8");
-  if (config.markdownCacheHeaders) {
-    const { maxAge, swr } = config.markdownCacheHeaders;
-    const cacheControl = swr ? `public, max-age=${maxAge}, stale-while-revalidate=${maxAge}` : `public, max-age=${maxAge}`;
-    setHeader(event, "cache-control", cacheControl);
-  }
-  return result.markdown;
-});

package/dist/runtime/server/plugins/sitemap-lastmod.d.ts

@@ -1,2 +0,0 @@
-declare const _default: import("nitropack/types").NitroAppPlugin;
-export default _default;

package/dist/runtime/server/plugins/sitemap-lastmod.js

@@ -1,22 +0,0 @@
-import { readFile } from "node:fs/promises";
-import { defineNitroPlugin, useRuntimeConfig } from "nitropack/runtime";
-export default defineNitroPlugin((nitroApp) => {
-  const config = useRuntimeConfig();
-  const manifestPath = config["nuxt-ai-ready"]?.timestampsManifestPath;
-  if (!manifestPath) {
-    return;
-  }
-  nitroApp.hooks.hook("sitemap:resolved", async (ctx) => {
-    const manifest = await readFile(manifestPath, "utf-8").then((data) => JSON.parse(data)).catch(() => null);
-    if (!manifest) {
-      return;
-    }
-    for (const url of ctx.urls) {
-      const route = url.loc.replace(/^https?:\/\/[^/]+/, "").replace(/\/$/, "") || "/";
-      const pageData = manifest.pages[route];
-      if (pageData?.updatedAt) {
-        url.lastmod = pageData.updatedAt;
-      }
-    }
-  });
-});