nuxt-ai-ready 0.2.3 → 0.3.0

package/README.md

@@ -4,7 +4,7 @@
 [![npm downloads][npm-downloads-src]][npm-downloads-href]
 [![Nuxt][nuxt-src]][nuxt-href]
 
-> Best practice AI & LLM discoverability for Nuxt sites 
+> Best practice AI & LLM discoverability for Nuxt sites
 
 ## Why Nuxt AI Ready?
 
@@ -27,6 +27,7 @@ Nuxt AI Ready converts your indexable pages into clean markdown that AI systems
 - 🌐 **Indexable Pages**: Integrating with [Nuxt Sitemap](https://nuxtseo.com/sitemap) to index only AI-allowed pages
 - 📦 **Bulk Chunk Export**: Exported token optimized chunks ready for RAG and semantic search
 - ⚡ **MCP Integration**: Let AI agents query your site directly
+- ⏱️ **Automatic Last Updated**: Content freshness automatically tracked and added to your [sitemap.xml](https://nuxtseo.com/docs/ai-ready/guides/automatic-updated-at)
 
 ## Installation
 

package/dist/module.json

@@ -4,7 +4,7 @@
     "nuxt": ">=4.0.0"
   },
   "configKey": "aiReady",
-  "version": "0.2.3",
+  "version": "0.3.0",
   "builder": {
     "@nuxt/module-builder": "1.0.2",
     "unbuild": "3.6.1"

package/dist/module.mjs

@@ -1,3 +1,4 @@
+import { dirname, join } from 'node:path';
 import { useLogger, useNuxt, defineNuxtModule, createResolver, addTypeTemplate, hasNuxtModule, addServerHandler, addPlugin } from '@nuxt/kit';
 import defu from 'defu';
 import { useSiteConfig, installNuxtSiteConfig, withSiteUrl } from 'nuxt-site-config/kit';
@@ -5,13 +6,113 @@ import { relative } from 'pathe';
 import { readPackageJSON } from 'pkg-types';
 import { createHash } from 'node:crypto';
 import { mkdirSync, createWriteStream } from 'node:fs';
-import { stat, open } from 'node:fs/promises';
-import { join, dirname } from 'node:path';
+import { mkdir, stat, open } from 'node:fs/promises';
 import { encodeLines } from '@toon-format/toon';
 import { createLlmsTxtStream } from 'mdream/llms-txt';
+import { createStorage } from 'unstorage';
+import fsDriver from 'unstorage/drivers/fs';
 
 const logger = useLogger("nuxt-ai-ready");
 
+function createContentHashManager(options) {
+  const { storagePath, debug = false } = options;
+  let storage;
+  let manifest = {
+    pages: {},
+    version: "1"
+  };
+  async function initStorage() {
+    await mkdir(dirname(storagePath), { recursive: true });
+    storage = createStorage({
+      driver: fsDriver({ base: dirname(storagePath) })
+    });
+  }
+  function hashContent(markdown) {
+    return createHash("sha256").update(markdown).digest("hex");
+  }
+  async function getManifest() {
+    if (!storage) {
+      await initStorage();
+    }
+    const stored = await storage.getItem("content-hashes.json");
+    if (stored) {
+      manifest = stored;
+      if (debug) {
+        logger.debug(`Loaded manifest with ${Object.keys(manifest.pages).length} pages`);
+      }
+    } else {
+      if (debug) {
+        logger.debug("No existing manifest found, starting fresh");
+      }
+    }
+    return manifest;
+  }
+  async function saveManifest() {
+    if (!storage) {
+      await initStorage();
+    }
+    await storage.setItem("content-hashes.json", manifest);
+    if (debug) {
+      logger.debug(`Saved manifest with ${Object.keys(manifest.pages).length} pages`);
+    }
+  }
+  function updatePageHash(route, markdown, previousManifest) {
+    const contentHash = hashContent(markdown);
+    const now = (/* @__PURE__ */ new Date()).toISOString();
+    const existing = previousManifest.pages[route];
+    let result;
+    if (!existing) {
+      result = {
+        contentHash,
+        updatedAt: now,
+        firstSeenAt: now
+      };
+      if (debug) {
+        logger.debug(`New page detected: ${route}`);
+      }
+    } else if (existing.contentHash !== contentHash) {
+      result = {
+        contentHash,
+        updatedAt: now,
+        firstSeenAt: existing.firstSeenAt
+      };
+      if (debug) {
+        logger.debug(`Content changed: ${route}`);
+      }
+    } else {
+      result = {
+        contentHash: existing.contentHash,
+        updatedAt: existing.updatedAt,
+        firstSeenAt: existing.firstSeenAt
+      };
+      if (debug) {
+        logger.debug(`Content unchanged: ${route}`);
+      }
+    }
+    manifest.pages[route] = result;
+    return result;
+  }
+  function setPageTimestamp(route, markdown, timestamp, previousManifest) {
+    const contentHash = hashContent(markdown);
+    const existing = previousManifest.pages[route];
+    manifest.pages[route] = {
+      contentHash,
+      updatedAt: timestamp,
+      firstSeenAt: existing?.firstSeenAt || timestamp
+    };
+    if (debug) {
+      logger.debug(`Manual timestamp set for ${route}: ${timestamp}`);
+    }
+  }
+  return {
+    getManifest,
+    saveManifest,
+    hashContent,
+    updatePageHash,
+    setPageTimestamp
+  };
+}
+
 function generateVectorId(route, chunkIdx) {
   const hash = createHash("sha256").update(route).digest("hex").substring(0, 8);
   return `${hash}-${chunkIdx}`;
@@ -30,7 +131,7 @@ async function updateFirstLine(filePath, newFirstLine) {
     await fh.close();
   }
 }
-function setupPrerenderHandler(llmsTxtConfig) {
+function setupPrerenderHandler(llmsTxtConfig, timestampsConfig) {
   const nuxt = useNuxt();
   nuxt.hooks.hook("nitro:init", async (nitro) => {
     let writer = null;
@@ -41,6 +142,18 @@ function setupPrerenderHandler(llmsTxtConfig) {
     const startTime = Date.now();
     const pagesChunksPath = join(nitro.options.output.publicDir, "llms-full.toon");
     const pagesPath = join(nitro.options.output.publicDir, "llms.toon");
+    let contentHashManager = null;
+    let previousManifest = null;
+    if (timestampsConfig?.enabled) {
+      const manifestPath = join(
+        nuxt.options.rootDir,
+        timestampsConfig.manifestPath || "node_modules/.cache/nuxt-seo/ai-index/content-hashes.json"
+      );
+      contentHashManager = createContentHashManager({
+        storagePath: manifestPath,
+        debug: !!nuxt.options.debug
+      });
+    }
     nitro.hooks.hook("prerender:generate", async (route) => {
       if (!route.fileName?.endsWith(".md")) {
         return;
@@ -61,15 +174,39 @@ function setupPrerenderHandler(llmsTxtConfig) {
           notes: llmsTxtConfig.notes
         });
         writer = stream.getWriter();
+        if (contentHashManager && !previousManifest) {
+          previousManifest = await contentHashManager.getManifest();
+        }
         mkdirSync(dirname(pagesChunksPath), { recursive: true });
         mkdirSync(dirname(pagesPath), { recursive: true });
         chunksStream = createWriteStream(pagesChunksPath, { encoding: "utf-8" });
         chunksStream.write("pageChunks[999999]{id,route,content}:\n");
         pagesStream = createWriteStream(pagesPath, { encoding: "utf-8" });
-        pagesStream.write("pages[999999]{route,title,description,headings,chunkIds}:\n");
+        pagesStream.write("pages[999999]{route,title,description,headings,chunkIds,updatedAt}:\n");
       }
-      const { chunks, title, description, headings } = JSON.parse(route.contents || "{}");
+      const { chunks, title, description, headings, updatedAt: metaUpdatedAt } = JSON.parse(route.contents || "{}");
       const markdown = chunks.map((c) => c.content).join("\n\n");
+      let pageTimestamp = {
+        updatedAt: (/* @__PURE__ */ new Date()).toISOString()
+      };
+      let usedMetaTimestamp = false;
+      if (metaUpdatedAt) {
+        const parsedDate = new Date(metaUpdatedAt);
+        if (!Number.isNaN(parsedDate.getTime())) {
+          pageTimestamp.updatedAt = parsedDate.toISOString();
+          usedMetaTimestamp = true;
+          if (contentHashManager && previousManifest) {
+            contentHashManager.setPageTimestamp(pageRoute, markdown, pageTimestamp.updatedAt, previousManifest);
+          }
+        }
+      }
+      if (!usedMetaTimestamp && contentHashManager && previousManifest) {
+        pageTimestamp = contentHashManager.updatePageHash(
+          pageRoute,
+          markdown,
+          previousManifest
+        );
+      }
       await writer.write({
         url: pageRoute,
         title,
@@ -121,7 +258,8 @@ function setupPrerenderHandler(llmsTxtConfig) {
           ([tag, texts]) => texts.map((text) => `${tag}:${text}`)
         ).join("|") : "",
         // Join chunkIds array to comma-separated string
-        chunkIds: chunkIds.join(",")
+        chunkIds: chunkIds.join(","),
+        updatedAt: pageTimestamp.updatedAt
       };
       if (pagesStream) {
         const lines = Array.from(encodeLines({ pages: [pageDoc] }));
@@ -152,7 +290,11 @@ function setupPrerenderHandler(llmsTxtConfig) {
         });
       }
       await updateFirstLine(pagesChunksPath, `pageChunks[${chunksProcessed}]{id,route,content}:`);
-      await updateFirstLine(pagesPath, `pages[${pageCount}]{route,title,description,headings,chunkIds}:`);
+      await updateFirstLine(pagesPath, `pages[${pageCount}]{route,title,description,headings,chunkIds,updatedAt}:`);
+      if (contentHashManager) {
+        await contentHashManager.saveManifest();
+        logger.debug("Saved content hash manifest");
+      }
       logger.info(`Wrote llms-full.toon with ${chunksProcessed} chunks`);
       logger.info(`Wrote llms.toon with ${pageCount} pages`);
       const llmsTxtPath = join(nitro.options.output.publicDir, "llms.txt");
@@ -227,6 +369,10 @@ const module$1 = defineNuxtModule({
         maxAge: 3600,
         // 1 hour
         swr: true
+      },
+      timestamps: {
+        enabled: false,
+        manifestPath: "node_modules/.cache/nuxt-seo/ai-index/content-hashes.json"
       }
     };
   },
@@ -350,6 +496,7 @@ export {}
     await nuxt.callHook("ai-ready:llms-txt", llmsTxtPayload);
     mergedLlmsTxt.sections = llmsTxtPayload.sections;
     mergedLlmsTxt.notes = llmsTxtPayload.notes.length > 0 ? llmsTxtPayload.notes : void 0;
+    const timestampsManifestPath = config.timestamps?.enabled ? join(nuxt.options.rootDir, config.timestamps.manifestPath || "node_modules/.cache/nuxt-seo/ai-index/content-hashes.json") : void 0;
     nuxt.options.runtimeConfig["nuxt-ai-ready"] = {
       version: version || "0.0.0",
       debug: config.debug || false,
@@ -358,8 +505,15 @@ export {}
         maxAge: 3600,
         swr: true
       }),
-      llmsTxt: mergedLlmsTxt
+      llmsTxt: mergedLlmsTxt,
+      timestampsManifestPath
     };
+    if (config.timestamps?.enabled && hasNuxtModule("@nuxtjs/sitemap")) {
+      nuxt.hook("nitro:config", (nitroConfig) => {
+        nitroConfig.plugins = nitroConfig.plugins || [];
+        nitroConfig.plugins.push(resolve("./runtime/server/plugins/sitemap-lastmod"));
+      });
+    }
     addServerHandler({
       middleware: true,
       handler: resolve("./runtime/server/middleware/mdream")
@@ -371,7 +525,7 @@ export {}
     addServerHandler({ route: "/llms-full.txt", handler: resolve("./runtime/server/routes/llms.txt.get") });
     const isStatic = nuxt.options.nitro.static || nuxt.options._generate || false;
     if (isStatic || nuxt.options.nitro.prerender?.routes?.length) {
-      setupPrerenderHandler(mergedLlmsTxt);
+      setupPrerenderHandler(mergedLlmsTxt, config.timestamps);
     }
     nuxt.options.nitro.routeRules = nuxt.options.nitro.routeRules || {};
     nuxt.options.nitro.routeRules["/llms.toon"] = { headers: { "Content-Type": "text/toon; charset=utf-8" } };

package/dist/runtime/server/plugins/sitemap-lastmod.d.ts

@@ -0,0 +1,2 @@
+declare const _default: import("nitropack/types").NitroAppPlugin;
+export default _default;

package/dist/runtime/server/plugins/sitemap-lastmod.js

@@ -0,0 +1,22 @@
+import { readFile } from "node:fs/promises";
+import { defineNitroPlugin, useRuntimeConfig } from "nitropack/runtime";
+export default defineNitroPlugin((nitroApp) => {
+  const config = useRuntimeConfig();
+  const manifestPath = config["nuxt-ai-ready"]?.timestampsManifestPath;
+  if (!manifestPath) {
+    return;
+  }
+  nitroApp.hooks.hook("sitemap:resolved", async (ctx) => {
+    const manifest = await readFile(manifestPath, "utf-8").then((data) => JSON.parse(data)).catch(() => null);
+    if (!manifest) {
+      return;
+    }
+    for (const url of ctx.urls) {
+      const route = url.loc.replace(/^https?:\/\/[^/]+/, "").replace(/\/$/, "") || "/";
+      const pageData = manifest.pages[route];
+      if (pageData?.updatedAt) {
+        url.lastmod = pageData.updatedAt;
+      }
+    }
+  });
+});

package/dist/runtime/server/utils.d.ts

@@ -1,7 +1,8 @@
 import type { ModulePublicRuntimeConfig } from '../../module.js';
 export declare function convertHtmlToMarkdownChunks(html: string, url: string, mdreamOptions: ModulePublicRuntimeConfig['mdreamOptions']): Promise<{
+    headings: Record<string, string[]>;
+    updatedAt?: string | undefined;
     chunks: import("mdream").MarkdownChunk[];
     title: string;
     description: string;
-    headings: Record<string, string[]>;
 }>;

package/dist/runtime/server/utils.js

@@ -6,12 +6,19 @@ import { estimateTokenCount } from "tokenx";
 export async function convertHtmlToMarkdownChunks(html, url, mdreamOptions) {
   let title = "";
   let description = "";
+  let updatedAt;
   const extractPlugin = extractionPlugin({
     title(el) {
       title = el.textContent;
     },
     'meta[name="description"]': (el) => {
       description = el.attributes.content || "";
+    },
+    // Extract timestamp from various meta tag formats
+    'meta[property="article:modified_time"], meta[name="last-modified"], meta[name="updated"], meta[property="og:updated_time"], meta[name="lastmod"]': (el) => {
+      if (!updatedAt && el.attributes.content) {
+        updatedAt = el.attributes.content;
+      }
     }
   });
   let options = {
@@ -38,13 +45,19 @@ export async function convertHtmlToMarkdownChunks(html, url, mdreamOptions) {
   for await (const chunk of chunksStream) {
     chunks.push(chunk);
   }
-  return { chunks, title, description, headings: chunks.reduce((set, m) => {
-    Object.entries(m.metadata?.headers || {}).forEach(([k, v]) => {
-      if (!set[k])
-        set[k] = [];
-      if (v && !set[k].includes(v))
-        set[k].push(v);
-    });
-    return set;
-  }, {}) };
+  return {
+    chunks,
+    title,
+    description,
+    ...updatedAt && { updatedAt },
+    headings: chunks.reduce((set, m) => {
+      Object.entries(m.metadata?.headers || {}).forEach(([k, v]) => {
+        if (!set[k])
+          set[k] = [];
+        if (v && !set[k].includes(v))
+          set[k].push(v);
+      });
+      return set;
+    }, {})
+  };
 }

package/dist/runtime/types.d.ts

@@ -82,6 +82,21 @@ export interface ModuleOptions {
             pagesChunks?: boolean;
         };
     };
+    /**
+     * Content timestamp tracking configuration
+     */
+    timestamps?: {
+        /**
+         * Enable timestamp tracking
+         * @default false
+         */
+        enabled?: boolean;
+        /**
+         * Path to store content hash manifest
+         * @default 'node_modules/.cache/nuxt-seo/ai-index/content-hashes.json'
+         */
+        manifestPath?: string;
+    };
 }
 /**
  * Individual chunk entry in llms-full.toon (one per chunk)
@@ -112,6 +127,8 @@ export interface BulkDocument {
     headings: Array<Record<string, string>>;
     /** All chunk IDs for this page (first ID can be used as document ID) */
     chunkIds: string[];
+    /** ISO 8601 timestamp of last content update */
+    updatedAt?: string;
 }
 /**
  * Hook context for markdown processing (Nitro runtime hook)

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "nuxt-ai-ready",
   "type": "module",
-  "version": "0.2.3",
+  "version": "0.3.0",
   "description": "Best practice AI & LLM discoverability for Nuxt sites.",
   "author": {
     "name": "Harlan Wilton",
@@ -33,20 +33,21 @@
   ],
   "dependencies": {
     "@nuxt/kit": "4.2.1",
-    "@toon-format/toon": "^2.0.1",
+    "@toon-format/toon": "^2.1.0",
     "consola": "^3.4.2",
     "defu": "^6.1.4",
-    "mdream": "^0.15.1",
+    "mdream": "^0.15.2",
     "minimatch": "^10.1.1",
     "nuxt-site-config": "^3.2.11",
     "pathe": "^2.0.3",
     "pkg-types": "^2.3.0",
     "std-env": "^3.10.0",
     "tokenx": "^1.2.1",
-    "ufo": "^1.6.1"
+    "ufo": "^1.6.1",
+    "unstorage": "^1.17.3"
   },
   "devDependencies": {
-    "@antfu/eslint-config": "^6.4.1",
+    "@antfu/eslint-config": "^6.4.2",
     "@arethetypeswrong/cli": "^0.18.2",
     "@headlessui/vue": "^1.7.23",
     "@nuxt/content": "^3.9.0",
@@ -57,7 +58,7 @@
     "@nuxtjs/eslint-config-typescript": "^12.1.0",
     "@nuxtjs/i18n": "^10.2.1",
     "@nuxtjs/mcp-toolkit": "^0.5.1",
-    "@nuxtjs/robots": "^5.6.0",
+    "@nuxtjs/robots": "^5.6.1",
     "@nuxtjs/sitemap": "^7.4.8",
     "@vitest/coverage-v8": "^4.0.15",
     "@vueuse/nuxt": "^14.1.0",
@@ -75,7 +76,7 @@
     "vitest": "^4.0.15",
     "vue": "^3.5.25",
     "vue-router": "^4.6.3",
-    "vue-tsc": "^3.1.5",
+    "vue-tsc": "^3.1.6",
     "zod": "^4.1.13"
   },
   "resolutions": {