nuxt-ai-ready 0.7.2 → 0.7.4

package/dist/module.json

@@ -4,7 +4,7 @@
     "nuxt": ">=4.0.0"
   },
   "configKey": "aiReady",
-  "version": "0.7.2",
+  "version": "0.7.4",
   "builder": {
     "@nuxt/module-builder": "1.0.2",
     "unbuild": "3.6.1"

package/dist/module.mjs

@@ -271,8 +271,9 @@ function setupPrerenderHandler(dbPath, siteInfo, llmsTxtConfig) {
       const publicDataDir = join(nitro.options.output.publicDir, "__ai-ready");
       await mkdir(publicDataDir, { recursive: true });
       if (state.db) {
-        const pages = await queryAllPages(state.db);
-        const errorRoutesList = (await queryAllPages(state.db, { includeErrors: true })).filter((p) => p.isError).map((p) => p.route);
+        const allPages = await queryAllPages(state.db, { includeErrors: true, excludeMarkdown: true });
+        const pages = allPages.filter((p) => !p.isError);
+        const errorRoutesList = allPages.filter((p) => p.isError).map((p) => p.route);
         const jsonContent = JSON.stringify({
           pages: pages.map((p) => ({
             route: p.route,
@@ -541,7 +542,7 @@ const module$1 = defineNuxtModule({
     nuxt.hooks.hook("nitro:config", (nitroConfig) => {
       nitroConfig.experimental = nitroConfig.experimental || {};
       nitroConfig.experimental.asyncContext = true;
-      if (config.cron) {
+      if (config.cron && !nuxt.options.dev) {
         const cronSchedule = "* * * * *";
         const isVercel = nitroConfig.preset === "vercel" || nitroConfig.preset === "vercel-edge";
         if (isVercel) {
@@ -662,7 +663,7 @@ export const errorRoutes = []`;
         addServerHandler({ route: "/__ai-ready/status", handler: resolve("./runtime/server/routes/__ai-ready/status.get") });
       }
     }
-    if (config.cron) {
+    if (config.cron && !nuxt.options.dev) {
       addServerHandler({ route: "/__ai-ready/cron", handler: resolve("./runtime/server/routes/__ai-ready/cron.get") });
     }
     const isStatic = nuxt.options.nitro.static || nuxt.options._generate || false;

package/dist/runtime/llms-txt-utils.js

@@ -127,20 +127,22 @@ Canonical Origin: ${siteConfig.url}`);
     parts.push(normalizedContent);
     parts.push("");
   }
-  const pages = await queryPages(event);
-  const urls = await fetchSitemapUrls(event);
-  const errorRoutes = await queryPages(event, { where: { hasError: true } });
-  const errorSet = new Set(errorRoutes.map((e) => e.route));
-  const devModeHint = import.meta.dev && pages.length === 0 ? " (dev mode - run `nuxi generate` for page titles)" : "";
-  const prerendered = [];
+  const [pages, errorPages, urls] = await Promise.all([
+    queryPages(event),
+    queryPages(event, { where: { hasError: true } }),
+    fetchSitemapUrls(event)
+  ]);
   const seenPaths = /* @__PURE__ */ new Set();
+  const errorSet = new Set(errorPages.map((p) => p.route));
+  const prerendered = [];
   for (const page of pages) {
-    prerendered.push({ pathname: page.route, title: page.title, description: page.description });
     seenPaths.add(page.route);
+    prerendered.push({ pathname: page.route, title: page.title, description: page.description });
   }
+  const devModeHint = import.meta.dev && prerendered.length === 0 ? " (dev mode - run `nuxi generate` for page titles)" : "";
   const other = [];
   for (const url of urls) {
-    const pathname = url.loc.startsWith("http") ? new URL(url.loc).pathname : url.loc;
+    const pathname = url.loc.startsWith("/") ? url.loc : new URL(url.loc).pathname;
     if (!seenPaths.has(pathname) && !errorSet.has(pathname)) {
       other.push({ pathname });
       seenPaths.add(pathname);

package/dist/runtime/server/db/queries.d.ts

@@ -24,6 +24,7 @@ export interface PageEntry {
     headings: string;
     keywords: string[];
     updatedAt: string;
+    isError: boolean;
 }
 export interface PageData extends PageEntry {
     markdown: string;

package/dist/runtime/server/db/queries.js

@@ -29,13 +29,18 @@ async function getPrerenderDb() {
   const data = await m.readPageDataFromFilesystem();
   const pages = data.pages || [];
   const errorRoutes = new Set(data.errorRoutes || []);
+  const wantsMarkdown = (sql) => sql.includes("SELECT *") || sql.toLowerCase().includes("markdown");
   return {
     all: async (sql, params = []) => {
       const isErrorQuery = sql.includes("is_error = 1") || params.includes(1) && sql.includes("is_error");
       const excludeErrors = sql.includes("is_error = 0");
+      const includeMarkdown = wantsMarkdown(sql);
       if (isErrorQuery) {
         return pages.filter((p) => errorRoutes.has(p.route)).map((p) => ({
-          ...p,
+          route: p.route,
+          title: p.title,
+          description: p.description,
+          ...includeMarkdown ? { markdown: p.markdown } : {},
           headings: p.headings,
           keywords: JSON.stringify(p.keywords),
           updated_at: p.updatedAt,
@@ -48,7 +53,7 @@ async function getPrerenderDb() {
         route: p.route,
         title: p.title,
         description: p.description,
-        markdown: p.markdown,
+        ...includeMarkdown ? { markdown: p.markdown } : {},
         headings: p.headings,
         keywords: JSON.stringify(p.keywords),
         updated_at: p.updatedAt,
@@ -62,11 +67,12 @@ async function getPrerenderDb() {
         const page = pages.find((p) => p.route === route);
         if (!page)
           return void 0;
+        const includeMarkdown = wantsMarkdown(sql);
         return {
           route: page.route,
           title: page.title,
           description: page.description,
-          markdown: page.markdown,
+          ...includeMarkdown ? { markdown: page.markdown } : {},
           headings: page.headings,
           keywords: JSON.stringify(page.keywords),
           updated_at: page.updatedAt,
@@ -110,7 +116,8 @@ function rowToEntry(row) {
     description: row.description,
     headings: row.headings,
     keywords: JSON.parse(row.keywords || "[]"),
-    updatedAt: row.updated_at
+    updatedAt: row.updated_at,
+    isError: row.is_error === 1
   };
 }
 function rowToData(row) {

package/dist/runtime/server/db/shared.d.ts

@@ -52,16 +52,37 @@ export interface PageOutput {
     updatedAt: string;
     isError: boolean;
 }
+export interface PageMetaOutput {
+    route: string;
+    title: string;
+    description: string;
+    headings: string;
+    keywords: string[];
+    contentHash?: string;
+    updatedAt: string;
+    isError: boolean;
+}
 /**
  * Insert or update a page
  */
 export declare function insertPage(db: DatabaseAdapter, page: PageInput): Promise<void>;
+export interface QueryAllPagesOptions {
+    includeErrors?: boolean;
+    excludeMarkdown?: boolean;
+}
 /**
  * Query all pages from database
+ * @param db - Database adapter
+ * @param options - Query options
+ * @param options.excludeMarkdown - If true, omit markdown field to reduce memory usage
  */
-export declare function queryAllPages(db: DatabaseAdapter, options?: {
-    includeErrors?: boolean;
+export declare function queryAllPages(db: DatabaseAdapter, options?: QueryAllPagesOptions & {
+    excludeMarkdown: true;
+}): Promise<PageMetaOutput[]>;
+export declare function queryAllPages(db: DatabaseAdapter, options?: QueryAllPagesOptions & {
+    excludeMarkdown?: false;
 }): Promise<PageOutput[]>;
+export declare function queryAllPages(db: DatabaseAdapter, options?: QueryAllPagesOptions): Promise<PageOutput[] | PageMetaOutput[]>;
 export interface DumpRow {
     route: string;
     route_key: string;
@@ -79,7 +100,8 @@ export interface DumpRow {
     last_seen_at: number | null;
 }
 /**
- * Export database as compressed dump (base64 gzip)
+ * Export database as compressed dump (base64 gzip) using batched streaming
+ * Processes rows in batches to avoid loading entire database into memory
  */
 export declare function exportDbDump(db: DatabaseAdapter): Promise<string>;
 /**

package/dist/runtime/server/db/shared.js

@@ -86,12 +86,13 @@ export async function insertPage(db, page) {
 }
 export async function queryAllPages(db, options) {
   const where = options?.includeErrors ? "" : "WHERE is_error = 0";
-  const rows = await db.all(`SELECT route, title, description, markdown, headings, keywords, content_hash, updated_at, is_error FROM ai_ready_pages ${where}`);
+  const fields = options?.excludeMarkdown ? "route, title, description, headings, keywords, content_hash, updated_at, is_error" : "route, title, description, markdown, headings, keywords, content_hash, updated_at, is_error";
+  const rows = await db.all(`SELECT ${fields} FROM ai_ready_pages ${where}`);
   return rows.map((row) => ({
     route: row.route,
     title: row.title,
     description: row.description,
-    markdown: row.markdown,
+    ...options?.excludeMarkdown ? {} : { markdown: row.markdown },
     headings: row.headings,
     keywords: JSON.parse(row.keywords || "[]"),
     contentHash: row.content_hash || void 0,
@@ -99,12 +100,53 @@ export async function queryAllPages(db, options) {
     isError: row.is_error === 1
   }));
 }
+const DUMP_BATCH_SIZE = 500;
 export async function exportDbDump(db) {
-  const rows = await db.all(`
-    SELECT route, route_key, title, description, markdown, headings, keywords, content_hash, updated_at, indexed_at, is_error, indexed, source, last_seen_at
-    FROM ai_ready_pages
-  `);
-  return compressToBase64(rows);
+  const encoder = new TextEncoder();
+  const chunks = [];
+  const compressionStream = new CompressionStream("gzip");
+  const writer = compressionStream.writable.getWriter();
+  const reader = compressionStream.readable.getReader();
+  const readPromise = (async () => {
+    while (true) {
+      const { done, value } = await reader.read();
+      if (done)
+        break;
+      chunks.push(value);
+    }
+  })();
+  await writer.write(encoder.encode("["));
+  let offset = 0;
+  let first = true;
+  while (true) {
+    const rows = await db.all(`
+      SELECT route, route_key, title, description, markdown, headings, keywords, content_hash, updated_at, indexed_at, is_error, indexed, source, last_seen_at
+      FROM ai_ready_pages
+      ORDER BY route
+      LIMIT ${DUMP_BATCH_SIZE} OFFSET ${offset}
+    `);
+    if (rows.length === 0)
+      break;
+    for (const row of rows) {
+      const prefix = first ? "" : ",";
+      first = false;
+      await writer.write(encoder.encode(prefix + JSON.stringify(row)));
+    }
+    if (rows.length < DUMP_BATCH_SIZE)
+      break;
+    offset += DUMP_BATCH_SIZE;
+  }
+  await writer.write(encoder.encode("]"));
+  await writer.close();
+  await readPromise;
+  const totalLength = chunks.reduce((sum, chunk) => sum + chunk.length, 0);
+  const result = new Uint8Array(totalLength);
+  let pos = 0;
+  for (const chunk of chunks) {
+    result.set(chunk, pos);
+    pos += chunk.length;
+  }
+  return Buffer.from(result).toString("base64");
 }
 export async function importDbDump(db, rows) {
   for (const row of rows) {

package/dist/runtime/server/routes/__ai-ready-debug.get.js

@@ -19,8 +19,10 @@ export default eventHandler(async (event) => {
   } else {
     mode = "production";
   }
-  const pages = await queryPages(event);
-  const errorRoutes = await queryPages(event, { where: { hasError: true } });
+  const [pages, errorRoutes] = await Promise.all([
+    queryPages(event),
+    queryPages(event, { where: { hasError: true } })
+  ]);
   let source;
   if (isDev) {
     source = "empty (dev mode returns empty array)";

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "nuxt-ai-ready",
   "type": "module",
-  "version": "0.7.2",
+  "version": "0.7.4",
   "description": "Best practice AI & LLM discoverability for Nuxt sites.",
   "author": {
     "name": "Harlan Wilton",
@@ -74,11 +74,11 @@
     "@nuxtjs/robots": "^5.6.7",
     "@nuxtjs/sitemap": "^7.5.2",
     "@types/better-sqlite3": "^7.6.13",
-    "@vitest/coverage-v8": "^4.0.16",
+    "@vitest/coverage-v8": "^4.0.17",
     "@vue/test-utils": "^2.4.6",
     "@vueuse/nuxt": "^14.1.0",
     "agents": "^0.3.4",
-    "ai": "^6.0.27",
+    "ai": "^6.0.30",
     "better-sqlite3": "^12.6.0",
     "bumpp": "^10.3.2",
     "eslint": "^9.39.2",
@@ -90,7 +90,7 @@
     "playwright-core": "^1.57.0",
     "postgres": "^3.4.8",
     "typescript": "^5.9.3",
-    "vitest": "^4.0.16",
+    "vitest": "^4.0.17",
     "vue": "^3.5.26",
     "vue-router": "^4.6.4",
     "vue-tsc": "^3.2.2",