nuxt-ai-ready 0.6.2 → 0.7.0

package/dist/module.mjs

@@ -1,6 +1,6 @@
-import { appendFile, mkdir, writeFile, readFile, stat, access } from 'node:fs/promises';
-import { join as join$1, dirname } from 'node:path';
-import { useLogger, useNuxt, defineNuxtModule, createResolver, addTypeTemplate, hasNuxtModule, addServerHandler, addPlugin } from '@nuxt/kit';
+import { mkdir, writeFile, appendFile, stat, access } from 'node:fs/promises';
+import { join, dirname } from 'node:path';
+import { useLogger, useNuxt, addTypeTemplate, defineNuxtModule, createResolver, hasNuxtModule, addServerHandler, addPlugin } from '@nuxt/kit';
 import defu from 'defu';
 import { useSiteConfig, installNuxtSiteConfig, withSiteUrl } from 'nuxt-site-config/kit';
 import { readPackageJSON } from 'pkg-types';
@@ -10,7 +10,9 @@ import { isTest, isCI } from 'std-env';
 import { parseSitemapXml } from '@nuxtjs/sitemap/utils';
 import { colorize } from 'consola/utils';
 import { withBase } from 'ufo';
-import { join, isAbsolute } from 'pathe';
+import { createAdapter, initSchema, computeContentHash, insertPage, queryAllPages, exportDbDump } from '../dist/runtime/server/db/shared.js';
+import { buildLlmsFullTxtHeader, formatPageForLlmsFullTxt } from '../dist/runtime/server/utils/llms-full.js';
+import { join as join$1, isAbsolute } from 'pathe';
 
 const logger = useLogger("nuxt-ai-ready");
 
@@ -19,7 +21,7 @@ function hookNuxtSeoProLicense() {
   const isBuild = !nuxt.options.dev && !nuxt.options._prepare;
   if (isBuild && !nuxt._isNuxtSeoProVerifying) {
     const license = nuxt.options.runtimeConfig.seoProKey || process.env.NUXT_SEO_PRO_KEY;
-    if (isTest) {
+    if (isTest || process.env.VITEST) {
       return;
     }
     if (!isCI && !license) {
@@ -65,118 +67,28 @@ function hookNuxtSeoProLicense() {
   }
 }
 
-async function resolveDatabaseAdapter(type, _opts) {
-  const connectors = {
-    d1: "db0/connectors/cloudflare-d1",
-    libsql: "db0/connectors/libsql/node"
-  };
-  if (type !== "sqlite" && connectors[type]) {
-    return connectors[type];
-  }
-  if (process.versions.bun) {
-    return "db0/connectors/bun-sqlite";
-  }
-  const nodeVersion = Number.parseInt(process.versions.node?.split(".")[0] || "0");
-  if (nodeVersion >= 22) {
-    return "db0/connectors/node-sqlite";
-  }
-  return "db0/connectors/better-sqlite3";
-}
-function refineDatabaseConfig(config, rootDir) {
-  const type = config.type || "sqlite";
-  if (type === "sqlite") {
-    const filename = config.filename || ".data/ai-ready/pages.db";
-    return {
-      type: "sqlite",
-      filename: isAbsolute(filename) ? filename : join(rootDir, filename)
-    };
-  }
-  if (type === "d1") {
-    return {
-      type: "d1",
-      bindingName: config.bindingName || "AI_READY_DB"
-    };
-  }
-  return {
-    type: "libsql",
-    url: config.url,
-    authToken: config.authToken
-  };
-}
-
-function normalizeLink(link) {
-  const parts = [];
-  parts.push(`- [${link.title}](${link.href})`);
-  if (link.description)
-    parts.push(`  ${link.description}`);
-  return parts.join("\n");
-}
-function normalizeSection(section) {
-  const parts = [];
-  parts.push(`## ${section.title}`);
-  parts.push("");
-  if (section.description) {
-    const descriptions = Array.isArray(section.description) ? section.description : [section.description];
-    parts.push(...descriptions);
-    parts.push("");
-  }
-  if (section.links?.length)
-    parts.push(...section.links.map(normalizeLink));
-  return parts.join("\n");
-}
-function normalizeLlmsTxtConfig(config) {
-  const parts = [];
-  if (config.sections?.length)
-    parts.push(...config.sections.map(normalizeSection));
-  if (config.notes) {
-    parts.push("## Notes");
-    parts.push("");
-    const notes = Array.isArray(config.notes) ? config.notes : [config.notes];
-    parts.push(...notes);
-  }
-  return parts.join("\n\n");
-}
-function createCrawlerState(pageDataPath, llmsFullTxtPath, siteInfo, llmsTxtConfig) {
+function createCrawlerState(dbPath, llmsFullTxtPath, siteInfo, llmsTxtConfig) {
   return {
     prerenderedRoutes: /* @__PURE__ */ new Set(),
     errorRoutes: /* @__PURE__ */ new Set(),
     totalProcessingTime: 0,
     initialized: false,
-    jsonlInitialized: false,
-    pageDataPath,
+    dbPath,
     llmsFullTxtPath,
     siteInfo,
     llmsTxtConfig
   };
 }
-function buildLlmsFullTxtHeader(siteInfo, llmsTxtConfig) {
-  const parts = [];
-  parts.push(`# ${siteInfo?.name || siteInfo?.url || "Site"}`);
-  if (siteInfo?.description)
-    parts.push(`
-> ${siteInfo.description}`);
-  if (siteInfo?.url)
-    parts.push(`
-Canonical Origin: ${siteInfo.url}`);
-  parts.push("");
-  if (llmsTxtConfig) {
-    const normalizedContent = normalizeLlmsTxtConfig(llmsTxtConfig);
-    if (normalizedContent) {
-      parts.push(normalizedContent);
-      parts.push("");
-    }
-  }
-  parts.push("## Pages\n\n");
-  return parts.join("\n");
-}
 async function initCrawler(state) {
   if (state.initialized)
     return;
-  if (state.pageDataPath) {
-    await mkdir(dirname(state.pageDataPath), { recursive: true });
-    await writeFile(state.pageDataPath, "", "utf-8");
-    state.jsonlInitialized = true;
-    logger.debug(`Crawler initialized with JSONL at ${state.pageDataPath}`);
+  if (state.dbPath) {
+    await mkdir(dirname(state.dbPath), { recursive: true });
+    const { default: betterSqlite3 } = await import('db0/connectors/better-sqlite3');
+    const connector = betterSqlite3({ path: state.dbPath });
+    state.db = createAdapter(connector);
+    await initSchema(state.db);
+    logger.debug(`Crawler initialized with SQLite at ${state.dbPath}`);
   }
   if (state.llmsFullTxtPath) {
     await mkdir(dirname(state.llmsFullTxtPath), { recursive: true });
@@ -189,34 +101,6 @@ async function initCrawler(state) {
 function flattenHeadings(headings) {
   return (headings || []).map((h) => Object.entries(h).map(([tag, text]) => `${tag}:${text}`).join("")).join("|");
 }
-function stripFrontmatter(markdown) {
-  return markdown.replace(/^---\n[\s\S]*?\n---\n*/, "");
-}
-function normalizeHeadings(markdown) {
-  return markdown.replace(/^(#{1,6})\s+(.+)$/gm, (_, hashes, text) => {
-    const level = hashes.length;
-    return `h${level}. ${text}`;
-  });
-}
-function formatPageForLlmsFullTxt(route, title, description, markdown, siteUrl) {
-  const canonicalUrl = siteUrl ? `${siteUrl.replace(/\/$/, "")}${route}` : route;
-  const heading = title && title !== route ? `### ${title}` : `### ${route}`;
-  let content = stripFrontmatter(markdown);
-  content = normalizeHeadings(content);
-  const parts = [heading, ""];
-  parts.push(`Source: ${canonicalUrl}`);
-  if (description)
-    parts.push(`Description: ${description}`);
-  parts.push("");
-  if (content.trim()) {
-    parts.push(content.trim());
-    parts.push("");
-  }
-  parts.push("---");
-  parts.push("");
-  return `${parts.join("\n")}
-`;
-}
 async function processMarkdownRoute(state, nuxt, route, parsed, lastmod, options) {
   const { markdown, title, description, headings, keywords, updatedAt: metaUpdatedAt } = parsed;
   let updatedAt = (lastmod instanceof Date ? lastmod.toISOString() : lastmod) || (/* @__PURE__ */ new Date()).toISOString();
@@ -226,18 +110,22 @@ async function processMarkdownRoute(state, nuxt, route, parsed, lastmod, options
       updatedAt = parsedDate.toISOString();
   }
   await nuxt.hooks.callHook("ai-ready:page:markdown", { route, markdown, title, description, headings });
-  if (state.jsonlInitialized && state.pageDataPath) {
-    const pageData = {
+  if (state.db) {
+    const contentHash = await computeContentHash(markdown);
+    await insertPage(state.db, {
       route,
       title,
       description,
+      markdown,
       headings: flattenHeadings(headings),
       keywords: keywords || [],
-      updatedAt,
-      markdown
-    };
-    await appendFile(state.pageDataPath, `${JSON.stringify(pageData)}
-`, "utf-8");
+      contentHash,
+      updatedAt
+    });
+  }
+  if (state.llmsFullTxtPath && !options?.skipLlmsFullTxt) {
+    const pageContent = formatPageForLlmsFullTxt(route, title, description, markdown, state.siteInfo?.url);
+    await appendFile(state.llmsFullTxtPath, pageContent, "utf-8");
   }
   state.prerenderedRoutes.add(route);
 }
@@ -275,7 +163,7 @@ async function crawlSitemapEntries(state, nuxt, nitro, entries) {
       logger.debug(`Skipping ${route}: Response is not JSON (likely HTML instead of markdown conversion)`, err);
       continue;
     }
-    await processMarkdownRoute(state, nuxt, route, parsed, lastmod);
+    await processMarkdownRoute(state, nuxt, route, parsed, lastmod, { skipLlmsFullTxt: true });
     crawled++;
   }
   logger.debug(`Sitemap crawl complete: ${crawled} crawled, ${skipped} skipped`);
@@ -314,92 +202,6 @@ function detectSitemapPrerender(sitemapName = "sitemap.xml") {
     usePrerenderHook: shouldHookIntoPrerender && !prerenderSitemap
   };
 }
-async function createDatabaseDump(entries, errorRoutes, dbConfig) {
-  const Database = (await import('better-sqlite3')).default;
-  const dbPath = dbConfig.filename || ".data/ai-ready/pages.db";
-  await mkdir(dirname(dbPath), { recursive: true });
-  const db = new Database(dbPath);
-  db.exec(`
-    CREATE TABLE IF NOT EXISTS pages (
-      id INTEGER PRIMARY KEY AUTOINCREMENT,
-      route TEXT UNIQUE NOT NULL,
-      route_key TEXT UNIQUE NOT NULL,
-      title TEXT NOT NULL DEFAULT '',
-      description TEXT NOT NULL DEFAULT '',
-      markdown TEXT NOT NULL DEFAULT '',
-      headings TEXT NOT NULL DEFAULT '[]',
-      keywords TEXT NOT NULL DEFAULT '[]',
-      updated_at TEXT NOT NULL,
-      indexed_at INTEGER NOT NULL,
-      is_error INTEGER NOT NULL DEFAULT 0
-    );
-    CREATE INDEX IF NOT EXISTS idx_pages_route ON pages(route);
-    CREATE INDEX IF NOT EXISTS idx_pages_is_error ON pages(is_error);
-    CREATE VIRTUAL TABLE IF NOT EXISTS pages_fts USING fts5(
-      route, title, description, markdown, headings, keywords,
-      content=pages, content_rowid=id
-    );
-    CREATE TRIGGER IF NOT EXISTS pages_ai AFTER INSERT ON pages BEGIN
-      INSERT INTO pages_fts(rowid, route, title, description, markdown, headings, keywords)
-      VALUES (new.id, new.route, new.title, new.description, new.markdown, new.headings, new.keywords);
-    END;
-    CREATE TRIGGER IF NOT EXISTS pages_ad AFTER DELETE ON pages BEGIN
-      INSERT INTO pages_fts(pages_fts, rowid, route, title, description, markdown, headings, keywords)
-      VALUES('delete', old.id, old.route, old.title, old.description, old.markdown, old.headings, old.keywords);
-    END;
-    CREATE TRIGGER IF NOT EXISTS pages_au AFTER UPDATE ON pages BEGIN
-      INSERT INTO pages_fts(pages_fts, rowid, route, title, description, markdown, headings, keywords)
-      VALUES('delete', old.id, old.route, old.title, old.description, old.markdown, old.headings, old.keywords);
-      INSERT INTO pages_fts(rowid, route, title, description, markdown, headings, keywords)
-      VALUES (new.id, new.route, new.title, new.description, new.markdown, new.headings, new.keywords);
-    END;
-  `);
-  const insertStmt = db.prepare(`
-    INSERT OR REPLACE INTO pages (route, route_key, title, description, markdown, headings, keywords, updated_at, indexed_at, is_error)
-    VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
-  `);
-  const normalizeRouteKey = (route) => route.replace(/^\//, "").replace(/\//g, ":") || "index";
-  const now = Date.now();
-  for (const entry of entries) {
-    insertStmt.run(
-      entry.route,
-      normalizeRouteKey(entry.route),
-      entry.title,
-      entry.description,
-      entry.markdown,
-      entry.headings,
-      JSON.stringify(entry.keywords),
-      entry.updatedAt,
-      now,
-      0
-    );
-  }
-  for (const route of errorRoutes) {
-    insertStmt.run(
-      route,
-      normalizeRouteKey(route),
-      "",
-      "",
-      "",
-      "[]",
-      "[]",
-      (/* @__PURE__ */ new Date()).toISOString(),
-      now,
-      1
-    );
-  }
-  const rows = db.prepare(`
-    SELECT route, route_key, title, description, markdown, headings, keywords, updated_at, indexed_at, is_error
-    FROM pages
-  `).all();
-  db.close();
-  const json = JSON.stringify(rows);
-  const encoder = new TextEncoder();
-  const stream = new Blob([encoder.encode(json)]).stream();
-  const compressed = stream.pipeThrough(new CompressionStream("gzip"));
-  const buffer = await new Response(compressed).arrayBuffer();
-  return Buffer.from(buffer).toString("base64");
-}
 async function prerenderRoute(nitro, route) {
   const start = Date.now();
   const encodedRoute = encodeURI(route);
@@ -409,7 +211,7 @@ async function prerenderRoute(nitro, route) {
     retry: nitro.options.prerender.retry,
     retryDelay: nitro.options.prerender.retryDelay
   });
-  const filePath = join$1(nitro.options.output.publicDir, route);
+  const filePath = join(nitro.options.output.publicDir, route);
   await mkdir(dirname(filePath), { recursive: true });
   const data = res._data;
   if (data === void 0)
@@ -423,12 +225,11 @@ async function prerenderRoute(nitro, route) {
   nitro._prerenderedRoutes.push(_route);
   return stat(filePath);
 }
-function setupPrerenderHandler(pageDataPath, siteInfo, llmsTxtConfig) {
+function setupPrerenderHandler(dbPath, siteInfo, llmsTxtConfig) {
   const nuxt = useNuxt();
-  const dbConfig = refineDatabaseConfig({}, nuxt.options.rootDir);
   nuxt.hooks.hook("nitro:init", async (nitro) => {
-    const llmsFullTxtPath = join$1(nitro.options.output.publicDir, "llms-full.txt");
-    const state = createCrawlerState(pageDataPath, llmsFullTxtPath, siteInfo, llmsTxtConfig);
+    const llmsFullTxtPath = join(nitro.options.output.publicDir, "llms-full.txt");
+    const state = createCrawlerState(dbPath, llmsFullTxtPath, siteInfo, llmsTxtConfig);
     let initPromise = null;
     nitro.hooks.hook("prerender:generate", async (route) => {
       if (route.error) {
@@ -447,76 +248,49 @@ function setupPrerenderHandler(pageDataPath, siteInfo, llmsTxtConfig) {
         initPromise = initCrawler(state);
       await initPromise;
       const parsed = JSON.parse(route.contents || "{}");
-      const { markdown, title, description, headings, keywords, updatedAt: metaUpdatedAt } = parsed;
-      let updatedAt = (/* @__PURE__ */ new Date()).toISOString();
-      if (metaUpdatedAt) {
-        const parsedDate = new Date(metaUpdatedAt);
-        if (!Number.isNaN(parsedDate.getTime()))
-          updatedAt = parsedDate.toISOString();
-      }
-      await nuxt.hooks.callHook("ai-ready:page:markdown", {
-        route: pageRoute,
-        markdown,
-        title,
-        description,
-        headings
-      });
-      if (state.jsonlInitialized && state.pageDataPath) {
-        const pageData = {
-          route: pageRoute,
-          title,
-          description,
-          headings: flattenHeadings(headings),
-          keywords: keywords || [],
-          updatedAt,
-          markdown
-        };
-        await appendFile(state.pageDataPath, `${JSON.stringify(pageData)}
-`, "utf-8");
-      }
-      if (state.llmsFullTxtPath) {
-        const pageContent = formatPageForLlmsFullTxt(pageRoute, title, description, markdown, state.siteInfo?.url);
-        await appendFile(state.llmsFullTxtPath, pageContent, "utf-8");
-      }
-      state.prerenderedRoutes.add(pageRoute);
-      route.contents = markdown;
+      await processMarkdownRoute(state, nuxt, pageRoute, parsed);
+      route.contents = parsed.markdown;
       state.totalProcessingTime += Date.now() - pageStartTime;
     });
     async function writeLlmsFiles() {
-      if (state.pageDataPath && state.errorRoutes.size > 0) {
+      if (state.db && state.errorRoutes.size > 0) {
         for (const route of state.errorRoutes) {
-          await appendFile(state.pageDataPath, `${JSON.stringify({ route, _error: true })}
-`, "utf-8");
+          await insertPage(state.db, {
+            route,
+            title: "",
+            description: "",
+            markdown: "",
+            headings: "",
+            keywords: [],
+            updatedAt: (/* @__PURE__ */ new Date()).toISOString(),
+            isError: true
+          });
         }
-        logger.debug(`Wrote ${state.errorRoutes.size} error routes to page data`);
+        logger.debug(`Wrote ${state.errorRoutes.size} error routes to database`);
       }
-      const publicDataDir = join$1(nitro.options.output.publicDir, "__ai-ready");
+      const publicDataDir = join(nitro.options.output.publicDir, "__ai-ready");
       await mkdir(publicDataDir, { recursive: true });
-      if (state.pageDataPath) {
-        const jsonlContent = await readFile(state.pageDataPath, "utf-8").catch(() => "");
-        if (jsonlContent) {
-          const entries = jsonlContent.trim().split("\n").filter(Boolean).map((line) => JSON.parse(line));
-          const pages = entries.filter((e) => !e._error);
-          const errorRoutesList = entries.filter((e) => e._error).map((e) => e.route);
-          const jsonContent = JSON.stringify({
-            pages: pages.map((p) => ({
-              route: p.route,
-              title: p.title,
-              description: p.description,
-              headings: p.headings,
-              keywords: p.keywords || [],
-              updatedAt: p.updatedAt
-            })),
-            errorRoutes: errorRoutesList
-          });
-          const publicJsonPath = join$1(publicDataDir, "pages.json");
-          await writeFile(publicJsonPath, jsonContent, "utf-8");
-          logger.debug(`Wrote ${pages.length} pages to __ai-ready/pages.json`);
-          const dumpData = await createDatabaseDump(pages, errorRoutesList, dbConfig);
-          const dumpPath = join$1(publicDataDir, "pages.dump");
-          await writeFile(dumpPath, dumpData, "utf-8");
-          logger.debug(`Created database dump at __ai-ready/pages.dump (${(dumpData.length / 1024).toFixed(1)}kb compressed)`);
-        }
+      if (state.db) {
+        const pages = await queryAllPages(state.db);
+        const errorRoutesList = (await queryAllPages(state.db, { includeErrors: true })).filter((p) => p.isError).map((p) => p.route);
+        const jsonContent = JSON.stringify({
+          pages: pages.map((p) => ({
+            route: p.route,
+            title: p.title,
+            description: p.description,
+            headings: p.headings,
+            keywords: p.keywords || [],
+            updatedAt: p.updatedAt
+          })),
+          errorRoutes: errorRoutesList
+        });
+        const publicJsonPath = join(publicDataDir, "pages.json");
+        await writeFile(publicJsonPath, jsonContent, "utf-8");
+        logger.debug(`Wrote ${pages.length} pages to __ai-ready/pages.json`);
+        const dumpData = await exportDbDump(state.db);
+        const dumpPath = join(publicDataDir, "pages.dump");
+        await writeFile(dumpPath, dumpData, "utf-8");
+        logger.debug(`Created database dump at __ai-ready/pages.dump (${(dumpData.length / 1024).toFixed(1)}kb compressed)`);
       }
       const llmsStats = await prerenderRoute(nitro, "/llms.txt");
       const llmsFullStats = await stat(state.llmsFullTxtPath);
@@ -555,6 +329,90 @@ function setupPrerenderHandler(pageDataPath, siteInfo, llmsTxtConfig) {
   });
 }
 
+function registerTypeTemplates(_ctx) {
+  addTypeTemplate({
+    filename: "types/nuxt-ai-ready.d.ts",
+    getContents: () => `// Generated by nuxt-ai-ready
+import type { MarkdownContext, PageIndexedContext } from 'nuxt-ai-ready'
+import type { HTMLToMarkdownOptions } from 'mdream'
+
+declare module 'nitropack/types' {
+  interface NitroRuntimeHooks {
+    'ai-ready:markdown': (context: MarkdownContext) => void | Promise<void>
+    'ai-ready:mdreamConfig': (config: HTMLToMarkdownOptions) => void | Promise<void>
+    'ai-ready:page:indexed': (context: PageIndexedContext) => void | Promise<void>
+  }
+}
+
+declare module '#ai-ready-virtual/read-page-data.mjs' {
+  export function readPageDataFromFilesystem(): Promise<{
+    pages: Array<{
+      route: string
+      title: string
+      description: string
+      headings: string
+      keywords: string[]
+      updatedAt: string
+      markdown: string
+    }>
+    errorRoutes: string[]
+  }>
+}
+
+declare module '#ai-ready-virtual/page-data.mjs' {
+  export const pages: never[]
+}
+
+declare module '#ai-ready/adapter' {
+  import type { Connector } from 'db0'
+  const connector: (config: unknown) => Connector
+  export default connector
+}
+
+export {}
+`
+  });
+}
+
+async function resolveDatabaseAdapter(type) {
+  const connectors = {
+    d1: "db0/connectors/cloudflare-d1",
+    libsql: "db0/connectors/libsql/node"
+  };
+  if (type !== "sqlite" && connectors[type]) {
+    return connectors[type];
+  }
+  if (process.versions.bun) {
+    return "db0/connectors/bun-sqlite";
+  }
+  const nodeVersion = Number.parseInt(process.versions.node?.split(".")[0] || "0");
+  if (nodeVersion >= 22) {
+    return "db0/connectors/node-sqlite";
+  }
+  return "db0/connectors/better-sqlite3";
+}
+function refineDatabaseConfig(config, rootDir) {
+  const type = config.type || "sqlite";
+  if (type === "sqlite") {
+    const filename = config.filename || ".data/ai-ready/pages.db";
+    return {
+      type: "sqlite",
+      filename: isAbsolute(filename) ? filename : join$1(rootDir, filename)
+    };
+  }
+  if (type === "d1") {
+    return {
+      type: "d1",
+      bindingName: config.bindingName || "AI_READY_DB"
+    };
+  }
+  return {
+    type: "libsql",
+    url: config.url,
+    authToken: config.authToken
+  };
+}
+
 const module$1 = defineNuxtModule({
   meta: {
     name: "nuxt-ai-ready",
@@ -606,7 +464,6 @@ const module$1 = defineNuxtModule({
     hookNuxtSeoProLicense();
     nuxt.options.nitro.alias = nuxt.options.nitro.alias || {};
     nuxt.options.alias["#ai-ready"] = resolve("./runtime");
-    ({ resolver: createResolver(import.meta.url) });
     const dbType = config.database?.type || "sqlite";
     const adapterPath = await resolveDatabaseAdapter(dbType);
     nuxt.options.alias["#ai-ready/adapter"] = adapterPath;
@@ -628,48 +485,7 @@ const module$1 = defineNuxtModule({
         contentSignal: [`ai-train=${config.contentSignal.aiTrain ? "yes" : "no"}`, `search=${config.contentSignal.search ? "yes" : "no"}`, `ai-input=${config.contentSignal.aiInput ? "yes" : "no"}`]
       });
     }
-    addTypeTemplate({
-      filename: "types/nuxt-ai-ready.d.ts",
-      getContents: () => `// Generated by nuxt-ai-ready
-import type { MarkdownContext, PageIndexedContext } from 'nuxt-ai-ready'
-import type { HTMLToMarkdownOptions } from 'mdream'
-
-declare module 'nitropack/types' {
-  interface NitroRuntimeHooks {
-    'ai-ready:markdown': (context: MarkdownContext) => void | Promise<void>
-    'ai-ready:mdreamConfig': (config: HTMLToMarkdownOptions) => void | Promise<void>
-    'ai-ready:page:indexed': (context: PageIndexedContext) => void | Promise<void>
-  }
-}
-
-declare module '#ai-ready-virtual/read-page-data.mjs' {
-  export function readPageDataFromFilesystem(): Promise<{
-    pages: Array<{
-      route: string
-      title: string
-      description: string
-      headings: string
-      keywords: string[]
-      updatedAt: string
-      markdown: string
-    }>
-    errorRoutes: string[]
-  }>
-}
-
-declare module '#ai-ready-virtual/page-data.mjs' {
-  export const pages: never[]
-}
-
-declare module '#ai-ready/adapter' {
-  import type { Connector } from 'db0'
-  const connector: (config: unknown) => Connector
-  export default connector
-}
-
-export {}
-`
-    }, { nitro: true });
+    registerTypeTemplates();
     const defaultLlmsTxtSections = [];
     const llmsFullRoute = withSiteUrl("llms-full.txt");
     defaultLlmsTxtSections.push({
@@ -720,24 +536,55 @@ export {}
     await nuxt.callHook("ai-ready:llms-txt", llmsTxtPayload);
     mergedLlmsTxt.sections = llmsTxtPayload.sections;
     mergedLlmsTxt.notes = llmsTxtPayload.notes.length > 0 ? llmsTxtPayload.notes : void 0;
-    const prerenderCacheDir = join$1(nuxt.options.rootDir, "node_modules/.cache/nuxt-seo/ai-ready/routes");
-    const pageDataPath = join$1(nuxt.options.buildDir, ".data/ai-ready/page-data.jsonl");
+    const prerenderCacheDir = join(nuxt.options.rootDir, "node_modules/.cache/nuxt-seo/ai-ready/routes");
+    const buildDbPath = join(nuxt.options.buildDir, ".data/ai-ready/build.db");
     nuxt.hooks.hook("nitro:config", (nitroConfig) => {
       nitroConfig.experimental = nitroConfig.experimental || {};
       nitroConfig.experimental.asyncContext = true;
+      const runtimeSyncEnabled2 = config.runtimeSync?.enabled ?? false;
+      const cron = config.runtimeSync?.cron;
+      if (runtimeSyncEnabled2 && cron) {
+        nitroConfig.tasks = nitroConfig.tasks || {};
+        nitroConfig.tasks["ai-ready:index"] = {
+          handler: resolve("./runtime/server/tasks/ai-ready-index")
+        };
+        nitroConfig.scheduledTasks = nitroConfig.scheduledTasks || {};
+        nitroConfig.scheduledTasks[cron] = nitroConfig.scheduledTasks[cron] || [];
+        nitroConfig.scheduledTasks[cron].push("ai-ready:index");
+      }
       nitroConfig.virtual = nitroConfig.virtual || {};
       nitroConfig.virtual["#ai-ready-virtual/read-page-data.mjs"] = `
-import { readFile } from 'node:fs/promises'
-
 export async function readPageDataFromFilesystem() {
   if (!import.meta.prerender) {
     return { pages: [], errorRoutes: [] }
   }
-  const data = await readFile(${JSON.stringify(pageDataPath)}, 'utf-8').catch(() => null)
-  if (!data) return { pages: [], errorRoutes: [] }
-  const entries = data.trim().split('\\n').filter(Boolean).map(line => JSON.parse(line))
-  const pages = entries.filter(e => !e._error)
-  const errorRoutes = entries.filter(e => e._error).map(e => e.route)
+
+  const dbPath = ${JSON.stringify(buildDbPath)}
+
+  // Check if database file exists
+  const { existsSync } = await import('node:fs')
+  if (!existsSync(dbPath)) {
+    return { pages: [], errorRoutes: [] }
+  }
+
+  // Use better-sqlite3 to read pages
+  const Database = (await import('better-sqlite3')).default
+  const db = new Database(dbPath, { readonly: true })
+
+  const rows = db.prepare('SELECT route, title, description, markdown, headings, keywords, updated_at, is_error FROM ai_ready_pages').all()
+  db.close()
+
+  const pages = rows.filter(r => !r.is_error).map(r => ({
+    route: r.route,
+    title: r.title,
+    description: r.description,
+    markdown: r.markdown,
+    headings: r.headings,
+    keywords: JSON.parse(r.keywords || '[]'),
+    updatedAt: r.updated_at,
+  }))
+  const errorRoutes = rows.filter(r => r.is_error).map(r => r.route)
+
   return { pages, errorRoutes }
 }
 `;
@@ -745,6 +592,9 @@ export async function readPageDataFromFilesystem() {
 export const errorRoutes = []`;
     });
     const database = refineDatabaseConfig(config.database || {}, nuxt.options.rootDir);
+    const runtimeSyncEnabled = config.runtimeSync?.enabled ?? false;
+    const indexNowKey = config.indexNow?.key || process.env.NUXT_AI_READY_INDEXNOW_KEY;
+    const indexNowEnabled = !!(config.indexNow?.enabled !== false && indexNowKey);
     nuxt.options.runtimeConfig["nuxt-ai-ready"] = {
       version: version || "0.0.0",
       debug: config.debug || false,
@@ -756,12 +606,24 @@ export const errorRoutes = []`;
       llmsTxt: mergedLlmsTxt,
       cacheMaxAgeSeconds: config.cacheMaxAgeSeconds ?? 600,
       prerenderCacheDir,
-      ttl: config.ttl ?? 0,
-      database
+      database,
+      runtimeSync: {
+        enabled: runtimeSyncEnabled,
+        ttl: config.runtimeSync?.ttl ?? 3600,
+        batchSize: config.runtimeSync?.batchSize ?? 20,
+        secret: config.runtimeSync?.secret,
+        pruneTtl: config.runtimeSync?.pruneTtl ?? 0
+      },
+      indexNow: indexNowEnabled ? {
+        enabled: true,
+        key: indexNowKey,
+        host: config.indexNow?.host || "api.indexnow.org"
+      } : void 0
     };
     nuxt.options.nitro.plugins = nuxt.options.nitro.plugins || [];
     nuxt.options.nitro.plugins.push(resolve("./runtime/server/plugins/db-restore"));
-    nuxt.options.nitro.plugins.push(resolve("./runtime/server/plugins/page-indexer"));
+    if (runtimeSyncEnabled)
+      nuxt.options.nitro.plugins.push(resolve("./runtime/server/plugins/sitemap-seeder"));
     addServerHandler({
       middleware: true,
       handler: resolve("./runtime/server/middleware/markdown.prerender")
@@ -773,7 +635,7 @@ export const errorRoutes = []`;
     if (nuxt.options.build) {
       addPlugin({
         mode: "server",
-        src: resolve("./runtime/nuxt/plugins/md-hints.prerender")
+        src: resolve("./runtime/app/plugins/md-hints.prerender")
       });
     }
     addServerHandler({ route: "/llms.txt", handler: resolve("./runtime/server/routes/llms.txt.get") });
@@ -781,6 +643,18 @@ export const errorRoutes = []`;
     if (config.debug) {
       addServerHandler({ route: "/__ai-ready-debug", handler: resolve("./runtime/server/routes/__ai-ready-debug.get") });
     }
+    if (runtimeSyncEnabled) {
+      addServerHandler({ route: "/__ai-ready/status", handler: resolve("./runtime/server/routes/__ai-ready/status.get") });
+      addServerHandler({ route: "/__ai-ready/poll", method: "post", handler: resolve("./runtime/server/routes/__ai-ready/poll.post") });
+      addServerHandler({ route: "/__ai-ready/prune", method: "post", handler: resolve("./runtime/server/routes/__ai-ready/prune.post") });
+    }
+    if (indexNowEnabled && indexNowKey) {
+      addServerHandler({ route: `/${indexNowKey}.txt`, handler: resolve("./runtime/server/routes/indexnow-key.get") });
+      addServerHandler({ route: "/__ai-ready/indexnow", method: "post", handler: resolve("./runtime/server/routes/__ai-ready/indexnow.post") });
+      if (!runtimeSyncEnabled) {
+        addServerHandler({ route: "/__ai-ready/status", handler: resolve("./runtime/server/routes/__ai-ready/status.get") });
+      }
+    }
     const isStatic = nuxt.options.nitro.static || nuxt.options._generate || false;
     const hasPrerenderedRoutes = nuxt.options.nitro.prerender?.routes?.length;
     const isSPA = nuxt.options.ssr === false;
@@ -794,7 +668,7 @@ export const errorRoutes = []`;
     }
     if (isStatic || hasPrerenderedRoutes) {
       const siteConfig = useSiteConfig();
-      setupPrerenderHandler(pageDataPath, {
+      setupPrerenderHandler(buildDbPath, {
         name: siteConfig.name,
         url: siteConfig.url,
         description: siteConfig.description
@@ -805,7 +679,7 @@ export const errorRoutes = []`;
     nuxt.options.nitro.routeRules["/llms-full.txt"] = { headers: { "Content-Type": "text/plain; charset=utf-8" } };
     nuxt.hooks.hook("nitro:build:before", (nitro) => {
       nitro.hooks.hook("compiled", async () => {
-        const headersPath = join$1(nitro.options.output.publicDir, "_headers");
+        const headersPath = join(nitro.options.output.publicDir, "_headers");
         const exists = await access(headersPath).then(() => true).catch(() => false);
         if (exists) {
           await appendFile(headersPath, `