nuxt-ai-ready 0.5.2 → 0.6.0

package/dist/runtime/server/db/queries.js

@@ -0,0 +1,79 @@
+function normalizeRouteKey(route) {
+  return route.replace(/^\//, "").replace(/\//g, ":") || "index";
+}
+function rowToEntry(row) {
+  return {
+    route: row.route,
+    title: row.title,
+    description: row.description,
+    headings: row.headings,
+    keywords: JSON.parse(row.keywords || "[]"),
+    updatedAt: row.updated_at
+  };
+}
+export async function getAllPages(db) {
+  const rows = await db.all("SELECT * FROM pages WHERE is_error = 0");
+  return rows.map(rowToEntry);
+}
+export async function getPage(db, route) {
+  const row = await db.first("SELECT * FROM pages WHERE route = ?", [route]);
+  return row ? rowToEntry(row) : void 0;
+}
+export async function getPageWithMarkdown(db, route) {
+  const row = await db.first("SELECT * FROM pages WHERE route = ?", [route]);
+  return row ? { ...rowToEntry(row), markdown: row.markdown } : void 0;
+}
+export async function searchPages(db, query, opts = {}) {
+  const { limit = 10 } = opts;
+  const sanitized = query.replace(/[*:^"()]/g, " ").trim();
+  if (!sanitized) return [];
+  const terms = sanitized.split(/\s+/).map((t) => `${t}*`).join(" ");
+  return db.all(`
+    SELECT p.route, p.title, p.description, bm25(pages_fts, 5.0, 3.0, 1.0, 0.5, 2.0, 2.0) as score
+    FROM pages_fts
+    JOIN pages p ON pages_fts.rowid = p.id
+    WHERE pages_fts MATCH ?
+    ORDER BY score
+    LIMIT ?
+  `, [terms, limit]);
+}
+export async function upsertPage(db, page) {
+  const routeKey = normalizeRouteKey(page.route);
+  const keywordsJson = JSON.stringify(page.keywords);
+  const indexedAt = Date.now();
+  await db.exec(`
+    INSERT INTO pages (route, route_key, title, description, markdown, headings, keywords, updated_at, indexed_at, is_error)
+    VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+    ON CONFLICT(route) DO UPDATE SET
+      title = excluded.title,
+      description = excluded.description,
+      markdown = excluded.markdown,
+      headings = excluded.headings,
+      keywords = excluded.keywords,
+      updated_at = excluded.updated_at,
+      indexed_at = excluded.indexed_at,
+      is_error = excluded.is_error
+  `, [page.route, routeKey, page.title, page.description, page.markdown, page.headings, keywordsJson, page.updatedAt, indexedAt, page.isError ? 1 : 0]);
+}
+export async function getErrorRoutes(db) {
+  const rows = await db.all("SELECT route FROM pages WHERE is_error = 1");
+  return rows.map((r) => r.route);
+}
+export async function isPageFresh(db, route, ttlSeconds) {
+  if (ttlSeconds <= 0) return false;
+  const row = await db.first("SELECT indexed_at FROM pages WHERE route = ?", [route]);
+  if (!row) return false;
+  const age = (Date.now() - row.indexed_at) / 1e3;
+  return age < ttlSeconds;
+}
+export async function deletePage(db, route) {
+  await db.exec("DELETE FROM pages WHERE route = ?", [route]);
+}
+export async function getPageCount(db) {
+  const row = await db.first("SELECT COUNT(*) as count FROM pages WHERE is_error = 0");
+  return row?.count || 0;
+}
+export async function hasPages(db) {
+  const count = await getPageCount(db);
+  return count > 0;
+}

package/dist/runtime/server/db/schema.d.ts

@@ -0,0 +1,8 @@
+export declare const SCHEMA_VERSION = "v1.0.0";
+export declare const createTablesSQL = "\nCREATE TABLE IF NOT EXISTS pages (\n  id INTEGER PRIMARY KEY AUTOINCREMENT,\n  route TEXT UNIQUE NOT NULL,\n  route_key TEXT UNIQUE NOT NULL,\n  title TEXT NOT NULL DEFAULT '',\n  description TEXT NOT NULL DEFAULT '',\n  markdown TEXT NOT NULL DEFAULT '',\n  headings TEXT NOT NULL DEFAULT '[]',\n  keywords TEXT NOT NULL DEFAULT '[]',\n  updated_at TEXT NOT NULL,\n  indexed_at INTEGER NOT NULL,\n  is_error INTEGER NOT NULL DEFAULT 0\n);\n\nCREATE INDEX IF NOT EXISTS idx_pages_route ON pages(route);\nCREATE INDEX IF NOT EXISTS idx_pages_is_error ON pages(is_error);\n\nCREATE VIRTUAL TABLE IF NOT EXISTS pages_fts USING fts5(\n  route, title, description, markdown, headings, keywords,\n  content=pages, content_rowid=id\n);\n\nCREATE TRIGGER IF NOT EXISTS pages_ai AFTER INSERT ON pages BEGIN\n  INSERT INTO pages_fts(rowid, route, title, description, markdown, headings, keywords)\n  VALUES (new.id, new.route, new.title, new.description, new.markdown, new.headings, new.keywords);\nEND;\n\nCREATE TRIGGER IF NOT EXISTS pages_ad AFTER DELETE ON pages BEGIN\n  INSERT INTO pages_fts(pages_fts, rowid, route, title, description, markdown, headings, keywords)\n  VALUES('delete', old.id, old.route, old.title, old.description, old.markdown, old.headings, old.keywords);\nEND;\n\nCREATE TRIGGER IF NOT EXISTS pages_au AFTER UPDATE ON pages BEGIN\n  INSERT INTO pages_fts(pages_fts, rowid, route, title, description, markdown, headings, keywords)\n  VALUES('delete', old.id, old.route, old.title, old.description, old.markdown, old.headings, old.keywords);\n  INSERT INTO pages_fts(rowid, route, title, description, markdown, headings, keywords)\n  VALUES (new.id, new.route, new.title, new.description, new.markdown, new.headings, new.keywords);\nEND;\n\nCREATE TABLE IF NOT EXISTS _ai_ready_info (\n  id TEXT PRIMARY KEY,\n  version TEXT,\n  checksum TEXT,\n  ready INTEGER DEFAULT 0\n);\n";
+export interface DatabaseAdapter {
+    all: <T>(sql: string, params?: unknown[]) => Promise<T[]>;
+    first: <T>(sql: string, params?: unknown[]) => Promise<T | undefined>;
+    exec: (sql: string, params?: unknown[]) => Promise<void>;
+}
+export declare function initSchema(db: DatabaseAdapter): Promise<void>;

package/dist/runtime/server/db/schema.js

@@ -0,0 +1,121 @@
+export const SCHEMA_VERSION = "v1.0.0";
+const DROP_TABLES_SQL = [
+  "DROP TABLE IF EXISTS pages_fts",
+  "DROP TABLE IF EXISTS pages",
+  "DROP TABLE IF EXISTS _ai_ready_info"
+];
+export const createTablesSQL = `
+CREATE TABLE IF NOT EXISTS pages (
+  id INTEGER PRIMARY KEY AUTOINCREMENT,
+  route TEXT UNIQUE NOT NULL,
+  route_key TEXT UNIQUE NOT NULL,
+  title TEXT NOT NULL DEFAULT '',
+  description TEXT NOT NULL DEFAULT '',
+  markdown TEXT NOT NULL DEFAULT '',
+  headings TEXT NOT NULL DEFAULT '[]',
+  keywords TEXT NOT NULL DEFAULT '[]',
+  updated_at TEXT NOT NULL,
+  indexed_at INTEGER NOT NULL,
+  is_error INTEGER NOT NULL DEFAULT 0
+);
+
+CREATE INDEX IF NOT EXISTS idx_pages_route ON pages(route);
+CREATE INDEX IF NOT EXISTS idx_pages_is_error ON pages(is_error);
+
+CREATE VIRTUAL TABLE IF NOT EXISTS pages_fts USING fts5(
+  route, title, description, markdown, headings, keywords,
+  content=pages, content_rowid=id
+);
+
+CREATE TRIGGER IF NOT EXISTS pages_ai AFTER INSERT ON pages BEGIN
+  INSERT INTO pages_fts(rowid, route, title, description, markdown, headings, keywords)
+  VALUES (new.id, new.route, new.title, new.description, new.markdown, new.headings, new.keywords);
+END;
+
+CREATE TRIGGER IF NOT EXISTS pages_ad AFTER DELETE ON pages BEGIN
+  INSERT INTO pages_fts(pages_fts, rowid, route, title, description, markdown, headings, keywords)
+  VALUES('delete', old.id, old.route, old.title, old.description, old.markdown, old.headings, old.keywords);
+END;
+
+CREATE TRIGGER IF NOT EXISTS pages_au AFTER UPDATE ON pages BEGIN
+  INSERT INTO pages_fts(pages_fts, rowid, route, title, description, markdown, headings, keywords)
+  VALUES('delete', old.id, old.route, old.title, old.description, old.markdown, old.headings, old.keywords);
+  INSERT INTO pages_fts(rowid, route, title, description, markdown, headings, keywords)
+  VALUES (new.id, new.route, new.title, new.description, new.markdown, new.headings, new.keywords);
+END;
+
+CREATE TABLE IF NOT EXISTS _ai_ready_info (
+  id TEXT PRIMARY KEY,
+  version TEXT,
+  checksum TEXT,
+  ready INTEGER DEFAULT 0
+);
+`;
+export async function initSchema(db) {
+  const needsRebuild = await checkSchemaVersion(db);
+  if (needsRebuild) {
+    for (const sql of DROP_TABLES_SQL) {
+      await db.exec(sql);
+    }
+  }
+  const statements = [
+    // Main table
+    `CREATE TABLE IF NOT EXISTS pages (
+      id INTEGER PRIMARY KEY AUTOINCREMENT,
+      route TEXT UNIQUE NOT NULL,
+      route_key TEXT UNIQUE NOT NULL,
+      title TEXT NOT NULL DEFAULT '',
+      description TEXT NOT NULL DEFAULT '',
+      markdown TEXT NOT NULL DEFAULT '',
+      headings TEXT NOT NULL DEFAULT '[]',
+      keywords TEXT NOT NULL DEFAULT '[]',
+      updated_at TEXT NOT NULL,
+      indexed_at INTEGER NOT NULL,
+      is_error INTEGER NOT NULL DEFAULT 0
+    )`,
+    // Indexes
+    `CREATE INDEX IF NOT EXISTS idx_pages_route ON pages(route)`,
+    `CREATE INDEX IF NOT EXISTS idx_pages_is_error ON pages(is_error)`,
+    // FTS5 virtual table
+    `CREATE VIRTUAL TABLE IF NOT EXISTS pages_fts USING fts5(
+      route, title, description, markdown, headings, keywords,
+      content=pages, content_rowid=id
+    )`,
+    // Triggers for FTS sync
+    `CREATE TRIGGER IF NOT EXISTS pages_ai AFTER INSERT ON pages BEGIN
+      INSERT INTO pages_fts(rowid, route, title, description, markdown, headings, keywords)
+      VALUES (new.id, new.route, new.title, new.description, new.markdown, new.headings, new.keywords);
+    END`,
+    `CREATE TRIGGER IF NOT EXISTS pages_ad AFTER DELETE ON pages BEGIN
+      INSERT INTO pages_fts(pages_fts, rowid, route, title, description, markdown, headings, keywords)
+      VALUES('delete', old.id, old.route, old.title, old.description, old.markdown, old.headings, old.keywords);
+    END`,
+    `CREATE TRIGGER IF NOT EXISTS pages_au AFTER UPDATE ON pages BEGIN
+      INSERT INTO pages_fts(pages_fts, rowid, route, title, description, markdown, headings, keywords)
+      VALUES('delete', old.id, old.route, old.title, old.description, old.markdown, old.headings, old.keywords);
+      INSERT INTO pages_fts(rowid, route, title, description, markdown, headings, keywords)
+      VALUES (new.id, new.route, new.title, new.description, new.markdown, new.headings, new.keywords);
+    END`,
+    // Info table
+    `CREATE TABLE IF NOT EXISTS _ai_ready_info (
+      id TEXT PRIMARY KEY,
+      version TEXT,
+      checksum TEXT,
+      ready INTEGER DEFAULT 0
+    )`
+  ];
+  for (const statement of statements) {
+    await db.exec(statement);
+  }
+  await db.exec(
+    "INSERT OR REPLACE INTO _ai_ready_info (id, version) VALUES (?, ?)",
+    ["schema", SCHEMA_VERSION]
+  );
+}
+async function checkSchemaVersion(db) {
+  const info = await db.first(
+    "SELECT version FROM _ai_ready_info WHERE id = ?",
+    ["schema"]
+  ).catch(() => null);
+  return !info || info.version !== SCHEMA_VERSION;
+}

package/dist/runtime/server/mcp/tools/search-pages.js

@@ -0,0 +1,19 @@
+import { z } from "zod";
+import { useDatabase } from "../../db/index.js";
+import { searchPages } from "../../db/queries.js";
+const inputSchema = {
+  query: z.string().describe("Search query"),
+  limit: z.number().optional().default(10).describe("Max results")
+};
+const tool = {
+  name: "search_pages",
+  description: "Search pages by title, description, route, headings, keywords, and content using full-text search.",
+  inputSchema,
+  cache: "5m",
+  async handler({ query, limit }) {
+    const db = await useDatabase();
+    const results = await searchPages(db, query, { limit });
+    return { content: [{ type: "text", text: JSON.stringify(results) }] };
+  }
+};
+export default tool;

package/dist/runtime/server/middleware/markdown.prerender.js

@@ -1,6 +1,7 @@
 import { withSiteUrl } from "#site-config/server/composables/utils";
 import { createError, defineEventHandler } from "h3";
 import { useRuntimeConfig } from "nitropack/runtime";
+import { extractKeywords, stripMarkdown } from "../utils/keywords.js";
 import { convertHtmlToMarkdownMeta, getMarkdownRenderInfo } from "../utils.js";
 export default defineEventHandler(async (event) => {
   if (!import.meta.prerender) {
@@ -26,10 +27,18 @@ export default defineEventHandler(async (event) => {
       message: `Page rendered as error: ${path}`
     });
   }
-  const result = convertHtmlToMarkdownMeta(
+  const result = await convertHtmlToMarkdownMeta(
     html,
     withSiteUrl(event, path),
     runtimeConfig.mdreamOptions
   );
-  return JSON.stringify(result);
+  const keywords = extractKeywords(stripMarkdown(result.markdown), result.metaKeywords);
+  return JSON.stringify({
+    markdown: result.markdown,
+    title: result.title,
+    description: result.description,
+    headings: result.headings,
+    keywords,
+    ...result.updatedAt && { updatedAt: result.updatedAt }
+  });
 });

package/dist/runtime/server/plugins/db-restore.d.ts

@@ -0,0 +1,2 @@
+declare const _default: import("nitropack/types").NitroAppPlugin;
+export default _default;

package/dist/runtime/server/plugins/db-restore.js

@@ -0,0 +1,24 @@
+import { defineNitroPlugin } from "nitropack/runtime";
+import { useDatabase } from "../db/index.js";
+import { hasPages } from "../db/queries.js";
+import { decompressDump, importDump } from "../db/dump.js";
+import { logger } from "../logger.js";
+export default defineNitroPlugin(async () => {
+  if (import.meta.prerender) return;
+  if (import.meta.dev) return;
+  const db = await useDatabase();
+  if (await hasPages(db)) {
+    logger.debug("[db-restore] Database already has data, skipping restore");
+    return;
+  }
+  const dumpData = await globalThis.$fetch("/__ai-ready/pages.dump", {
+    responseType: "text"
+  }).catch(() => null);
+  if (!dumpData) {
+    logger.debug("[db-restore] No dump found, starting with empty database");
+    return;
+  }
+  const rows = await decompressDump(dumpData);
+  await importDump(db, rows);
+  logger.info(`[db-restore] Restored ${rows.length} pages from dump`);
+});

package/dist/runtime/server/plugins/page-indexer.d.ts

@@ -0,0 +1,2 @@
+declare const _default: import("nitropack/types").NitroAppPlugin;
+export default _default;

package/dist/runtime/server/plugins/page-indexer.js

@@ -0,0 +1,68 @@
+import { defineNitroPlugin, useRuntimeConfig } from "nitropack/runtime";
+import { useDatabase } from "../db/index.js";
+import { getPage, isPageFresh, upsertPage } from "../db/queries.js";
+import { logger } from "../logger.js";
+import { convertHtmlToMarkdownMeta } from "../utils.js";
+import { extractKeywords, stripMarkdown } from "../utils/keywords.js";
+export default defineNitroPlugin((nitro) => {
+  const config = useRuntimeConfig()["nuxt-ai-ready"];
+  const ttl = config.ttl ?? 0;
+  nitro.hooks.hook("afterResponse", (event, response) => {
+    const body = response?.body;
+    const path = event.path;
+    if (path.startsWith("/api") || path.startsWith("/_") || path.startsWith("/@") || path.endsWith(".md") || path.includes(".")) {
+      return;
+    }
+    const contentType = event.node.res.getHeader("content-type");
+    if (!contentType?.toString().includes("text/html"))
+      return;
+    const status = event.node.res.statusCode;
+    if (status >= 400)
+      return;
+    event.waitUntil((async () => {
+      const html = typeof body === "string" ? body : null;
+      if (!html) {
+        logger.debug(`[runtime-indexing] Skipping ${path} - no HTML body`);
+        return;
+      }
+      const db = await useDatabase(event);
+      if (ttl > 0 && await isPageFresh(db, path, ttl)) {
+        logger.debug(`[runtime-indexing] Skipping ${path} - still fresh`);
+        return;
+      }
+      const existing = await getPage(db, path);
+      const isUpdate = !!existing;
+      const isError = html.includes("__NUXT_ERROR__") || html.includes("nuxt-error-page");
+      const result = await convertHtmlToMarkdownMeta(html, path, config.mdreamOptions);
+      const updatedAt = result.updatedAt || (/* @__PURE__ */ new Date()).toISOString();
+      const headings = JSON.stringify(result.headings);
+      const keywords = extractKeywords(stripMarkdown(result.markdown), result.metaKeywords);
+      await upsertPage(db, {
+        route: path,
+        title: result.title,
+        description: result.description,
+        markdown: result.markdown,
+        headings,
+        keywords,
+        updatedAt,
+        isError
+      });
+      logger.debug(`[runtime-indexing] Indexed: ${path} "${result.title}"`);
+      if (!isError) {
+        const hookContext = {
+          route: path,
+          title: result.title,
+          description: result.description,
+          headings,
+          keywords,
+          markdown: result.markdown,
+          updatedAt,
+          isUpdate
+        };
+        await nitro.hooks.callHook("ai-ready:page:indexed", hookContext);
+      }
+    })().catch((err) => {
+      logger.error(`[runtime-indexing] Failed to index ${path}:`, err);
+    }));
+  });
+});

package/dist/runtime/server/utils/indexPage.d.ts

@@ -0,0 +1,38 @@
+import type { H3Event } from 'h3';
+export interface IndexPageOptions {
+    /** Skip if page was indexed within TTL (uses config ttl if not specified) */
+    ttl?: number;
+    /** Force re-index even if fresh */
+    force?: boolean;
+    /** Skip calling the ai-ready:page:indexed hook */
+    skipHook?: boolean;
+}
+export interface IndexPageResult {
+    success: boolean;
+    /** True if page was already fresh and skipped */
+    skipped?: boolean;
+    /** True if this was an update vs new index */
+    isUpdate?: boolean;
+    /** Page data if successful */
+    data?: {
+        route: string;
+        title: string;
+        description: string;
+        headings: string;
+        keywords: string[];
+        markdown: string;
+        updatedAt: string;
+    };
+    /** Error message if failed */
+    error?: string;
+}
+/**
+ * Manually index a page's HTML content into database
+ * Use this from custom plugins or API routes to trigger indexing
+ */
+export declare function indexPage(route: string, html: string, options?: IndexPageOptions): Promise<IndexPageResult>;
+/**
+ * Index a page by fetching its HTML first
+ * Convenience wrapper around indexPage that handles fetching
+ */
+export declare function indexPageByRoute(route: string, event?: H3Event, options?: IndexPageOptions): Promise<IndexPageResult>;

package/dist/runtime/server/utils/indexPage.js

@@ -0,0 +1,76 @@
+import { useNitroApp, useRuntimeConfig } from "nitropack/runtime";
+import { useDatabase } from "../db/index.js";
+import { getPage, isPageFresh, upsertPage } from "../db/queries.js";
+import { logger } from "../logger.js";
+import { convertHtmlToMarkdownMeta } from "../utils.js";
+import { extractKeywords, stripMarkdown } from "./keywords.js";
+export async function indexPage(route, html, options = {}) {
+  const config = useRuntimeConfig()["nuxt-ai-ready"];
+  const db = await useDatabase();
+  const ttl = options.ttl ?? config.ttl ?? 0;
+  if (!options.force && await isPageFresh(db, route, ttl)) {
+    logger.debug(`[indexPage] Skipping ${route} - still fresh`);
+    return { success: true, skipped: true, isUpdate: true };
+  }
+  const existing = await getPage(db, route);
+  const isUpdate = !!existing;
+  const isError = html.includes("__NUXT_ERROR__") || html.includes("nuxt-error-page");
+  const result = await convertHtmlToMarkdownMeta(html, route, config.mdreamOptions);
+  const updatedAt = result.updatedAt || (/* @__PURE__ */ new Date()).toISOString();
+  const headings = JSON.stringify(result.headings);
+  const keywords = extractKeywords(stripMarkdown(result.markdown), result.metaKeywords);
+  await upsertPage(db, {
+    route,
+    title: result.title,
+    description: result.description,
+    markdown: result.markdown,
+    headings,
+    keywords,
+    updatedAt,
+    isError
+  });
+  logger.debug(`[indexPage] Indexed: ${route} "${result.title}"`);
+  if (!options.skipHook) {
+    const nitro = useNitroApp();
+    const hookContext = {
+      route,
+      title: result.title,
+      description: result.description,
+      headings,
+      keywords,
+      markdown: result.markdown,
+      updatedAt,
+      isUpdate
+    };
+    await nitro.hooks.callHook("ai-ready:page:indexed", hookContext);
+  }
+  return {
+    success: true,
+    isUpdate,
+    data: {
+      route,
+      title: result.title,
+      description: result.description,
+      headings,
+      keywords,
+      markdown: result.markdown,
+      updatedAt
+    }
+  };
+}
+export async function indexPageByRoute(route, event, options = {}) {
+  const fetchFn = event?.fetch || globalThis.$fetch;
+  const html = await fetchFn(route, {
+    headers: { "x-ai-ready-internal": "1" }
+  }).catch((err) => {
+    logger.error(`[indexPageByRoute] Failed to fetch ${route}:`, err);
+    return null;
+  });
+  if (!html) {
+    return { success: false, error: `Failed to fetch HTML for ${route}` };
+  }
+  if (typeof html !== "string") {
+    return { success: false, error: `Response for ${route} is not HTML` };
+  }
+  return indexPage(route, html, options);
+}

package/dist/runtime/server/utils/keywords.d.ts

@@ -0,0 +1,8 @@
+/**
+ * Extract keywords from text using word frequency
+ */
+export declare function extractKeywords(text: string, metaKeywords?: string, max?: number): string[];
+/**
+ * Strip markdown formatting to get plain text
+ */
+export declare function stripMarkdown(markdown: string): string;